Merge pull request #430 from GuillaumeGomez/download-config

Add `download` config

.github/workflows/ci.yml

@@ -49,19 +49,10 @@ jobs:
       # `llvm-14-tools` is needed to install the `FileCheck` binary which is used for asm tests.
       run: sudo apt-get install ninja-build ripgrep llvm-14-tools
 
-    - name: Download artifact
-      run: curl -LO https://github.com/antoyo/gcc/releases/latest/download/gcc-13.deb
-
-    - name: Setup path to libgccjit
-      run: |
-          sudo dpkg --force-overwrite -i gcc-13.deb
-          echo 'gcc-path = "/usr/lib/"' > config.toml
-
     - name: Set env
       run: |
         echo "workspace="$GITHUB_WORKSPACE >> $GITHUB_ENV
-        echo "LIBRARY_PATH=/usr/lib" >> $GITHUB_ENV
+        echo 'download-gccjit = true' > config.toml
-        echo "LD_LIBRARY_PATH=/usr/lib" >> $GITHUB_ENV
 
     #- name: Cache rust repository
       ## We only clone the rust repository for rustc tests
@@ -78,6 +69,15 @@ jobs:
         # TODO: remove --features master when it is back to the default.
         ./y.sh build --features master
         # TODO: remove --features master when it is back to the default.
+
+    - name: Set env (part 2)
+      run: |
+        # Set the `LD_LIBRARY_PATH` and `LIBRARY_PATH` env variables...
+        echo "LD_LIBRARY_PATH="$(./y.sh info | grep -v Using) >> $GITHUB_ENV
+        echo "LIBRARY_PATH="$(./y.sh info | grep -v Using) >> $GITHUB_ENV
+
+    - name: Build (part 2)
+      run: |
         cargo test --features master
         ./y.sh clean all
 

build_system/src/build.rs

@@ -207,7 +207,7 @@ fn build_codegen(args: &mut BuildArg) -> Result<(), String> {
     }
     run_command_with_output_and_env(&command, None, Some(&env))?;
 
-    args.config_info.setup(&mut env, None)?;
+    args.config_info.setup(&mut env, false)?;
 
     // We voluntarily ignore the error.
     let _ = fs::remove_dir_all("target/out");
@@ -229,7 +229,7 @@ pub fn run() -> Result<(), String> {
         Some(args) => args,
         None => return Ok(()),
     };
-    args.config_info.setup_gcc_path(None)?;
+    args.config_info.setup_gcc_path()?;
     build_codegen(&mut args)?;
     Ok(())
 }

build_system/src/cargo.rs

@@ -77,7 +77,7 @@ pub fn run() -> Result<(), String> {
     })?;
 
     let mut env: HashMap<String, String> = std::env::vars().collect();
-    ConfigInfo::default().setup(&mut env, None)?;
+    ConfigInfo::default().setup(&mut env, false)?;
     let toolchain = get_toolchain()?;
 
     let toolchain_version = rustc_toolchain_version_info(&toolchain)?;

build_system/src/config.rs

@@ -1,9 +1,11 @@
-use crate::utils::{get_os_name, rustc_version_info, split_args};
+use crate::utils::{
+    create_symlink, get_os_name, run_command_with_output, rustc_version_info, split_args,
+};
 use std::collections::HashMap;
 use std::env as std_env;
 use std::ffi::OsStr;
 use std::fs;
-use std::path::Path;
+use std::path::{Path, PathBuf};
 
 use boml::{types::TomlValue, Toml};
 
@@ -23,8 +25,12 @@ impl Channel {
     }
 }
 
-fn failed_config_parsing(config_file: &str, err: &str) -> Result<ConfigFile, String> {
+fn failed_config_parsing(config_file: &Path, err: &str) -> Result<ConfigFile, String> {
-    Err(format!("Failed to parse `{}`: {}", config_file, err))
+    Err(format!(
+        "Failed to parse `{}`: {}",
+        config_file.display(),
+        err
+    ))
 }
 
 #[derive(Default)]
@@ -34,12 +40,11 @@ pub struct ConfigFile {
 }
 
 impl ConfigFile {
-    pub fn new(config_file: Option<&str>) -> Result<Self, String> {
+    pub fn new(config_file: &Path) -> Result<Self, String> {
-        let config_file = config_file.unwrap_or("config.toml");
         let content = fs::read_to_string(config_file).map_err(|_| {
             format!(
                 "Failed to read `{}`. Take a look at `Readme.md` to see how to set up the project",
-                config_file,
+                config_file.display(),
             )
         })?;
         let toml = Toml::parse(&content).map_err(|err| {
@@ -70,20 +75,31 @@ impl ConfigFile {
                 _ => return failed_config_parsing(config_file, &format!("Unknown key `{}`", key)),
             }
         }
-        if config.gcc_path.is_none() && config.download_gccjit.is_none() {
+        match (config.gcc_path.as_mut(), config.download_gccjit) {
+            (None, None | Some(false)) => {
                 return failed_config_parsing(
                     config_file,
                     "At least one of `gcc-path` or `download-gccjit` value must be set",
+                )
+            }
+            (Some(_), Some(true)) => {
+                println!(
+                    "WARNING: both `gcc-path` and `download-gccjit` arguments are used, \
+                    ignoring `gcc-path`"
                 );
             }
-        if let Some(gcc_path) = config.gcc_path.as_mut() {
+            (Some(gcc_path), _) => {
                 let path = Path::new(gcc_path);
                 *gcc_path = path
                     .canonicalize()
-                .map_err(|err| format!("Failed to get absolute path of `{}`: {:?}", gcc_path, err))?
+                    .map_err(|err| {
+                        format!("Failed to get absolute path of `{}`: {:?}", gcc_path, err)
+                    })?
                     .display()
                     .to_string();
             }
+            _ => {}
+        }
         Ok(config)
     }
 }
@@ -104,6 +120,13 @@ pub struct ConfigInfo {
     pub sysroot_path: String,
     pub gcc_path: String,
     config_file: Option<String>,
+    // This is used in particular in rust compiler bootstrap because it doesn't run at the root
+    // of the `cg_gcc` folder, making it complicated for us to get access to local files we need
+    // like `libgccjit.version` or `config.toml`.
+    cg_gcc_path: Option<PathBuf>,
+    // Needed for the `info` command which doesn't want to actually download the lib if needed,
+    // just to set the `gcc_path` field to display it.
+    pub no_download: bool,
 }
 
 impl ConfigInfo {
@@ -146,6 +169,14 @@ impl ConfigInfo {
             "--release-sysroot" => self.sysroot_release_channel = true,
             "--release" => self.channel = Channel::Release,
             "--sysroot-panic-abort" => self.sysroot_panic_abort = true,
+            "--cg_gcc-path" => match args.next() {
+                Some(arg) if !arg.is_empty() => {
+                    self.cg_gcc_path = Some(arg.into());
+                }
+                _ => {
+                    return Err("Expected a value after `--cg_gcc-path`, found nothing".to_string())
+                }
+            },
             _ => return Ok(false),
         }
         Ok(true)
@@ -159,31 +190,166 @@ impl ConfigInfo {
         command
     }
 
-    pub fn setup_gcc_path(&mut self, override_gcc_path: Option<&str>) -> Result<(), String> {
+    fn download_gccjit(
-        let ConfigFile { gcc_path, .. } = ConfigFile::new(self.config_file.as_deref())?;
+        &self,
+        output_dir: &Path,
+        libgccjit_so_name: &str,
+        commit: &str,
+    ) -> Result<(), String> {
+        // Download time!
+        let tempfile_name = format!("{}.download", libgccjit_so_name);
+        let tempfile = output_dir.join(&tempfile_name);
+        let is_in_ci = std::env::var("GITHUB_ACTIONS").is_ok();
 
-        self.gcc_path = match override_gcc_path {
+        let url = format!(
-            Some(path) => {
+            "https://github.com/antoyo/gcc/releases/download/master-{}/libgccjit.so",
-                if gcc_path.is_some() {
+            commit,
-                    println!(
+        );
-                        "overriding setting from `{}`",
+
-                        self.config_file.as_deref().unwrap_or("config.toml")
+        println!("Downloading `{}`...", url);
+        // Try curl. If that fails and we are on windows, fallback to PowerShell.
+        let mut ret = run_command_with_output(
+            &[
+                &"curl",
+                &"--speed-time",
+                &"30",
+                &"--speed-limit",
+                &"10", // timeout if speed is < 10 bytes/sec for > 30 seconds
+                &"--connect-timeout",
+                &"30", // timeout if cannot connect within 30 seconds
+                &"-o",
+                &tempfile_name,
+                &"--retry",
+                &"3",
+                &"-SRfL",
+                if is_in_ci { &"-s" } else { &"--progress-bar" },
+                &url.as_str(),
+            ],
+            Some(&output_dir),
+        );
+        if ret.is_err() && cfg!(windows) {
+            eprintln!("Fallback to PowerShell");
+            ret = run_command_with_output(
+                &[
+                    &"PowerShell.exe",
+                    &"/nologo",
+                    &"-Command",
+                    &"[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
+                    &format!(
+                        "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')",
+                        url,
+                        tempfile_name,
+                    ).as_str(),
+                ],
+                Some(&output_dir),
             );
         }
-                path.to_string()
+        ret?;
+
+        let libgccjit_so = output_dir.join(libgccjit_so_name);
+        // If we reach this point, it means the file was correctly downloaded, so let's
+        // rename it!
+        std::fs::rename(&tempfile, &libgccjit_so).map_err(|err| {
+            format!(
+                "Failed to rename `{}` into `{}`: {:?}",
+                tempfile.display(),
+                libgccjit_so.display(),
+                err,
+            )
+        })?;
+
+        println!("Downloaded libgccjit.so version {} successfully!", commit);
+        // We need to create a link named `libgccjit.so.0` because that's what the linker is
+        // looking for.
+        create_symlink(
+            &libgccjit_so,
+            output_dir.join(&format!("{}.0", libgccjit_so_name)),
+        )
     }
-            None => {
+
-                match gcc_path {
+    fn download_gccjit_if_needed(&mut self) -> Result<(), String> {
+        let output_dir = Path::new(
+            std::env::var("CARGO_TARGET_DIR")
+                .as_deref()
+                .unwrap_or("target"),
+        )
+        .join("libgccjit");
+
+        let commit_hash_file = self.compute_path("libgccjit.version");
+        let content = fs::read_to_string(&commit_hash_file).map_err(|_| {
+            format!(
+                "Failed to read `{}`. Take a look at `Readme.md` to see how to set up the project",
+                commit_hash_file.display(),
+            )
+        })?;
+        let commit = content.trim();
+        // This is a very simple check to ensure this is not a path. For the rest, it'll just fail
+        // when trying to download the file so we should be fine.
+        if commit.contains('/') || commit.contains('\\') {
+            return Err(format!(
+                "{}: invalid commit hash `{}`",
+                commit_hash_file.display(),
+                commit,
+            ));
+        }
+        let output_dir = output_dir.join(commit);
+        if !output_dir.is_dir() {
+            std::fs::create_dir_all(&output_dir).map_err(|err| {
+                format!(
+                    "failed to create folder `{}`: {:?}",
+                    output_dir.display(),
+                    err,
+                )
+            })?;
+        }
+        let output_dir = output_dir.canonicalize().map_err(|err| {
+            format!(
+                "Failed to get absolute path of `{}`: {:?}",
+                output_dir.display(),
+                err
+            )
+        })?;
+
+        let libgccjit_so_name = "libgccjit.so";
+        let libgccjit_so = output_dir.join(libgccjit_so_name);
+        if !libgccjit_so.is_file() && !self.no_download {
+            self.download_gccjit(&output_dir, libgccjit_so_name, commit)?;
+        }
+
+        self.gcc_path = output_dir.display().to_string();
+        println!("Using `{}` as path for libgccjit", self.gcc_path);
+        Ok(())
+    }
+
+    pub fn compute_path<P: AsRef<Path>>(&self, other: P) -> PathBuf {
+        match self.cg_gcc_path {
+            Some(ref path) => path.join(other),
+            None => PathBuf::new().join(other),
+        }
+    }
+
+    pub fn setup_gcc_path(&mut self) -> Result<(), String> {
+        let config_file = match self.config_file.as_deref() {
+            Some(config_file) => config_file.into(),
+            None => self.compute_path("config.toml"),
+        };
+        let ConfigFile {
+            gcc_path,
+            download_gccjit,
+        } = ConfigFile::new(&config_file)?;
+
+        if let Some(true) = download_gccjit {
+            self.download_gccjit_if_needed()?;
+            return Ok(());
+        }
+        self.gcc_path = match gcc_path {
             Some(path) => path,
-                    // FIXME: Once we support "download", rewrite this.
             None => {
                 return Err(format!(
                     "missing `gcc-path` value from `{}`",
-                            self.config_file.as_deref().unwrap_or("config.toml"),
+                    config_file.display(),
                 ))
             }
-                }
-            }
         };
         Ok(())
     }
@@ -191,12 +357,12 @@ impl ConfigInfo {
     pub fn setup(
         &mut self,
         env: &mut HashMap<String, String>,
-        override_gcc_path: Option<&str>,
+        use_system_gcc: bool,
     ) -> Result<(), String> {
         env.insert("CARGO_INCREMENTAL".to_string(), "0".to_string());
 
-        if self.gcc_path.is_empty() || override_gcc_path.is_some() {
+        if self.gcc_path.is_empty() && !use_system_gcc {
-            self.setup_gcc_path(override_gcc_path)?;
+            self.setup_gcc_path()?;
         }
         env.insert("GCC_PATH".to_string(), self.gcc_path.clone());
 
@@ -375,7 +541,9 @@ impl ConfigInfo {
     --release              : Build in release mode
     --release-sysroot      : Build sysroot in release mode
     --sysroot-panic-abort  : Build the sysroot without unwinding support
-    --config-file          : Location of the config file to be used"
+    --config-file          : Location of the config file to be used
+    --cg_gcc-path          : Location of the rustc_codegen_gcc root folder (used
+                             when ran from another directory)"
         );
     }
 }

build_system/src/info.rs

@@ -0,0 +1,19 @@
+use crate::config::ConfigInfo;
+
+pub fn run() -> Result<(), String> {
+    let mut config = ConfigInfo::default();
+
+    // We skip binary name and the `info` command.
+    let mut args = std::env::args().skip(2);
+    while let Some(arg) = args.next() {
+        if arg == "--help" {
+            println!("Display the path where the libgccjit will be located");
+            return Ok(());
+        }
+        config.parse_argument(&arg, &mut args)?;
+    }
+    config.no_download = true;
+    config.setup_gcc_path()?;
+    println!("{}", config.gcc_path);
+    Ok(())
+}

build_system/src/main.rs

@@ -5,6 +5,7 @@ mod build;
 mod cargo;
 mod clean;
 mod config;
+mod info;
 mod prepare;
 mod rustc_info;
 mod test;
@@ -29,6 +30,7 @@ Available commands for build_system:
     prepare  : Run prepare command
     build    : Run build command
     test     : Run test command
+    info:    : Run info command
     --help   : Show this message"
     );
 }
@@ -39,6 +41,7 @@ pub enum Command {
     Prepare,
     Build,
     Test,
+    Info,
 }
 
 fn main() {
@@ -52,6 +55,7 @@ fn main() {
         Some("prepare") => Command::Prepare,
         Some("build") => Command::Build,
         Some("test") => Command::Test,
+        Some("info") => Command::Info,
         Some("--help") => {
             usage();
             process::exit(0);
@@ -70,6 +74,7 @@ fn main() {
         Command::Prepare => prepare::run(),
         Command::Build => build::run(),
         Command::Test => test::run(),
+        Command::Info => info::run(),
     } {
         eprintln!("Command failed to run: {e}");
         process::exit(1);

build_system/src/test.rs

@@ -31,7 +31,10 @@ fn get_runners() -> Runners {
         "--test-failing-rustc",
         ("Run failing rustc tests", test_failing_rustc),
     );
-    runners.insert("--projects", ("Run the tests of popular crates", test_projects));
+    runners.insert(
+        "--projects",
+        ("Run the tests of popular crates", test_projects),
+    );
     runners.insert("--test-libcore", ("Run libcore tests", test_libcore));
     runners.insert("--clean", ("Empty cargo target directory", clean));
     runners.insert("--build-sysroot", ("Build sysroot", build_sysroot));
@@ -109,7 +112,7 @@ fn show_usage() {
 struct TestArg {
     no_default_features: bool,
     build_only: bool,
-    gcc_path: Option<String>,
+    use_system_gcc: bool,
     runners: BTreeSet<String>,
     flags: Vec<String>,
     backend: Option<String>,
@@ -121,7 +124,6 @@ struct TestArg {
 
 impl TestArg {
     fn new() -> Result<Option<Self>, String> {
-        let mut use_system_gcc = false;
         let mut test_arg = Self::default();
 
         // We skip binary name and the `test` command.
@@ -147,7 +149,10 @@ impl TestArg {
                         return Err("Expected an argument after `--features`, found nothing".into())
                     }
                 },
-                "--use-system-gcc" => use_system_gcc = true,
+                "--use-system-gcc" => {
+                    println!("Using system GCC");
+                    test_arg.use_system_gcc = true;
+                }
                 "--build-only" => test_arg.build_only = true,
                 "--use-backend" => match args.next() {
                     Some(backend) if !backend.is_empty() => test_arg.backend = Some(backend),
@@ -180,11 +185,6 @@ impl TestArg {
                     }
                 }
             }
-
-            if use_system_gcc {
-                println!("Using system GCC");
-                test_arg.gcc_path = Some("gcc".to_string());
-            }
         }
         match (test_arg.current_part, test_arg.nb_parts) {
             (Some(_), Some(_)) | (None, None) => {}
@@ -703,7 +703,7 @@ fn test_projects(env: &Env, args: &TestArg) -> Result<(), String> {
         //"https://github.com/rust-lang/cargo", // TODO: very slow, only run on master?
     ];
 
-    let run_tests = |projects_path, iter: &mut dyn Iterator<Item=&&str>| -> Result<(), String> {
+    let run_tests = |projects_path, iter: &mut dyn Iterator<Item = &&str>| -> Result<(), String> {
         for project in iter {
             let clone_result = git_clone(project, Some(projects_path), true)?;
             let repo_path = Path::new(&clone_result.repo_dir);
@@ -727,8 +727,7 @@ fn test_projects(env: &Env, args: &TestArg) -> Result<(), String> {
         let start = current_part * count;
         // We remove the projects we don't want to test.
         run_tests(projects_path, &mut projects.iter().skip(start).take(count))?;
-    }
+    } else {
-    else {
         run_tests(projects_path, &mut projects.iter())?;
     }
 
@@ -1166,7 +1165,8 @@ pub fn run() -> Result<(), String> {
     };
     let mut env: HashMap<String, String> = std::env::vars().collect();
 
-    args.config_info.setup_gcc_path(None)?;
+    if !args.use_system_gcc {
+        args.config_info.setup_gcc_path()?;
         env.insert(
             "LIBRARY_PATH".to_string(),
             args.config_info.gcc_path.clone(),
@@ -1175,6 +1175,7 @@ pub fn run() -> Result<(), String> {
             "LD_LIBRARY_PATH".to_string(),
             args.config_info.gcc_path.clone(),
         );
+    }
 
     build_if_no_backend(&env, &args)?;
     if args.build_only {
@@ -1182,7 +1183,7 @@ pub fn run() -> Result<(), String> {
         return Ok(());
     }
 
-    args.config_info.setup(&mut env, args.gcc_path.as_deref())?;
+    args.config_info.setup(&mut env, args.use_system_gcc)?;
 
     if args.runners.is_empty() {
         run_all(&env, &args)?;

build_system/src/utils.rs

@@ -254,7 +254,11 @@ pub struct CloneResult {
     pub repo_dir: String,
 }
 
-pub fn git_clone(to_clone: &str, dest: Option<&Path>, shallow_clone: bool) -> Result<CloneResult, String> {
+pub fn git_clone(
+    to_clone: &str,
+    dest: Option<&Path>,
+    shallow_clone: bool,
+) -> Result<CloneResult, String> {
     let repo_name = to_clone.split('/').last().unwrap();
     let repo_name = match repo_name.strip_suffix(".git") {
         Some(n) => n.to_string(),
@@ -370,6 +374,22 @@ pub fn remove_file<P: AsRef<Path> + ?Sized>(file_path: &P) -> Result<(), String>
     })
 }
 
+pub fn create_symlink<P: AsRef<Path>, Q: AsRef<Path>>(original: P, link: Q) -> Result<(), String> {
+    #[cfg(windows)]
+    let symlink = std::os::windows::fs::symlink_file;
+    #[cfg(not(windows))]
+    let symlink = std::os::unix::fs::symlink;
+
+    symlink(&original, &link).map_err(|err| {
+        format!(
+            "failed to create a symlink `{}` to `{}`: {:?}",
+            original.as_ref().display(),
+            link.as_ref().display(),
+            err,
+        )
+    })
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

libgccjit.version

@@ -0,0 +1 @@
+2fc8940e1

tests/lang_tests_common.rs

@@ -21,11 +21,16 @@ pub fn main_inner(profile: Profile) {
     let tempdir = TempDir::new().expect("temp dir");
     let current_dir = current_dir().expect("current dir");
     let current_dir = current_dir.to_str().expect("current dir").to_string();
-    let gcc_path = Toml::parse(include_str!("../config.toml"))
+    let toml = Toml::parse(include_str!("../config.toml"))
-        .expect("Failed to parse `config.toml`")
+        .expect("Failed to parse `config.toml`");
-        .get_string("gcc-path")
+    let gcc_path = if let Ok(gcc_path) = toml.get_string("gcc-path") {
-        .expect("Missing `gcc-path` key in `config.toml`")
+        PathBuf::from(gcc_path.to_string())
-        .to_string();
+    } else {
+        // then we try to retrieve it from the `target` folder.
+        let commit = include_str!("../libgccjit.version").trim();
+        Path::new("target/libgccjit").join(commit)
+    };
+
     let gcc_path = Path::new(&gcc_path)
         .canonicalize()
         .expect("failed to get absolute path of `gcc-path`")