mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #430 from GuillaumeGomez/download-config

Browse Source 
Add `download` config
This commit is contained in:
antoyo 2024-02-13 10:49:10 -05:00 committed by GitHub
commit ac708d9485
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 306 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
.github/workflows/ci.yml vendored
View File

@ -49,19 +49,10 @@ jobs:
# `llvm-14-tools` is needed to install the `FileCheck` binary which is used for asm tests.
run: sudo apt-get install ninja-build ripgrep llvm-14-tools
- name: Download artifact
run: curl -LO https://github.com/antoyo/gcc/releases/latest/download/gcc-13.deb
- name: Setup path to libgccjit
run: |
sudo dpkg --force-overwrite -i gcc-13.deb
echo 'gcc-path = "/usr/lib/"' > config.toml
- name: Set env
run: |
echo "workspace="$GITHUB_WORKSPACE >> $GITHUB_ENV
echo "LIBRARY_PATH=/usr/lib" >> $GITHUB_ENV
echo "LD_LIBRARY_PATH=/usr/lib" >> $GITHUB_ENV
echo 'download-gccjit = true' > config.toml
#- name: Cache rust repository
## We only clone the rust repository for rustc tests
@ -78,6 +69,15 @@ jobs:
# TODO: remove --features master when it is back to the default.
./y.sh build --features master
# TODO: remove --features master when it is back to the default.
- name: Set env (part 2)
run: |
# Set the `LD_LIBRARY_PATH` and `LIBRARY_PATH` env variables...
echo "LD_LIBRARY_PATH="$(./y.sh info | grep -v Using) >> $GITHUB_ENV
echo "LIBRARY_PATH="$(./y.sh info | grep -v Using) >> $GITHUB_ENV
- name: Build (part 2)
run: |
cargo test --features master
./y.sh clean all

4
build_system/src/build.rs
View File

@ -207,7 +207,7 @@ fn build_codegen(args: &mut BuildArg) -> Result<(), String> {
}
run_command_with_output_and_env(&command, None, Some(&env))?;
args.config_info.setup(&mut env, None)?;
args.config_info.setup(&mut env, false)?;
// We voluntarily ignore the error.
let _ = fs::remove_dir_all("target/out");
@ -229,7 +229,7 @@ pub fn run() -> Result<(), String> {
Some(args) => args,
None => return Ok(()),
};
args.config_info.setup_gcc_path(None)?;
args.config_info.setup_gcc_path()?;
build_codegen(&mut args)?;
Ok(())
}

2
build_system/src/cargo.rs
View File

@ -77,7 +77,7 @@ pub fn run() -> Result<(), String> {
})?;
let mut env: HashMap<String, String> = std::env::vars().collect();
ConfigInfo::default().setup(&mut env, None)?;
ConfigInfo::default().setup(&mut env, false)?;
let toolchain = get_toolchain()?;
let toolchain_version = rustc_toolchain_version_info(&toolchain)?;

260
build_system/src/config.rs
View File

@ -1,9 +1,11 @@
use crate::utils::{get_os_name, rustc_version_info, split_args};
use crate::utils::{
create_symlink, get_os_name, run_command_with_output, rustc_version_info, split_args,
};
use std::collections::HashMap;
use std::env as std_env;
use std::ffi::OsStr;
use std::fs;
use std::path::Path;
use std::path::{Path, PathBuf};
use boml::{types::TomlValue, Toml};
@ -23,8 +25,12 @@ impl Channel {
}
}
fn failed_config_parsing(config_file: &str, err: &str) -> Result<ConfigFile, String> {
Err(format!("Failed to parse `{}`: {}", config_file, err))
fn failed_config_parsing(config_file: &Path, err: &str) -> Result<ConfigFile, String> {
Err(format!(
"Failed to parse `{}`: {}",
config_file.display(),
err
))
}
#[derive(Default)]
@ -34,12 +40,11 @@ pub struct ConfigFile {
}
impl ConfigFile {
pub fn new(config_file: Option<&str>) -> Result<Self, String> {
let config_file = config_file.unwrap_or("config.toml");
pub fn new(config_file: &Path) -> Result<Self, String> {
let content = fs::read_to_string(config_file).map_err(|_| {
format!(
"Failed to read `{}`. Take a look at `Readme.md` to see how to set up the project",
config_file,
config_file.display(),
)
})?;
let toml = Toml::parse(&content).map_err(|err| {
@ -70,19 +75,30 @@ impl ConfigFile {
_ => return failed_config_parsing(config_file, &format!("Unknown key `{}`", key)),
}
}
if config.gcc_path.is_none() && config.download_gccjit.is_none() {
return failed_config_parsing(
config_file,
"At least one of `gcc-path` or `download-gccjit` value must be set",
);
}
if let Some(gcc_path) = config.gcc_path.as_mut() {
let path = Path::new(gcc_path);
*gcc_path = path
.canonicalize()
.map_err(|err| format!("Failed to get absolute path of `{}`: {:?}", gcc_path, err))?
.display()
.to_string();
match (config.gcc_path.as_mut(), config.download_gccjit) {
(None, None | Some(false)) => {
return failed_config_parsing(
config_file,
"At least one of `gcc-path` or `download-gccjit` value must be set",
)
}
(Some(_), Some(true)) => {
println!(
"WARNING: both `gcc-path` and `download-gccjit` arguments are used, \
ignoring `gcc-path`"
);
}
(Some(gcc_path), _) => {
let path = Path::new(gcc_path);
*gcc_path = path
.canonicalize()
.map_err(|err| {
format!("Failed to get absolute path of `{}`: {:?}", gcc_path, err)
})?
.display()
.to_string();
}
_ => {}
}
Ok(config)
}
@ -104,6 +120,13 @@ pub struct ConfigInfo {
pub sysroot_path: String,
pub gcc_path: String,
config_file: Option<String>,
// This is used in particular in rust compiler bootstrap because it doesn't run at the root
// of the `cg_gcc` folder, making it complicated for us to get access to local files we need
// like `libgccjit.version` or `config.toml`.
cg_gcc_path: Option<PathBuf>,
// Needed for the `info` command which doesn't want to actually download the lib if needed,
// just to set the `gcc_path` field to display it.
pub no_download: bool,
}
impl ConfigInfo {
@ -146,6 +169,14 @@ impl ConfigInfo {
"--release-sysroot" => self.sysroot_release_channel = true,
"--release" => self.channel = Channel::Release,
"--sysroot-panic-abort" => self.sysroot_panic_abort = true,
"--cg_gcc-path" => match args.next() {
Some(arg) if !arg.is_empty() => {
self.cg_gcc_path = Some(arg.into());
}
_ => {
return Err("Expected a value after `--cg_gcc-path`, found nothing".to_string())
}
},
_ => return Ok(false),
}
Ok(true)
@ -159,30 +190,165 @@ impl ConfigInfo {
command
}
pub fn setup_gcc_path(&mut self, override_gcc_path: Option<&str>) -> Result<(), String> {
let ConfigFile { gcc_path, .. } = ConfigFile::new(self.config_file.as_deref())?;
fn download_gccjit(
&self,
output_dir: &Path,
libgccjit_so_name: &str,
commit: &str,
) -> Result<(), String> {
// Download time!
let tempfile_name = format!("{}.download", libgccjit_so_name);
let tempfile = output_dir.join(&tempfile_name);
let is_in_ci = std::env::var("GITHUB_ACTIONS").is_ok();
self.gcc_path = match override_gcc_path {
Some(path) => {
if gcc_path.is_some() {
println!(
"overriding setting from `{}`",
self.config_file.as_deref().unwrap_or("config.toml")
);
}
path.to_string()
}
let url = format!(
"https://github.com/antoyo/gcc/releases/download/master-{}/libgccjit.so",
commit,
);
println!("Downloading `{}`...", url);
// Try curl. If that fails and we are on windows, fallback to PowerShell.
let mut ret = run_command_with_output(
&[
&"curl",
&"--speed-time",
&"30",
&"--speed-limit",
&"10", // timeout if speed is < 10 bytes/sec for > 30 seconds
&"--connect-timeout",
&"30", // timeout if cannot connect within 30 seconds
&"-o",
&tempfile_name,
&"--retry",
&"3",
&"-SRfL",
if is_in_ci { &"-s" } else { &"--progress-bar" },
&url.as_str(),
],
Some(&output_dir),
);
if ret.is_err() && cfg!(windows) {
eprintln!("Fallback to PowerShell");
ret = run_command_with_output(
&[
&"PowerShell.exe",
&"/nologo",
&"-Command",
&"[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
&format!(
"(New-Object System.Net.WebClient).DownloadFile('{}', '{}')",
url,
tempfile_name,
).as_str(),
],
Some(&output_dir),
);
}
ret?;
let libgccjit_so = output_dir.join(libgccjit_so_name);
// If we reach this point, it means the file was correctly downloaded, so let's
// rename it!
std::fs::rename(&tempfile, &libgccjit_so).map_err(|err| {
format!(
"Failed to rename `{}` into `{}`: {:?}",
tempfile.display(),
libgccjit_so.display(),
err,
)
})?;
println!("Downloaded libgccjit.so version {} successfully!", commit);
// We need to create a link named `libgccjit.so.0` because that's what the linker is
// looking for.
create_symlink(
&libgccjit_so,
output_dir.join(&format!("{}.0", libgccjit_so_name)),
)
}
fn download_gccjit_if_needed(&mut self) -> Result<(), String> {
let output_dir = Path::new(
std::env::var("CARGO_TARGET_DIR")
.as_deref()
.unwrap_or("target"),
)
.join("libgccjit");
let commit_hash_file = self.compute_path("libgccjit.version");
let content = fs::read_to_string(&commit_hash_file).map_err(|_| {
format!(
"Failed to read `{}`. Take a look at `Readme.md` to see how to set up the project",
commit_hash_file.display(),
)
})?;
let commit = content.trim();
// This is a very simple check to ensure this is not a path. For the rest, it'll just fail
// when trying to download the file so we should be fine.
if commit.contains('/') || commit.contains('\\') {
return Err(format!(
"{}: invalid commit hash `{}`",
commit_hash_file.display(),
commit,
));
}
let output_dir = output_dir.join(commit);
if !output_dir.is_dir() {
std::fs::create_dir_all(&output_dir).map_err(|err| {
format!(
"failed to create folder `{}`: {:?}",
output_dir.display(),
err,
)
})?;
}
let output_dir = output_dir.canonicalize().map_err(|err| {
format!(
"Failed to get absolute path of `{}`: {:?}",
output_dir.display(),
err
)
})?;
let libgccjit_so_name = "libgccjit.so";
let libgccjit_so = output_dir.join(libgccjit_so_name);
if !libgccjit_so.is_file() && !self.no_download {
self.download_gccjit(&output_dir, libgccjit_so_name, commit)?;
}
self.gcc_path = output_dir.display().to_string();
println!("Using `{}` as path for libgccjit", self.gcc_path);
Ok(())
}
pub fn compute_path<P: AsRef<Path>>(&self, other: P) -> PathBuf {
match self.cg_gcc_path {
Some(ref path) => path.join(other),
None => PathBuf::new().join(other),
}
}
pub fn setup_gcc_path(&mut self) -> Result<(), String> {
let config_file = match self.config_file.as_deref() {
Some(config_file) => config_file.into(),
None => self.compute_path("config.toml"),
};
let ConfigFile {
gcc_path,
download_gccjit,
} = ConfigFile::new(&config_file)?;
if let Some(true) = download_gccjit {
self.download_gccjit_if_needed()?;
return Ok(());
}
self.gcc_path = match gcc_path {
Some(path) => path,
None => {
match gcc_path {
Some(path) => path,
// FIXME: Once we support "download", rewrite this.
None => {
return Err(format!(
"missing `gcc-path` value from `{}`",
self.config_file.as_deref().unwrap_or("config.toml"),
))
}
}
return Err(format!(
"missing `gcc-path` value from `{}`",
config_file.display(),
))
}
};
Ok(())
@ -191,12 +357,12 @@ impl ConfigInfo {
pub fn setup(
&mut self,
env: &mut HashMap<String, String>,
override_gcc_path: Option<&str>,
use_system_gcc: bool,
) -> Result<(), String> {
env.insert("CARGO_INCREMENTAL".to_string(), "0".to_string());
if self.gcc_path.is_empty() || override_gcc_path.is_some() {
self.setup_gcc_path(override_gcc_path)?;
if self.gcc_path.is_empty() && !use_system_gcc {
self.setup_gcc_path()?;
}
env.insert("GCC_PATH".to_string(), self.gcc_path.clone());
@ -375,7 +541,9 @@ impl ConfigInfo {
--release : Build in release mode
--release-sysroot : Build sysroot in release mode
--sysroot-panic-abort : Build the sysroot without unwinding support
--config-file : Location of the config file to be used"
--config-file : Location of the config file to be used
--cg_gcc-path : Location of the rustc_codegen_gcc root folder (used
when ran from another directory)"
);
}
}

19
build_system/src/info.rs Normal file
View File

@ -0,0 +1,19 @@
use crate::config::ConfigInfo;
pub fn run() -> Result<(), String> {
let mut config = ConfigInfo::default();
// We skip binary name and the `info` command.
let mut args = std::env::args().skip(2);
while let Some(arg) = args.next() {
if arg == "--help" {
println!("Display the path where the libgccjit will be located");
return Ok(());
}
config.parse_argument(&arg, &mut args)?;
}
config.no_download = true;
config.setup_gcc_path()?;
println!("{}", config.gcc_path);
Ok(())
}

5
build_system/src/main.rs
View File

@ -5,6 +5,7 @@ mod build;
mod cargo;
mod clean;
mod config;
mod info;
mod prepare;
mod rustc_info;
mod test;
@ -29,6 +30,7 @@ Available commands for build_system:
prepare : Run prepare command
build : Run build command
test : Run test command
info: : Run info command
--help : Show this message"
);
}
@ -39,6 +41,7 @@ pub enum Command {
Prepare,
Build,
Test,
Info,
}
fn main() {
@ -52,6 +55,7 @@ fn main() {
Some("prepare") => Command::Prepare,
Some("build") => Command::Build,
Some("test") => Command::Test,
Some("info") => Command::Info,
Some("--help") => {
usage();
process::exit(0);
@ -70,6 +74,7 @@ fn main() {
Command::Prepare => prepare::run(),
Command::Build => build::run(),
Command::Test => test::run(),
Command::Info => info::run(),
} {
eprintln!("Command failed to run: {e}");
process::exit(1);

45
build_system/src/test.rs
View File

@ -31,7 +31,10 @@ fn get_runners() -> Runners {
"--test-failing-rustc",
("Run failing rustc tests", test_failing_rustc),
);
runners.insert("--projects", ("Run the tests of popular crates", test_projects));
runners.insert(
"--projects",
("Run the tests of popular crates", test_projects),
);
runners.insert("--test-libcore", ("Run libcore tests", test_libcore));
runners.insert("--clean", ("Empty cargo target directory", clean));
runners.insert("--build-sysroot", ("Build sysroot", build_sysroot));
@ -109,7 +112,7 @@ fn show_usage() {
struct TestArg {
no_default_features: bool,
build_only: bool,
gcc_path: Option<String>,
use_system_gcc: bool,
runners: BTreeSet<String>,
flags: Vec<String>,
backend: Option<String>,
@ -121,7 +124,6 @@ struct TestArg {
impl TestArg {
fn new() -> Result<Option<Self>, String> {
let mut use_system_gcc = false;
let mut test_arg = Self::default();
// We skip binary name and the `test` command.
@ -147,7 +149,10 @@ impl TestArg {
return Err("Expected an argument after `--features`, found nothing".into())
}
},
"--use-system-gcc" => use_system_gcc = true,
"--use-system-gcc" => {
println!("Using system GCC");
test_arg.use_system_gcc = true;
}
"--build-only" => test_arg.build_only = true,
"--use-backend" => match args.next() {
Some(backend) if !backend.is_empty() => test_arg.backend = Some(backend),
@ -180,11 +185,6 @@ impl TestArg {
}
}
}
if use_system_gcc {
println!("Using system GCC");
test_arg.gcc_path = Some("gcc".to_string());
}
}
match (test_arg.current_part, test_arg.nb_parts) {
(Some(_), Some(_)) | (None, None) => {}
@ -703,7 +703,7 @@ fn test_projects(env: &Env, args: &TestArg) -> Result<(), String> {
//"https://github.com/rust-lang/cargo", // TODO: very slow, only run on master?
];
let run_tests = |projects_path, iter: &mut dyn Iterator<Item=&&str>| -> Result<(), String> {
let run_tests = |projects_path, iter: &mut dyn Iterator<Item = &&str>| -> Result<(), String> {
for project in iter {
let clone_result = git_clone(project, Some(projects_path), true)?;
let repo_path = Path::new(&clone_result.repo_dir);
@ -727,8 +727,7 @@ fn test_projects(env: &Env, args: &TestArg) -> Result<(), String> {
let start = current_part * count;
// We remove the projects we don't want to test.
run_tests(projects_path, &mut projects.iter().skip(start).take(count))?;
}
else {
} else {
run_tests(projects_path, &mut projects.iter())?;
}
@ -1166,15 +1165,17 @@ pub fn run() -> Result<(), String> {
};
let mut env: HashMap<String, String> = std::env::vars().collect();
args.config_info.setup_gcc_path(None)?;
env.insert(
"LIBRARY_PATH".to_string(),
args.config_info.gcc_path.clone(),
);
env.insert(
"LD_LIBRARY_PATH".to_string(),
args.config_info.gcc_path.clone(),
);
if !args.use_system_gcc {
args.config_info.setup_gcc_path()?;
env.insert(
"LIBRARY_PATH".to_string(),
args.config_info.gcc_path.clone(),
);
env.insert(
"LD_LIBRARY_PATH".to_string(),
args.config_info.gcc_path.clone(),
);
}
build_if_no_backend(&env, &args)?;
if args.build_only {
@ -1182,7 +1183,7 @@ pub fn run() -> Result<(), String> {
return Ok(());
}
args.config_info.setup(&mut env, args.gcc_path.as_deref())?;
args.config_info.setup(&mut env, args.use_system_gcc)?;
if args.runners.is_empty() {
run_all(&env, &args)?;

22
build_system/src/utils.rs
View File

@ -254,7 +254,11 @@ pub struct CloneResult {
pub repo_dir: String,
}
pub fn git_clone(to_clone: &str, dest: Option<&Path>, shallow_clone: bool) -> Result<CloneResult, String> {
pub fn git_clone(
to_clone: &str,
dest: Option<&Path>,
shallow_clone: bool,
) -> Result<CloneResult, String> {
let repo_name = to_clone.split('/').last().unwrap();
let repo_name = match repo_name.strip_suffix(".git") {
Some(n) => n.to_string(),
@ -370,6 +374,22 @@ pub fn remove_file<P: AsRef<Path> + ?Sized>(file_path: &P) -> Result<(), String>
})
}
pub fn create_symlink<P: AsRef<Path>, Q: AsRef<Path>>(original: P, link: Q) -> Result<(), String> {
#[cfg(windows)]
let symlink = std::os::windows::fs::symlink_file;
#[cfg(not(windows))]
let symlink = std::os::unix::fs::symlink;
symlink(&original, &link).map_err(|err| {
format!(
"failed to create a symlink `{}` to `{}`: {:?}",
original.as_ref().display(),
link.as_ref().display(),
err,
)
})
}
#[cfg(test)]
mod tests {
use super::*;

1
libgccjit.version Normal file
View File

@ -0,0 +1 @@
2fc8940e1

15
tests/lang_tests_common.rs
View File

@ -21,11 +21,16 @@ pub fn main_inner(profile: Profile) {
let tempdir = TempDir::new().expect("temp dir");
let current_dir = current_dir().expect("current dir");
let current_dir = current_dir.to_str().expect("current dir").to_string();
let gcc_path = Toml::parse(include_str!("../config.toml"))
.expect("Failed to parse `config.toml`")
.get_string("gcc-path")
.expect("Missing `gcc-path` key in `config.toml`")
.to_string();
let toml = Toml::parse(include_str!("../config.toml"))
.expect("Failed to parse `config.toml`");
let gcc_path = if let Ok(gcc_path) = toml.get_string("gcc-path") {
PathBuf::from(gcc_path.to_string())
} else {
// then we try to retrieve it from the `target` folder.
let commit = include_str!("../libgccjit.version").trim();
Path::new("target/libgccjit").join(commit)
};
let gcc_path = Path::new(&gcc_path)
.canonicalize()
.expect("failed to get absolute path of `gcc-path`")