Merge pull request #427 from GuillaumeGomez/config-file

Switch to `config.toml` instead of `gcc-path`
This commit is contained in:
antoyo 2024-02-11 17:47:56 -05:00 committed by GitHub
commit 560e65c323
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
18 changed files with 227 additions and 90 deletions

View File

@ -55,13 +55,13 @@ jobs:
- name: Setup path to libgccjit
run: |
sudo dpkg --force-overwrite -i gcc-13.deb
echo /usr/lib/ > gcc_path
echo 'gcc-path = "/usr/lib/"' > config.toml
- name: Set env
run: |
echo "LIBRARY_PATH=$(cat gcc_path)" >> $GITHUB_ENV
echo "LD_LIBRARY_PATH=$(cat gcc_path)" >> $GITHUB_ENV
echo "workspace="$GITHUB_WORKSPACE >> $GITHUB_ENV
echo "LIBRARY_PATH=/usr/lib" >> $GITHUB_ENV
echo "LD_LIBRARY_PATH=/usr/lib" >> $GITHUB_ENV
#- name: Cache rust repository
## We only clone the rust repository for rustc tests

View File

@ -52,7 +52,10 @@ jobs:
- name: Setup path to libgccjit
if: matrix.libgccjit_version.gcc == 'libgccjit12.so'
run: echo /usr/lib/gcc/x86_64-linux-gnu/12 > gcc_path
run: |
echo 'gcc-path = "/usr/lib/gcc/x86_64-linux-gnu/12"' > config.toml
echo "LIBRARY_PATH=/usr/lib/gcc/x86_64-linux-gnu/12" >> $GITHUB_ENV
echo "LD_LIBRARY_PATH=/usr/lib/gcc/x86_64-linux-gnu/12" >> $GITHUB_ENV
- name: Download artifact
if: matrix.libgccjit_version.gcc != 'libgccjit12.so'
@ -62,12 +65,12 @@ jobs:
if: matrix.libgccjit_version.gcc != 'libgccjit12.so'
run: |
sudo dpkg --force-overwrite -i gcc-13.deb
echo /usr/lib/ > gcc_path
echo 'gcc-path = "/usr/lib"' > config.toml
echo "LIBRARY_PATH=/usr/lib" >> $GITHUB_ENV
echo "LD_LIBRARY_PATH=/usr/lib" >> $GITHUB_ENV
- name: Set env
run: |
echo "LIBRARY_PATH=$(cat gcc_path)" >> $GITHUB_ENV
echo "LD_LIBRARY_PATH=$(cat gcc_path)" >> $GITHUB_ENV
echo "workspace="$GITHUB_WORKSPACE >> $GITHUB_ENV
#- name: Cache rust repository

View File

@ -47,13 +47,13 @@ jobs:
run: sudo apt-get install ninja-build ripgrep llvm-14-tools libgccjit-12-dev
- name: Setup path to libgccjit
run: echo /usr/lib/gcc/x86_64-linux-gnu/12 > gcc_path
run: echo 'gcc-path = "/usr/lib/gcc/x86_64-linux-gnu/12"' > config.toml
- name: Set env
run: |
echo "LIBRARY_PATH=$(cat gcc_path)" >> $GITHUB_ENV
echo "LD_LIBRARY_PATH=$(cat gcc_path)" >> $GITHUB_ENV
echo "workspace="$GITHUB_WORKSPACE >> $GITHUB_ENV
echo "LIBRARY_PATH=/usr/lib/gcc/x86_64-linux-gnu/12" >> $GITHUB_ENV
echo "LD_LIBRARY_PATH=/usr/lib/gcc/x86_64-linux-gnu/12" >> $GITHUB_ENV
#- name: Cache rust repository
## We only clone the rust repository for rustc tests

View File

@ -65,13 +65,13 @@ jobs:
- name: Setup path to libgccjit
run: |
sudo dpkg -i gcc-m68k-13.deb
echo /usr/lib/ > gcc_path
echo 'gcc-path = "/usr/lib/"' > config.toml
- name: Set env
run: |
echo "LIBRARY_PATH=$(cat gcc_path)" >> $GITHUB_ENV
echo "LD_LIBRARY_PATH=$(cat gcc_path)" >> $GITHUB_ENV
echo "workspace="$GITHUB_WORKSPACE >> $GITHUB_ENV
echo "LIBRARY_PATH=/usr/lib" >> $GITHUB_ENV
echo "LD_LIBRARY_PATH=/usr/lib" >> $GITHUB_ENV
#- name: Cache rust repository
## We only clone the rust repository for rustc tests

View File

@ -42,13 +42,13 @@ jobs:
- name: Setup path to libgccjit
run: |
sudo dpkg --force-overwrite -i gcc-13.deb
echo /usr/lib/ > gcc_path
echo 'gcc-path = "/usr/lib/"' > config.toml
- name: Set env
run: |
echo "LIBRARY_PATH=$(cat gcc_path)" >> $GITHUB_ENV
echo "LD_LIBRARY_PATH=$(cat gcc_path)" >> $GITHUB_ENV
echo "workspace="$GITHUB_WORKSPACE >> $GITHUB_ENV
echo "LIBRARY_PATH=/usr/lib" >> $GITHUB_ENV
echo "LD_LIBRARY_PATH=/usr/lib" >> $GITHUB_ENV
- name: Build
run: |

View File

@ -56,13 +56,13 @@ jobs:
- name: Setup path to libgccjit
run: |
sudo dpkg --force-overwrite -i gcc-13.deb
echo /usr/lib/ > gcc_path
echo 'gcc-path = "/usr/lib/"' > config.toml
- name: Set env
run: |
echo "LIBRARY_PATH=$(cat gcc_path)" >> $GITHUB_ENV
echo "LD_LIBRARY_PATH=$(cat gcc_path)" >> $GITHUB_ENV
echo "workspace="$GITHUB_WORKSPACE >> $GITHUB_ENV
echo "LIBRARY_PATH=/usr/lib" >> $GITHUB_ENV
echo "LD_LIBRARY_PATH=/usr/lib" >> $GITHUB_ENV
- name: Build
run: |

1
.gitignore vendored
View File

@ -28,3 +28,4 @@ tools/llvmint-2
# The `llvm` folder is generated by the `tools/generate_intrinsics.py` script to update intrinsics.
llvm
build_system/target
config.toml

7
Cargo.lock generated
View File

@ -23,6 +23,12 @@ version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635"
[[package]]
name = "boml"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85fdb93f04c73bff54305fa437ffea5449c41edcaadfe882f35836206b166ac5"
[[package]]
name = "cc"
version = "1.0.79"
@ -185,6 +191,7 @@ checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78"
name = "rustc_codegen_gcc"
version = "0.1.0"
dependencies = [
"boml",
"gccjit",
"lang_tester",
"object",

View File

@ -37,6 +37,7 @@ tempfile = "3.7.1"
[dev-dependencies]
lang_tester = "0.3.9"
tempfile = "3.1.0"
boml = "0.3.1"
[profile.dev]
# By compiling dependencies with optimizations, performing tests gets much faster.

View File

@ -49,17 +49,27 @@ $ make check-jit
$ make check-jit RUNTESTFLAGS="-v -v -v jit.exp=jit.dg/test-asm.cc"
```
**Put the path to your custom build of libgccjit in the file `gcc_path`.**
**Put the path to your custom build of libgccjit in the file `config.toml`.**
If you followed the instructions exactly as written (ie, you have created a `gcc-build` folder
where gcc is built), the only thing you need to do is:
```bash
$ dirname $(readlink -f `find . -name libgccjit.so`) > gcc_path
$ cp config.example.toml config.toml
```
But if you did something different, you also need to set the `gcc-path` value in `config.toml` with
the result of this command:
```bash
$ dirname $(readlink -f `find . -name libgccjit.so`)
```
Then you can run commands like this:
```bash
$ ./y.sh prepare # download and patch sysroot src and install hyperfine for benchmarking
$ LIBRARY_PATH=$(cat gcc_path) LD_LIBRARY_PATH=$(cat gcc_path) ./y.sh build --release --features master
$ ./y.sh build --release --features master
```
To run the tests:
@ -100,7 +110,7 @@ error: failed to copy bitcode to object file: No such file or directory (os erro
> You should prefer using the Cargo method.
```bash
$ LIBRARY_PATH=$(cat gcc_path) LD_LIBRARY_PATH=$(cat gcc_path) rustc +$(cat $CG_GCCJIT_DIR/rust-toolchain | grep 'channel' | cut -d '=' -f 2 | sed 's/"//g' | sed 's/ //g') -Cpanic=abort -Zcodegen-backend=$CG_GCCJIT_DIR/target/release/librustc_codegen_gcc.so --sysroot $CG_GCCJIT_DIR/build_sysroot/sysroot my_crate.rs
$ LIBRARY_PATH="[gcc-path value]" LD_LIBRARY_PATH="[gcc-path value]" rustc +$(cat $CG_GCCJIT_DIR/rust-toolchain | grep 'channel' | cut -d '=' -f 2 | sed 's/"//g' | sed 's/ //g') -Cpanic=abort -Zcodegen-backend=$CG_GCCJIT_DIR/target/release/librustc_codegen_gcc.so --sysroot $CG_GCCJIT_DIR/build_sysroot/sysroot my_crate.rs
```
## Env vars
@ -322,7 +332,7 @@ generate it in [gimple.md](./doc/gimple.md).
#### Configuring rustc_codegen_gcc
* Run `./y.sh prepare --cross` so that the sysroot is patched for the cross-compiling case.
* Set the path to the cross-compiling libgccjit in `gcc_path`.
* Set the path to the cross-compiling libgccjit in `gcc-path` (in `config.toml`).
* Make sure you have the linker for your target (for instance `m68k-unknown-linux-gnu-gcc`) in your `$PATH`. Currently, the linker name is hardcoded as being `$TARGET-gcc`. Specify the target when building the sysroot: `./y.sh build --target-triple m68k-unknown-linux-gnu`.
* Build your project by specifying the target: `OVERWRITE_TARGET_TRIPLE=m68k-unknown-linux-gnu ../y.sh cargo build --target m68k-unknown-linux-gnu`.
@ -338,4 +348,4 @@ If you get the following error:
/usr/bin/ld: unrecognised emulation mode: m68kelf
```
Make sure you set `gcc_path` to the install directory.
Make sure you set `gcc-path` (in `config.toml`) to the install directory.

View File

@ -2,6 +2,15 @@
# It is not intended for manual editing.
version = 3
[[package]]
name = "boml"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85fdb93f04c73bff54305fa437ffea5449c41edcaadfe882f35836206b166ac5"
[[package]]
name = "y"
version = "0.1.0"
dependencies = [
"boml",
]

View File

@ -3,6 +3,9 @@ name = "y"
version = "0.1.0"
edition = "2021"
[dependencies]
boml = "0.3.1"
[[bin]]
name = "y"
path = "src/main.rs"

View File

@ -1,5 +1,5 @@
use crate::config::{Channel, ConfigInfo};
use crate::utils::{get_gcc_path, run_command, run_command_with_output_and_env, walk_dir};
use crate::utils::{run_command, run_command_with_output_and_env, walk_dir};
use std::collections::HashMap;
use std::ffi::OsStr;
use std::fs;
@ -8,17 +8,12 @@
#[derive(Default)]
struct BuildArg {
flags: Vec<String>,
gcc_path: String,
config_info: ConfigInfo,
}
impl BuildArg {
fn new() -> Result<Option<Self>, String> {
let gcc_path = get_gcc_path()?;
let mut build_arg = Self {
gcc_path,
..Default::default()
};
let mut build_arg = Self::default();
// We skip binary name and the `build` command.
let mut args = std::env::args().skip(2);
@ -169,7 +164,8 @@ pub fn build_sysroot(env: &HashMap<String, String>, config: &ConfigInfo) -> Resu
fs::create_dir_all(&sysroot_src_path).map_err(|error| {
format!(
"Failed to create directory `{}`: {:?}",
sysroot_src_path.display(), error
sysroot_src_path.display(),
error
)
})?;
run_command(
@ -188,8 +184,14 @@ pub fn build_sysroot(env: &HashMap<String, String>, config: &ConfigInfo) -> Resu
fn build_codegen(args: &mut BuildArg) -> Result<(), String> {
let mut env = HashMap::new();
env.insert("LD_LIBRARY_PATH".to_string(), args.gcc_path.clone());
env.insert("LIBRARY_PATH".to_string(), args.gcc_path.clone());
env.insert(
"LD_LIBRARY_PATH".to_string(),
args.config_info.gcc_path.clone(),
);
env.insert(
"LIBRARY_PATH".to_string(),
args.config_info.gcc_path.clone(),
);
let mut command: Vec<&dyn AsRef<OsStr>> = vec![&"cargo", &"rustc"];
if args.config_info.channel == Channel::Release {
@ -205,7 +207,7 @@ fn build_codegen(args: &mut BuildArg) -> Result<(), String> {
}
run_command_with_output_and_env(&command, None, Some(&env))?;
args.config_info.setup(&mut env, Some(&args.gcc_path))?;
args.config_info.setup(&mut env, None)?;
// We voluntarily ignore the error.
let _ = fs::remove_dir_all("target/out");
@ -227,6 +229,7 @@ pub fn run() -> Result<(), String> {
Some(args) => args,
None => return Ok(()),
};
args.config_info.setup_gcc_path(None)?;
build_codegen(&mut args)?;
Ok(())
}

View File

@ -1,7 +1,11 @@
use crate::utils::{get_gcc_path, get_os_name, rustc_version_info, split_args};
use crate::utils::{get_os_name, rustc_version_info, split_args};
use std::collections::HashMap;
use std::env as std_env;
use std::ffi::OsStr;
use std::fs;
use std::path::Path;
use boml::{types::TomlValue, Toml};
#[derive(Default, PartialEq, Eq, Clone, Copy, Debug)]
pub enum Channel {
@ -19,6 +23,71 @@ pub fn as_str(self) -> &'static str {
}
}
fn failed_config_parsing(config_file: &str, err: &str) -> Result<ConfigFile, String> {
Err(format!("Failed to parse `{}`: {}", config_file, err))
}
#[derive(Default)]
pub struct ConfigFile {
gcc_path: Option<String>,
download_gccjit: Option<bool>,
}
impl ConfigFile {
pub fn new(config_file: Option<&str>) -> Result<Self, String> {
let config_file = config_file.unwrap_or("config.toml");
let content = fs::read_to_string(config_file).map_err(|_| {
format!(
"Failed to read `{}`. Take a look at `Readme.md` to see how to set up the project",
config_file,
)
})?;
let toml = Toml::parse(&content).map_err(|err| {
format!(
"Error occurred around `{}`: {:?}",
&content[err.start..=err.end],
err.kind
)
})?;
let mut config = Self::default();
for (key, value) in toml.iter() {
match (key, value) {
("gcc-path", TomlValue::String(value)) => {
config.gcc_path = Some(value.as_str().to_string())
}
("gcc-path", _) => {
return failed_config_parsing(config_file, "Expected a string for `gcc-path`")
}
("download-gccjit", TomlValue::Boolean(value)) => {
config.download_gccjit = Some(*value)
}
("download-gccjit", _) => {
return failed_config_parsing(
config_file,
"Expected a boolean for `download-gccjit`",
)
}
_ => return failed_config_parsing(config_file, &format!("Unknown key `{}`", key)),
}
}
if config.gcc_path.is_none() && config.download_gccjit.is_none() {
return failed_config_parsing(
config_file,
"At least one of `gcc-path` or `download-gccjit` value must be set",
);
}
if let Some(gcc_path) = config.gcc_path.as_mut() {
let path = Path::new(gcc_path);
*gcc_path = path
.canonicalize()
.map_err(|err| format!("Failed to get absolute path of `{}`: {:?}", gcc_path, err))?
.display()
.to_string();
}
Ok(config)
}
}
#[derive(Default, Debug)]
pub struct ConfigInfo {
pub target: String,
@ -33,6 +102,8 @@ pub struct ConfigInfo {
pub sysroot_panic_abort: bool,
pub cg_backend_path: String,
pub sysroot_path: String,
pub gcc_path: String,
config_file: Option<String>,
}
impl ConfigInfo {
@ -64,6 +135,14 @@ pub fn parse_argument(
}
_ => return Err("Expected a value after `--out-dir`, found nothing".to_string()),
},
"--config-file" => match args.next() {
Some(arg) if !arg.is_empty() => {
self.config_file = Some(arg.to_string());
}
_ => {
return Err("Expected a value after `--config-file`, found nothing".to_string())
}
},
"--release-sysroot" => self.sysroot_release_channel = true,
"--release" => self.channel = Channel::Release,
"--sysroot-panic-abort" => self.sysroot_panic_abort = true,
@ -80,18 +159,46 @@ pub fn rustc_command_vec(&self) -> Vec<&dyn AsRef<OsStr>> {
command
}
pub fn setup_gcc_path(&mut self, override_gcc_path: Option<&str>) -> Result<(), String> {
let ConfigFile { gcc_path, .. } = ConfigFile::new(self.config_file.as_deref())?;
self.gcc_path = match override_gcc_path {
Some(path) => {
if gcc_path.is_some() {
println!(
"overriding setting from `{}`",
self.config_file.as_deref().unwrap_or("config.toml")
);
}
path.to_string()
}
None => {
match gcc_path {
Some(path) => path,
// FIXME: Once we support "download", rewrite this.
None => {
return Err(format!(
"missing `gcc-path` value from `{}`",
self.config_file.as_deref().unwrap_or("config.toml"),
))
}
}
}
};
Ok(())
}
pub fn setup(
&mut self,
env: &mut HashMap<String, String>,
gcc_path: Option<&str>,
override_gcc_path: Option<&str>,
) -> Result<(), String> {
env.insert("CARGO_INCREMENTAL".to_string(), "0".to_string());
let gcc_path = match gcc_path {
Some(path) => path.to_string(),
None => get_gcc_path()?,
};
env.insert("GCC_PATH".to_string(), gcc_path.clone());
if self.gcc_path.is_empty() || override_gcc_path.is_some() {
self.setup_gcc_path(override_gcc_path)?;
}
env.insert("GCC_PATH".to_string(), self.gcc_path.clone());
if self.cargo_target_dir.is_empty() {
match env.get("CARGO_TARGET_DIR").filter(|dir| !dir.is_empty()) {
@ -225,7 +332,9 @@ pub fn setup(
// line option to change it.
target = current_dir.join("target/out").display(),
sysroot = sysroot.display(),
gcc_path = self.gcc_path,
);
env.insert("LIBRARY_PATH".to_string(), ld_library_path.clone());
env.insert("LD_LIBRARY_PATH".to_string(), ld_library_path.clone());
env.insert("DYLD_LIBRARY_PATH".to_string(), ld_library_path);
@ -265,7 +374,8 @@ pub fn show_usage() {
--out-dir : Location where the files will be generated
--release : Build in release mode
--release-sysroot : Build sysroot in release mode
--sysroot-panic-abort : Build the sysroot without unwinding support."
--sysroot-panic-abort : Build the sysroot without unwinding support
--config-file : Location of the config file to be used"
);
}
}

View File

@ -1,7 +1,7 @@
use crate::build;
use crate::config::{Channel, ConfigInfo};
use crate::utils::{
get_gcc_path, get_toolchain, git_clone, remove_file, run_command, run_command_with_env,
get_toolchain, git_clone, remove_file, run_command, run_command_with_env,
run_command_with_output_and_env, rustc_version_info, split_args, walk_dir,
};
@ -109,7 +109,7 @@ fn show_usage() {
struct TestArg {
no_default_features: bool,
build_only: bool,
gcc_path: String,
gcc_path: Option<String>,
runners: BTreeSet<String>,
flags: Vec<String>,
backend: Option<String>,
@ -181,12 +181,10 @@ fn new() -> Result<Option<Self>, String> {
}
}
test_arg.gcc_path = if use_system_gcc {
if use_system_gcc {
println!("Using system GCC");
"gcc".to_string()
} else {
get_gcc_path()?
};
test_arg.gcc_path = Some("gcc".to_string());
}
}
match (test_arg.current_part, test_arg.nb_parts) {
(Some(_), Some(_)) | (None, None) => {}
@ -488,7 +486,8 @@ fn std_tests(env: &Env, args: &TestArg) -> Result<(), String> {
}
fn setup_rustc(env: &mut Env, args: &TestArg) -> Result<(), String> {
let toolchain = format!("+{channel}-{host}",
let toolchain = format!(
"+{channel}-{host}",
channel = get_toolchain()?, // May also include date
host = args.config_info.host_triple
);
@ -527,7 +526,12 @@ fn setup_rustc(env: &mut Env, args: &TestArg) -> Result<(), String> {
}
})?;
let rustc = String::from_utf8(
run_command_with_env(&[&"rustup", &toolchain, &"which", &"rustc"], rust_dir, Some(env))?.stdout,
run_command_with_env(
&[&"rustup", &toolchain, &"which", &"rustc"],
rust_dir,
Some(env),
)?
.stdout,
)
.map_err(|error| format!("Failed to retrieve rustc path: {:?}", error))
.and_then(|rustc| {
@ -1162,8 +1166,15 @@ pub fn run() -> Result<(), String> {
};
let mut env: HashMap<String, String> = std::env::vars().collect();
env.insert("LD_LIBRARY_PATH".to_string(), args.gcc_path.clone());
env.insert("LIBRARY_PATH".to_string(), args.gcc_path.clone());
args.config_info.setup_gcc_path(None)?;
env.insert(
"LIBRARY_PATH".to_string(),
args.config_info.gcc_path.clone(),
);
env.insert(
"LD_LIBRARY_PATH".to_string(),
args.config_info.gcc_path.clone(),
);
build_if_no_backend(&env, &args)?;
if args.build_only {
@ -1171,7 +1182,7 @@ pub fn run() -> Result<(), String> {
return Ok(());
}
args.config_info.setup(&mut env, Some(&args.gcc_path))?;
args.config_info.setup(&mut env, args.gcc_path.as_deref())?;
if args.runners.is_empty() {
run_all(&env, &args)?;

View File

@ -248,38 +248,6 @@ pub fn get_toolchain() -> Result<String, String> {
}
}
pub fn get_gcc_path() -> Result<String, String> {
let content = match fs::read_to_string("gcc_path") {
Ok(content) => content,
Err(_) => {
return Err(
"Please put the path to your custom build of libgccjit in the file \
`gcc_path`, see Readme.md for details"
.into(),
)
}
};
match content
.split('\n')
.map(|line| line.trim())
.filter(|line| !line.is_empty())
.next()
{
Some(gcc_path) => {
let path = Path::new(gcc_path);
if !path.exists() {
Err(format!(
"Path `{}` contained in the `gcc_path` file doesn't exist",
gcc_path,
))
} else {
Ok(gcc_path.into())
}
}
None => Err("No path found in `gcc_path` file".into()),
}
}
pub struct CloneResult {
pub ran_clone: bool,
pub repo_name: String,

2
config.example.toml Normal file
View File

@ -0,0 +1,2 @@
gcc-path = "gcc-build/gcc"
# download-gccjit = true

View File

@ -7,6 +7,7 @@
use lang_tester::LangTester;
use tempfile::TempDir;
use boml::Toml;
/// Controls the compile options (e.g., optimization level) used to compile
/// test code.
@ -20,8 +21,16 @@ pub fn main_inner(profile: Profile) {
let tempdir = TempDir::new().expect("temp dir");
let current_dir = current_dir().expect("current dir");
let current_dir = current_dir.to_str().expect("current dir").to_string();
let gcc_path = include_str!("../gcc_path");
let gcc_path = gcc_path.trim();
let gcc_path = Toml::parse(include_str!("../config.toml"))
.expect("Failed to parse `config.toml`")
.get_string("gcc-path")
.expect("Missing `gcc-path` key in `config.toml`")
.to_string();
let gcc_path = Path::new(&gcc_path)
.canonicalize()
.expect("failed to get absolute path of `gcc-path`")
.display()
.to_string();
env::set_var("LD_LIBRARY_PATH", gcc_path);
fn rust_filter(filename: &Path) -> bool {