mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge commit '7d53619064ab7045c383644cb445052d2a3d46db' into sync_cg_clif-2023-02-09

Browse Source
This commit is contained in:
bjorn3 2023-02-09 12:38:16 +01:00
commit e25566e20b
26 changed files with 638 additions and 329 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

146
compiler/rustc_codegen_cranelift/.github/workflows/main.yml vendored
View File

@ -21,6 +21,7 @@ jobs:
cargo fmt --check
rustfmt --check build_system/mod.rs
build:
runs-on: ${{ matrix.os }}
timeout-minutes: 60
@ -33,7 +34,7 @@ jobs:
fail-fast: false
matrix:
include:
- os: ubuntu-20.04 # FIXME switch to ubuntu-22.04 once #1303 is fixed
- os: ubuntu-latest
env:
TARGET_TRIPLE: x86_64-unknown-linux-gnu
- os: macos-latest
@ -112,23 +113,6 @@ jobs:
TARGET_TRIPLE: ${{ matrix.env.TARGET_TRIPLE }}
run: ./y.rs test
- name: Package prebuilt cg_clif
run: tar cvfJ cg_clif.tar.xz dist
- name: Upload prebuilt cg_clif
if: matrix.os == 'windows-latest' || matrix.env.TARGET_TRIPLE != 'x86_64-pc-windows-gnu'
uses: actions/upload-artifact@v3
with:
name: cg_clif-${{ matrix.env.TARGET_TRIPLE }}
path: cg_clif.tar.xz
- name: Upload prebuilt cg_clif (cross compile)
if: matrix.os != 'windows-latest' && matrix.env.TARGET_TRIPLE == 'x86_64-pc-windows-gnu'
uses: actions/upload-artifact@v3
with:
name: cg_clif-${{ runner.os }}-cross-x86_64-mingw
path: cg_clif.tar.xz
abi_cafe:
runs-on: ${{ matrix.os }}
@ -185,3 +169,129 @@ jobs:
env:
TARGET_TRIPLE: ${{ matrix.env.TARGET_TRIPLE }}
run: ./y.rs abi-cafe
bench:
runs-on: ubuntu-latest
timeout-minutes: 60
defaults:
run:
shell: bash
steps:
- uses: actions/checkout@v3
- name: Cache cargo target dir
uses: actions/cache@v3
with:
path: build/cg_clif
key: ${{ runner.os }}-x86_64-unknown-linux-gnu-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }}
- name: Cache cargo bin dir
uses: actions/cache@v3
with:
path: ~/.cargo/bin
key: ${{ runner.os }}-${{ matrix.env.TARGET_TRIPLE }}-cargo-bin-dir-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }}
- name: Use sparse cargo registry
run: |
cat >> ~/.cargo/config.toml <<EOF
[unstable]
sparse-registry = true
EOF
- name: Install hyperfine
run: cargo install hyperfine || true
- name: Prepare dependencies
run: ./y.rs prepare
- name: Build
run: CI_OPT=1 ./y.rs build --sysroot none
- name: Benchmark
run: CI_OPT=1 ./y.rs bench
dist:
runs-on: ${{ matrix.os }}
timeout-minutes: 60
defaults:
run:
shell: bash
strategy:
fail-fast: false
matrix:
include:
# FIXME update at some point in the future once most distros use a newer glibc
- os: ubuntu-20.04
env:
TARGET_TRIPLE: x86_64-unknown-linux-gnu
- os: macos-latest
env:
TARGET_TRIPLE: x86_64-apple-darwin
# cross-compile from Linux to Windows using mingw
- os: ubuntu-latest
env:
TARGET_TRIPLE: x86_64-pc-windows-gnu
- os: windows-latest
env:
TARGET_TRIPLE: x86_64-pc-windows-msvc
- os: windows-latest
env:
TARGET_TRIPLE: x86_64-pc-windows-gnu
steps:
- uses: actions/checkout@v3
- name: Cache cargo target dir
uses: actions/cache@v3
with:
path: build/cg_clif
key: ${{ runner.os }}-${{ matrix.env.TARGET_TRIPLE }}-dist-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }}
- name: Set MinGW as the default toolchain
if: matrix.os == 'windows-latest' && matrix.env.TARGET_TRIPLE == 'x86_64-pc-windows-gnu'
run: rustup set default-host x86_64-pc-windows-gnu
- name: Install MinGW toolchain and wine
if: matrix.os == 'ubuntu-latest' && matrix.env.TARGET_TRIPLE == 'x86_64-pc-windows-gnu'
run: |
sudo apt-get update
sudo apt-get install -y gcc-mingw-w64-x86-64 wine-stable
- name: Use sparse cargo registry
run: |
cat >> ~/.cargo/config.toml <<EOF
[unstable]
sparse-registry = true
EOF
- name: Prepare dependencies
run: ./y.rs prepare
- name: Build backend
run: CI_OPT=1 ./y.rs build --sysroot none
- name: Build sysroot
run: CI_OPT=1 ./y.rs build
- name: Package prebuilt cg_clif
run: tar cvfJ cg_clif.tar.xz dist
- name: Upload prebuilt cg_clif
if: matrix.os == 'windows-latest' || matrix.env.TARGET_TRIPLE != 'x86_64-pc-windows-gnu'
uses: actions/upload-artifact@v3
with:
name: cg_clif-${{ matrix.env.TARGET_TRIPLE }}
path: cg_clif.tar.xz
- name: Upload prebuilt cg_clif (cross compile)
if: matrix.os != 'windows-latest' && matrix.env.TARGET_TRIPLE == 'x86_64-pc-windows-gnu'
uses: actions/upload-artifact@v3
with:
name: cg_clif-${{ runner.os }}-cross-x86_64-mingw
path: cg_clif.tar.xz

4
compiler/rustc_codegen_cranelift/build_sysroot/Cargo.lock
View File

@ -34,9 +34,9 @@ dependencies = [
[[package]]
name = "cc"
version = "1.0.78"
version = "1.0.79"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a20104e2335ce8a659d6dd92a51a767a0c062599c73b343fd152cb401e828c3d"
checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f"
[[package]]
name = "cfg-if"

8
compiler/rustc_codegen_cranelift/build_system/abi_cafe.rs
View File

@ -6,11 +6,10 @@
use super::utils::{spawn_and_wait, CargoProject, Compiler};
use super::SysrootKind;
pub(crate) static ABI_CAFE_REPO: GitRepo =
static ABI_CAFE_REPO: GitRepo =
GitRepo::github("Gankra", "abi-cafe", "4c6dc8c9c687e2b3a760ff2176ce236872b37212", "abi-cafe");
pub(crate) static ABI_CAFE: CargoProject =
CargoProject::new(&ABI_CAFE_REPO.source_dir(), "abi_cafe");
static ABI_CAFE: CargoProject = CargoProject::new(&ABI_CAFE_REPO.source_dir(), "abi_cafe");
pub(crate) fn run(
channel: &str,
@ -19,6 +18,9 @@ pub(crate) fn run(
cg_clif_dylib: &Path,
bootstrap_host_compiler: &Compiler,
) {
ABI_CAFE_REPO.fetch(dirs);
spawn_and_wait(ABI_CAFE.fetch("cargo", &bootstrap_host_compiler.rustc, dirs));
eprintln!("Building sysroot for abi-cafe");
build_sysroot::build_sysroot(
dirs,

32
compiler/rustc_codegen_cranelift/build_system/bench.rs
View File

@ -5,9 +5,9 @@
use super::path::{Dirs, RelPath};
use super::prepare::GitRepo;
use super::rustc_info::get_file_name;
use super::utils::{hyperfine_command, is_ci, spawn_and_wait, CargoProject, Compiler};
use super::utils::{hyperfine_command, spawn_and_wait, CargoProject, Compiler};
pub(crate) static SIMPLE_RAYTRACER_REPO: GitRepo = GitRepo::github(
static SIMPLE_RAYTRACER_REPO: GitRepo = GitRepo::github(
"ebobby",
"simple-raytracer",
"804a7a21b9e673a482797aa289a18ed480e4d813",
@ -15,10 +15,10 @@
);
// Use a separate target dir for the initial LLVM build to reduce unnecessary recompiles
pub(crate) static SIMPLE_RAYTRACER_LLVM: CargoProject =
static SIMPLE_RAYTRACER_LLVM: CargoProject =
CargoProject::new(&SIMPLE_RAYTRACER_REPO.source_dir(), "simple_raytracer_llvm");
pub(crate) static SIMPLE_RAYTRACER: CargoProject =
static SIMPLE_RAYTRACER: CargoProject =
CargoProject::new(&SIMPLE_RAYTRACER_REPO.source_dir(), "simple_raytracer");
pub(crate) fn benchmark(dirs: &Dirs, bootstrap_host_compiler: &Compiler) {
@ -32,6 +32,15 @@ fn benchmark_simple_raytracer(dirs: &Dirs, bootstrap_host_compiler: &Compiler) {
std::process::exit(1);
}
if !SIMPLE_RAYTRACER_REPO.source_dir().to_path(dirs).exists() {
SIMPLE_RAYTRACER_REPO.fetch(dirs);
spawn_and_wait(SIMPLE_RAYTRACER.fetch(
&bootstrap_host_compiler.cargo,
&bootstrap_host_compiler.rustc,
dirs,
));
}
eprintln!("[LLVM BUILD] simple-raytracer");
let build_cmd = SIMPLE_RAYTRACER_LLVM.build(bootstrap_host_compiler, dirs);
spawn_and_wait(build_cmd);
@ -45,10 +54,7 @@ fn benchmark_simple_raytracer(dirs: &Dirs, bootstrap_host_compiler: &Compiler) {
)
.unwrap();
let run_runs = env::var("RUN_RUNS")
.unwrap_or(if is_ci() { "2" } else { "10" }.to_string())
.parse()
.unwrap();
let bench_runs = env::var("BENCH_RUNS").unwrap_or_else(|_| "10".to_string()).parse().unwrap();
eprintln!("[BENCH COMPILE] ebobby/simple-raytracer");
let cargo_clif =
@ -57,24 +63,24 @@ fn benchmark_simple_raytracer(dirs: &Dirs, bootstrap_host_compiler: &Compiler) {
let target_dir = SIMPLE_RAYTRACER.target_dir(dirs);
let clean_cmd = format!(
"cargo clean --manifest-path {manifest_path} --target-dir {target_dir}",
"RUSTC=rustc cargo clean --manifest-path {manifest_path} --target-dir {target_dir}",
manifest_path = manifest_path.display(),
target_dir = target_dir.display(),
);
let llvm_build_cmd = format!(
"cargo build --manifest-path {manifest_path} --target-dir {target_dir}",
"RUSTC=rustc cargo build --manifest-path {manifest_path} --target-dir {target_dir}",
manifest_path = manifest_path.display(),
target_dir = target_dir.display(),
);
let clif_build_cmd = format!(
"{cargo_clif} build --manifest-path {manifest_path} --target-dir {target_dir}",
"RUSTC=rustc {cargo_clif} build --manifest-path {manifest_path} --target-dir {target_dir}",
cargo_clif = cargo_clif.display(),
manifest_path = manifest_path.display(),
target_dir = target_dir.display(),
);
let bench_compile =
hyperfine_command(1, run_runs, Some(&clean_cmd), &llvm_build_cmd, &clif_build_cmd);
hyperfine_command(1, bench_runs, Some(&clean_cmd), &llvm_build_cmd, &clif_build_cmd);
spawn_and_wait(bench_compile);
@ -87,7 +93,7 @@ fn benchmark_simple_raytracer(dirs: &Dirs, bootstrap_host_compiler: &Compiler) {
let mut bench_run = hyperfine_command(
0,
run_runs,
bench_runs,
None,
Path::new(".").join(get_file_name("raytracer_cg_llvm", "bin")).to_str().unwrap(),
Path::new(".").join(get_file_name("raytracer_cg_clif", "bin")).to_str().unwrap(),

6
compiler/rustc_codegen_cranelift/build_system/build_backend.rs
View File

@ -3,7 +3,7 @@
use super::path::{Dirs, RelPath};
use super::rustc_info::get_file_name;
use super::utils::{is_ci, CargoProject, Compiler};
use super::utils::{is_ci, is_ci_opt, CargoProject, Compiler};
pub(crate) static CG_CLIF: CargoProject = CargoProject::new(&RelPath::SOURCE, "cg_clif");
@ -26,7 +26,9 @@ pub(crate) fn build_backend(
// Disabling incr comp reduces cache size and incr comp doesn't save as much on CI anyway
cmd.env("CARGO_BUILD_INCREMENTAL", "false");
cmd.env("CARGO_PROFILE_RELEASE_DEBUG_ASSERTIONS", "true");
if !is_ci_opt() {
cmd.env("CARGO_PROFILE_RELEASE_DEBUG_ASSERTIONS", "true");
}
}
if use_unstable_features {

3
compiler/rustc_codegen_cranelift/build_system/build_sysroot.rs
View File

@ -248,6 +248,9 @@ fn build_clif_sysroot_for_triple(
build_cmd.arg("--release");
}
build_cmd.env("__CARGO_DEFAULT_LIB_METADATA", "cg_clif");
if compiler.triple.contains("apple") {
build_cmd.env("CARGO_PROFILE_RELEASE_SPLIT_DEBUGINFO", "packed");
}
spawn_and_wait(build_cmd);
for entry in fs::read_dir(build_dir.join("deps")).unwrap() {

8
compiler/rustc_codegen_cranelift/build_system/mod.rs
View File

@ -2,7 +2,7 @@
use std::path::PathBuf;
use std::process;
use self::utils::{is_ci, Compiler};
use self::utils::{is_ci, is_ci_opt, Compiler};
mod abi_cafe;
mod bench;
@ -53,8 +53,10 @@ pub fn main() {
// Disabling incr comp reduces cache size and incr comp doesn't save as much on CI anyway
env::set_var("CARGO_BUILD_INCREMENTAL", "false");
// Enable the Cranelift verifier
env::set_var("CG_CLIF_ENABLE_VERIFIER", "1");
if !is_ci_opt() {
// Enable the Cranelift verifier
env::set_var("CG_CLIF_ENABLE_VERIFIER", "1");
}
}
let mut args = env::args().skip(1);

18
compiler/rustc_codegen_cranelift/build_system/prepare.rs
View File

@ -11,22 +11,18 @@
pub(crate) fn prepare(dirs: &Dirs) {
RelPath::DOWNLOAD.ensure_fresh(dirs);
spawn_and_wait(super::build_backend::CG_CLIF.fetch("cargo", dirs));
spawn_and_wait(super::build_backend::CG_CLIF.fetch("cargo", "rustc", dirs));
prepare_sysroot(dirs);
spawn_and_wait(super::build_sysroot::STANDARD_LIBRARY.fetch("cargo", dirs));
spawn_and_wait(super::tests::LIBCORE_TESTS.fetch("cargo", dirs));
spawn_and_wait(super::build_sysroot::STANDARD_LIBRARY.fetch("cargo", "rustc", dirs));
spawn_and_wait(super::tests::LIBCORE_TESTS.fetch("cargo", "rustc", dirs));
super::abi_cafe::ABI_CAFE_REPO.fetch(dirs);
spawn_and_wait(super::abi_cafe::ABI_CAFE.fetch("cargo", dirs));
super::tests::RAND_REPO.fetch(dirs);
spawn_and_wait(super::tests::RAND.fetch("cargo", dirs));
spawn_and_wait(super::tests::RAND.fetch("cargo", "rustc", dirs));
super::tests::REGEX_REPO.fetch(dirs);
spawn_and_wait(super::tests::REGEX.fetch("cargo", dirs));
spawn_and_wait(super::tests::REGEX.fetch("cargo", "rustc", dirs));
super::tests::PORTABLE_SIMD_REPO.fetch(dirs);
spawn_and_wait(super::tests::PORTABLE_SIMD.fetch("cargo", dirs));
super::bench::SIMPLE_RAYTRACER_REPO.fetch(dirs);
spawn_and_wait(super::bench::SIMPLE_RAYTRACER.fetch("cargo", dirs));
spawn_and_wait(super::tests::PORTABLE_SIMD.fetch("cargo", "rustc", dirs));
}
fn prepare_sysroot(dirs: &Dirs) {
@ -80,7 +76,7 @@ pub(crate) const fn source_dir(&self) -> RelPath {
}
}
fn fetch(&self, dirs: &Dirs) {
pub(crate) fn fetch(&self, dirs: &Dirs) {
match self.url {
GitRepoUrl::Github { user, repo } => {
clone_repo_shallow_github(

5
compiler/rustc_codegen_cranelift/build_system/tests.rs
View File

@ -1,4 +1,3 @@
use super::bench::SIMPLE_RAYTRACER;
use super::build_sysroot::{self, SYSROOT_SRC};
use super::config;
use super::path::{Dirs, RelPath};
@ -134,10 +133,6 @@ const fn jit_bin(config: &'static str, source: &'static str, args: &'static str)
spawn_and_wait(build_cmd);
}
}),
TestCase::custom("test.simple-raytracer", &|runner| {
SIMPLE_RAYTRACER.clean(&runner.dirs);
spawn_and_wait(SIMPLE_RAYTRACER.build(&runner.target_compiler, &runner.dirs));
}),
TestCase::custom("test.libcore", &|runner| {
LIBCORE_TESTS.clean(&runner.dirs);

18
compiler/rustc_codegen_cranelift/build_system/utils.rs
View File

@ -121,10 +121,18 @@ fn build_cmd(&self, command: &str, compiler: &Compiler, dirs: &Dirs) -> Command
}
#[must_use]
pub(crate) fn fetch(&self, cargo: impl AsRef<Path>, dirs: &Dirs) -> Command {
pub(crate) fn fetch(
&self,
cargo: impl AsRef<Path>,
rustc: impl AsRef<Path>,
dirs: &Dirs,
) -> Command {
let mut cmd = Command::new(cargo.as_ref());
cmd.arg("fetch").arg("--manifest-path").arg(self.manifest_path(dirs));
cmd.env("RUSTC", rustc.as_ref())
.arg("fetch")
.arg("--manifest-path")
.arg(self.manifest_path(dirs));
cmd
}
@ -271,5 +279,9 @@ pub(crate) fn copy_dir_recursively(from: &Path, to: &Path) {
}
pub(crate) fn is_ci() -> bool {
env::var("CI").as_deref() == Ok("true")
env::var("CI").is_ok()
}
pub(crate) fn is_ci_opt() -> bool {
env::var("CI_OPT").is_ok()
}

1
compiler/rustc_codegen_cranelift/config.txt
View File

@ -44,7 +44,6 @@ aot.issue-72793
testsuite.extended_sysroot
test.rust-random/rand
test.simple-raytracer
test.libcore
test.regex-shootout-regex-dna
test.regex

2
compiler/rustc_codegen_cranelift/rust-toolchain
View File

@ -1,3 +1,3 @@
[toolchain]
channel = "nightly-2023-01-20"
channel = "nightly-2023-02-06"
components = ["rust-src", "rustc-dev", "llvm-tools-preview"]

1
compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh
View File

@ -91,6 +91,7 @@ rm tests/ui/proc-macro/proc-macro-deprecated-attr.rs # same
rm tests/ui/proc-macro/quote-debug.rs # same
rm tests/ui/proc-macro/no-missing-docs.rs # same
rm tests/ui/rust-2018/proc-macro-crate-in-paths.rs # same
rm tests/ui/proc-macro/allowed-signatures.rs # same
# doesn't work due to the way the rustc test suite is invoked.
# should work when using ./x.py test the way it is intended

4
compiler/rustc_codegen_cranelift/src/abi/comments.rs
View File

@ -98,12 +98,12 @@ pub(super) fn add_local_place_comments<'tcx>(
}
CPlaceInner::VarPair(place_local, var1, var2) => {
assert_eq!(local, place_local);
("ssa", Cow::Owned(format!(",var=({}, {})", var1.index(), var2.index())))
("ssa", Cow::Owned(format!("var=({}, {})", var1.index(), var2.index())))
}
CPlaceInner::VarLane(_local, _var, _lane) => unreachable!(),
CPlaceInner::Addr(ptr, meta) => {
let meta = if let Some(meta) = meta {
Cow::Owned(format!(",meta={}", meta))
Cow::Owned(format!("meta={}", meta))
} else {
Cow::Borrowed("")
};

75
compiler/rustc_codegen_cranelift/src/allocator.rs
View File

@ -70,37 +70,13 @@ fn codegen_inner(
params: arg_tys.iter().cloned().map(AbiParam::new).collect(),
returns: output.into_iter().map(AbiParam::new).collect(),
};
let caller_name = format!("__rust_{}", method.name);
let callee_name = kind.fn_name(method.name);
let func_id = module.declare_function(&caller_name, Linkage::Export, &sig).unwrap();
let callee_func_id = module.declare_function(&callee_name, Linkage::Import, &sig).unwrap();
let mut ctx = Context::new();
ctx.func.signature = sig.clone();
{
let mut func_ctx = FunctionBuilderContext::new();
let mut bcx = FunctionBuilder::new(&mut ctx.func, &mut func_ctx);
let block = bcx.create_block();
bcx.switch_to_block(block);
let args = arg_tys
.into_iter()
.map(|ty| bcx.append_block_param(block, ty))
.collect::<Vec<Value>>();
let callee_func_ref = module.declare_func_in_func(callee_func_id, &mut bcx.func);
let call_inst = bcx.ins().call(callee_func_ref, &args);
let results = bcx.inst_results(call_inst).to_vec(); // Clone to prevent borrow error
bcx.ins().return_(&results);
bcx.seal_all_blocks();
bcx.finalize();
}
module.define_function(func_id, &mut ctx).unwrap();
unwind_context.add_function(func_id, &ctx, module.isa());
crate::common::create_wrapper_function(
module,
unwind_context,
sig,
&format!("__rust_{}", method.name),
&kind.fn_name(method.name),
);
}
let sig = Signature {
@ -108,36 +84,13 @@ fn codegen_inner(
params: vec![AbiParam::new(usize_ty), AbiParam::new(usize_ty)],
returns: vec![],
};
let callee_name = alloc_error_handler_kind.fn_name(sym::oom);
let func_id =
module.declare_function("__rust_alloc_error_handler", Linkage::Export, &sig).unwrap();
let callee_func_id = module.declare_function(&callee_name, Linkage::Import, &sig).unwrap();
let mut ctx = Context::new();
ctx.func.signature = sig;
{
let mut func_ctx = FunctionBuilderContext::new();
let mut bcx = FunctionBuilder::new(&mut ctx.func, &mut func_ctx);
let block = bcx.create_block();
bcx.switch_to_block(block);
let args = (&[usize_ty, usize_ty])
.iter()
.map(|&ty| bcx.append_block_param(block, ty))
.collect::<Vec<Value>>();
let callee_func_ref = module.declare_func_in_func(callee_func_id, &mut bcx.func);
bcx.ins().call(callee_func_ref, &args);
bcx.ins().trap(TrapCode::UnreachableCodeReached);
bcx.seal_all_blocks();
bcx.finalize();
}
module.define_function(func_id, &mut ctx).unwrap();
unwind_context.add_function(func_id, &ctx, module.isa());
crate::common::create_wrapper_function(
module,
unwind_context,
sig,
"__rust_alloc_error_handler",
&alloc_error_handler_kind.fn_name(sym::oom),
);
let data_id = module.declare_data(OomStrategy::SYMBOL, Linkage::Export, false, false).unwrap();
let mut data_ctx = DataContext::new();

91
compiler/rustc_codegen_cranelift/src/base.rs
View File

@ -21,23 +21,6 @@ pub(crate) struct CodegenedFunction {
func_debug_cx: Option<FunctionDebugContext>,
}
#[cfg_attr(not(feature = "jit"), allow(dead_code))]
pub(crate) fn codegen_and_compile_fn<'tcx>(
tcx: TyCtxt<'tcx>,
cx: &mut crate::CodegenCx,
cached_context: &mut Context,
module: &mut dyn Module,
instance: Instance<'tcx>,
) {
let _inst_guard =
crate::PrintOnPanic(|| format!("{:?} {}", instance, tcx.symbol_name(instance).name));
let cached_func = std::mem::replace(&mut cached_context.func, Function::new());
let codegened_func = codegen_fn(tcx, cx, cached_func, module, instance);
compile_fn(cx, cached_context, module, codegened_func);
}
pub(crate) fn codegen_fn<'tcx>(
tcx: TyCtxt<'tcx>,
cx: &mut crate::CodegenCx,
@ -47,6 +30,9 @@ pub(crate) fn codegen_fn<'tcx>(
) -> CodegenedFunction {
debug_assert!(!instance.substs.needs_infer());
let symbol_name = tcx.symbol_name(instance).name.to_string();
let _timer = tcx.prof.generic_activity_with_arg("codegen fn", &*symbol_name);
let mir = tcx.instance_mir(instance.def);
let _mir_guard = crate::PrintOnPanic(|| {
let mut buf = Vec::new();
@ -58,7 +44,6 @@ pub(crate) fn codegen_fn<'tcx>(
});
// Declare function
let symbol_name = tcx.symbol_name(instance).name.to_string();
let sig = get_function_sig(tcx, module.target_config().default_call_conv, instance);
let func_id = module.declare_function(&symbol_name, Linkage::Local, &sig).unwrap();
@ -112,7 +97,7 @@ pub(crate) fn codegen_fn<'tcx>(
next_ssa_var: 0,
};
tcx.sess.time("codegen clif ir", || codegen_fn_body(&mut fx, start_block));
tcx.prof.generic_activity("codegen clif ir").run(|| codegen_fn_body(&mut fx, start_block));
fx.bcx.seal_all_blocks();
fx.bcx.finalize();
@ -146,6 +131,9 @@ pub(crate) fn compile_fn(
module: &mut dyn Module,
codegened_func: CodegenedFunction,
) {
let _timer =
cx.profiler.generic_activity_with_arg("compile function", &*codegened_func.symbol_name);
let clif_comments = codegened_func.clif_comments;
// Store function in context
@ -191,9 +179,30 @@ pub(crate) fn compile_fn(
};
// Define function
cx.profiler.verbose_generic_activity("define function").run(|| {
cx.profiler.generic_activity("define function").run(|| {
context.want_disasm = cx.should_write_ir;
module.define_function(codegened_func.func_id, context).unwrap();
if cx.profiler.enabled() {
let mut recording_args = false;
cx.profiler
.generic_activity_with_arg_recorder(
"define function (clif pass timings)",
|recorder| {
let pass_times = cranelift_codegen::timing::take_current();
// Replace newlines with | as measureme doesn't allow control characters like
// newlines inside strings.
recorder.record_arg(format!("{}", pass_times).replace("\n", " | "));
recording_args = true;
},
)
.run(|| {
if recording_args {
// Wait a tiny bit to ensure chrome's profiler doesn't hide the event
std::thread::sleep(std::time::Duration::from_nanos(2))
}
});
}
});
if cx.should_write_ir {
@ -220,7 +229,7 @@ pub(crate) fn compile_fn(
let isa = module.isa();
let debug_context = &mut cx.debug_context;
let unwind_context = &mut cx.unwind_context;
cx.profiler.verbose_generic_activity("generate debug info").run(|| {
cx.profiler.generic_activity("generate debug info").run(|| {
if let Some(debug_context) = debug_context {
codegened_func.func_debug_cx.unwrap().finalize(
debug_context,
@ -237,7 +246,7 @@ pub(crate) fn verify_func(
writer: &crate::pretty_clif::CommentWriter,
func: &Function,
) {
tcx.sess.time("verify clif ir", || {
tcx.prof.generic_activity("verify clif ir").run(|| {
let flags = cranelift_codegen::settings::Flags::new(cranelift_codegen::settings::builder());
match cranelift_codegen::verify_function(&func, &flags) {
Ok(_) => {}
@ -273,7 +282,10 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
return;
}
fx.tcx.sess.time("codegen prelude", || crate::abi::codegen_fn_prelude(fx, start_block));
fx.tcx
.prof
.generic_activity("codegen prelude")
.run(|| crate::abi::codegen_fn_prelude(fx, start_block));
for (bb, bb_data) in fx.mir.basic_blocks.iter_enumerated() {
let block = fx.get_block(bb);
@ -434,7 +446,7 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
cleanup: _,
from_hir_call: _,
} => {
fx.tcx.sess.time("codegen call", || {
fx.tcx.prof.generic_activity("codegen call").run(|| {
crate::abi::codegen_terminator_call(
fx,
mir::SourceInfo { span: *fn_span, ..source_info },
@ -778,17 +790,30 @@ fn is_fat_ptr<'tcx>(fx: &FunctionCx<'_, '_, 'tcx>, ty: Ty<'tcx>) -> bool {
let val = CValue::const_val(fx, fx.layout_of(fx.tcx.types.usize), val.into());
lval.write_cvalue(fx, val);
}
Rvalue::Aggregate(ref kind, ref operands) => match kind.as_ref() {
AggregateKind::Array(_ty) => {
for (i, operand) in operands.iter().enumerate() {
let operand = codegen_operand(fx, operand);
let index = fx.bcx.ins().iconst(fx.pointer_type, i as i64);
let to = lval.place_index(fx, index);
to.write_cvalue(fx, operand);
Rvalue::Aggregate(ref kind, ref operands) => {
let (variant_index, variant_dest, active_field_index) = match **kind {
mir::AggregateKind::Adt(_, variant_index, _, _, active_field_index) => {
let variant_dest = lval.downcast_variant(fx, variant_index);
(variant_index, variant_dest, active_field_index)
}
_ => (VariantIdx::from_u32(0), lval, None),
};
if active_field_index.is_some() {
assert_eq!(operands.len(), 1);
}
_ => unreachable!("shouldn't exist at codegen {:?}", to_place_and_rval.1),
},
for (i, operand) in operands.iter().enumerate() {
let operand = codegen_operand(fx, operand);
let field_index = active_field_index.unwrap_or(i);
let to = if let mir::AggregateKind::Array(_) = **kind {
let index = fx.bcx.ins().iconst(fx.pointer_type, field_index as i64);
variant_dest.place_index(fx, index)
} else {
variant_dest.place_field(fx, mir::Field::new(field_index))
};
to.write_cvalue(fx, operand);
}
crate::discriminant::codegen_set_discriminant(fx, lval, variant_index);
}
}
}
StatementKind::StorageLive(_)

38
compiler/rustc_codegen_cranelift/src/common.rs
View File

@ -254,6 +254,44 @@ pub(crate) fn type_sign(ty: Ty<'_>) -> bool {
}
}
pub(crate) fn create_wrapper_function(
module: &mut dyn Module,
unwind_context: &mut UnwindContext,
sig: Signature,
wrapper_name: &str,
callee_name: &str,
) {
let wrapper_func_id = module.declare_function(wrapper_name, Linkage::Export, &sig).unwrap();
let callee_func_id = module.declare_function(callee_name, Linkage::Import, &sig).unwrap();
let mut ctx = Context::new();
ctx.func.signature = sig;
{
let mut func_ctx = FunctionBuilderContext::new();
let mut bcx = FunctionBuilder::new(&mut ctx.func, &mut func_ctx);
let block = bcx.create_block();
bcx.switch_to_block(block);
let func = &mut bcx.func.stencil;
let args = func
.signature
.params
.iter()
.map(|param| func.dfg.append_block_param(block, param.value_type))
.collect::<Vec<Value>>();
let callee_func_ref = module.declare_func_in_func(callee_func_id, &mut bcx.func);
let call_inst = bcx.ins().call(callee_func_ref, &args);
let results = bcx.inst_results(call_inst).to_vec(); // Clone to prevent borrow error
bcx.ins().return_(&results);
bcx.seal_all_blocks();
bcx.finalize();
}
module.define_function(wrapper_func_id, &mut ctx).unwrap();
unwind_context.add_function(wrapper_func_id, &ctx, module.isa());
}
pub(crate) struct FunctionCx<'m, 'clif, 'tcx: 'm> {
pub(crate) cx: &'clif mut crate::CodegenCx,
pub(crate) module: &'m mut dyn Module,

38
compiler/rustc_codegen_cranelift/src/compiler_builtins.rs
View File

@ -1,14 +1,33 @@
#[cfg(all(unix, feature = "jit"))]
use std::ffi::c_int;
#[cfg(feature = "jit")]
use std::ffi::c_void;
// FIXME replace with core::ffi::c_size_t once stablized
#[allow(non_camel_case_types)]
#[cfg(feature = "jit")]
type size_t = usize;
macro_rules! builtin_functions {
($register:ident; $(fn $name:ident($($arg_name:ident: $arg_ty:ty),*) -> $ret_ty:ty;)*) => {
(
$register:ident;
$(
$(#[$attr:meta])?
fn $name:ident($($arg_name:ident: $arg_ty:ty),*) -> $ret_ty:ty;
)*
) => {
#[cfg(feature = "jit")]
#[allow(improper_ctypes)]
extern "C" {
$(fn $name($($arg_name: $arg_ty),*) -> $ret_ty;)*
$(
$(#[$attr])?
fn $name($($arg_name: $arg_ty),*) -> $ret_ty;
)*
}
#[cfg(feature = "jit")]
pub(crate) fn $register(builder: &mut cranelift_jit::JITBuilder) {
for (name, val) in [$((stringify!($name), $name as *const u8)),*] {
for (name, val) in [$($(#[$attr])? (stringify!($name), $name as *const u8)),*] {
builder.symbol(name, val);
}
}
@ -40,4 +59,17 @@ pub(crate) fn $register(builder: &mut cranelift_jit::JITBuilder) {
fn __fixdfti(f: f64) -> i128;
fn __fixunssfti(f: f32) -> u128;
fn __fixunsdfti(f: f64) -> u128;
// allocator
// NOTE: These need to be mentioned here despite not being part of compiler_builtins because
// newer glibc resolve dlsym("malloc") to libc.so despite the override in the rustc binary to
// use jemalloc. Libraries opened with dlopen still get the jemalloc version, causing multiple
// allocators to be mixed, resulting in a crash.
fn calloc(nobj: size_t, size: size_t) -> *mut c_void;
#[cfg(unix)]
fn posix_memalign(memptr: *mut *mut c_void, align: size_t, size: size_t) -> c_int;
fn malloc(size: size_t) -> *mut c_void;
fn realloc(p: *mut c_void, size: size_t) -> *mut c_void;
fn free(p: *mut c_void) -> ();
}

8
compiler/rustc_codegen_cranelift/src/config.rs
View File

@ -42,12 +42,6 @@ pub struct BackendConfig {
/// Defaults to the value of `CG_CLIF_JIT_ARGS`.
pub jit_args: Vec<String>,
/// Display the time it took to perform codegen for a crate.
///
/// Defaults to true when the `CG_CLIF_DISPLAY_CG_TIME` env var is set to 1 or false otherwise.
/// Can be set using `-Cllvm-args=display_cg_time=...`.
pub display_cg_time: bool,
/// Enable the Cranelift ir verifier for all compilation passes. If not set it will only run
/// once before passing the clif ir to Cranelift for compilation.
///
@ -73,7 +67,6 @@ fn default() -> Self {
let args = std::env::var("CG_CLIF_JIT_ARGS").unwrap_or_else(|_| String::new());
args.split(' ').map(|arg| arg.to_string()).collect()
},
display_cg_time: bool_env_var("CG_CLIF_DISPLAY_CG_TIME"),
enable_verifier: cfg!(debug_assertions) || bool_env_var("CG_CLIF_ENABLE_VERIFIER"),
disable_incr_cache: bool_env_var("CG_CLIF_DISABLE_INCR_CACHE"),
}
@ -92,7 +85,6 @@ fn parse_bool(name: &str, value: &str) -> Result<bool, String> {
if let Some((name, value)) = opt.split_once('=') {
match name {
"mode" => config.codegen_mode = value.parse()?,
"display_cg_time" => config.display_cg_time = parse_bool(name, value)?,
"enable_verifier" => config.enable_verifier = parse_bool(name, value)?,
"disable_incr_cache" => config.disable_incr_cache = parse_bool(name, value)?,
_ => return Err(format!("Unknown option `{}`", name)),

128
compiler/rustc_codegen_cranelift/src/driver/aot.rs
View File

@ -272,25 +272,25 @@ fn module_codegen(
ConcurrencyLimiterToken,
),
) -> OngoingModuleCodegen {
let (cgu_name, mut cx, mut module, codegened_functions) = tcx.sess.time("codegen cgu", || {
let cgu = tcx.codegen_unit(cgu_name);
let mono_items = cgu.items_in_deterministic_order(tcx);
let (cgu_name, mut cx, mut module, codegened_functions) =
tcx.prof.verbose_generic_activity_with_arg("codegen cgu", cgu_name.as_str()).run(|| {
let cgu = tcx.codegen_unit(cgu_name);
let mono_items = cgu.items_in_deterministic_order(tcx);
let mut module = make_module(tcx.sess, &backend_config, cgu_name.as_str().to_string());
let mut module = make_module(tcx.sess, &backend_config, cgu_name.as_str().to_string());
let mut cx = crate::CodegenCx::new(
tcx,
backend_config.clone(),
module.isa(),
tcx.sess.opts.debuginfo != DebugInfo::None,
cgu_name,
);
super::predefine_mono_items(tcx, &mut module, &mono_items);
let mut codegened_functions = vec![];
for (mono_item, _) in mono_items {
match mono_item {
MonoItem::Fn(inst) => {
tcx.sess.time("codegen fn", || {
let mut cx = crate::CodegenCx::new(
tcx,
backend_config.clone(),
module.isa(),
tcx.sess.opts.debuginfo != DebugInfo::None,
cgu_name,
);
super::predefine_mono_items(tcx, &mut module, &mono_items);
let mut codegened_functions = vec![];
for (mono_item, _) in mono_items {
match mono_item {
MonoItem::Fn(inst) => {
let codegened_function = crate::base::codegen_fn(
tcx,
&mut cx,
@ -299,53 +299,68 @@ fn module_codegen(
inst,
);
codegened_functions.push(codegened_function);
});
}
MonoItem::Static(def_id) => {
crate::constant::codegen_static(tcx, &mut module, def_id)
}
MonoItem::GlobalAsm(item_id) => {
crate::global_asm::codegen_global_asm_item(tcx, &mut cx.global_asm, item_id);
}
MonoItem::Static(def_id) => {
crate::constant::codegen_static(tcx, &mut module, def_id)
}
MonoItem::GlobalAsm(item_id) => {
crate::global_asm::codegen_global_asm_item(
tcx,
&mut cx.global_asm,
item_id,
);
}
}
}
}
crate::main_shim::maybe_create_entry_wrapper(
tcx,
&mut module,
&mut cx.unwind_context,
false,
cgu.is_primary(),
);
crate::main_shim::maybe_create_entry_wrapper(
tcx,
&mut module,
&mut cx.unwind_context,
false,
cgu.is_primary(),
);
let cgu_name = cgu.name().as_str().to_owned();
let cgu_name = cgu.name().as_str().to_owned();
(cgu_name, cx, module, codegened_functions)
});
OngoingModuleCodegen::Async(std::thread::spawn(move || {
cx.profiler.clone().verbose_generic_activity("compile functions").run(|| {
let mut cached_context = Context::new();
for codegened_func in codegened_functions {
crate::base::compile_fn(&mut cx, &mut cached_context, &mut module, codegened_func);
}
(cgu_name, cx, module, codegened_functions)
});
let global_asm_object_file =
cx.profiler.verbose_generic_activity("compile assembly").run(|| {
OngoingModuleCodegen::Async(std::thread::spawn(move || {
cx.profiler.clone().verbose_generic_activity_with_arg("compile functions", &*cgu_name).run(
|| {
let mut cached_context = Context::new();
for codegened_func in codegened_functions {
crate::base::compile_fn(
&mut cx,
&mut cached_context,
&mut module,
codegened_func,
);
}
},
);
let global_asm_object_file = cx
.profiler
.verbose_generic_activity_with_arg("compile assembly", &*cgu_name)
.run(|| {
crate::global_asm::compile_global_asm(&global_asm_config, &cgu_name, &cx.global_asm)
})?;
let codegen_result = cx.profiler.verbose_generic_activity("write object file").run(|| {
emit_cgu(
&global_asm_config.output_filenames,
&cx.profiler,
cgu_name,
module,
cx.debug_context,
cx.unwind_context,
global_asm_object_file,
)
});
let codegen_result = cx
.profiler
.verbose_generic_activity_with_arg("write object file", &*cgu_name)
.run(|| {
emit_cgu(
&global_asm_config.output_filenames,
&cx.profiler,
cgu_name,
module,
cx.debug_context,
cx.unwind_context,
global_asm_object_file,
)
});
std::mem::drop(token);
codegen_result
}))
@ -375,7 +390,7 @@ pub(crate) fn run_aot(
let mut concurrency_limiter = ConcurrencyLimiter::new(tcx.sess, cgus.len());
let modules = super::time(tcx, backend_config.display_cg_time, "codegen mono items", || {
let modules = tcx.sess.time("codegen mono items", || {
cgus.iter()
.map(|cgu| {
let cgu_reuse = if backend_config.disable_incr_cache {
@ -437,7 +452,6 @@ pub(crate) fn run_aot(
};
let metadata_module = if need_metadata_module {
let _timer = tcx.prof.generic_activity("codegen crate metadata");
let (metadata_cgu_name, tmp_file) = tcx.sess.time("write compressed metadata", || {
use rustc_middle::mir::mono::CodegenUnitNameBuilder;

46
compiler/rustc_codegen_cranelift/src/driver/jit.rs
View File

@ -121,22 +121,20 @@ pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
.into_iter()
.collect::<Vec<(_, (_, _))>>();
super::time(tcx, backend_config.display_cg_time, "codegen mono items", || {
tcx.sess.time("codegen mono items", || {
super::predefine_mono_items(tcx, &mut jit_module, &mono_items);
for (mono_item, _) in mono_items {
match mono_item {
MonoItem::Fn(inst) => match backend_config.codegen_mode {
CodegenMode::Aot => unreachable!(),
CodegenMode::Jit => {
tcx.sess.time("codegen fn", || {
crate::base::codegen_and_compile_fn(
tcx,
&mut cx,
&mut cached_context,
&mut jit_module,
inst,
)
});
codegen_and_compile_fn(
tcx,
&mut cx,
&mut cached_context,
&mut jit_module,
inst,
);
}
CodegenMode::JitLazy => {
codegen_shim(tcx, &mut cx, &mut cached_context, &mut jit_module, inst)
@ -219,6 +217,24 @@ pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
}
}
pub(crate) fn codegen_and_compile_fn<'tcx>(
tcx: TyCtxt<'tcx>,
cx: &mut crate::CodegenCx,
cached_context: &mut Context,
module: &mut dyn Module,
instance: Instance<'tcx>,
) {
tcx.prof.generic_activity("codegen and compile fn").run(|| {
let _inst_guard =
crate::PrintOnPanic(|| format!("{:?} {}", instance, tcx.symbol_name(instance).name));
let cached_func = std::mem::replace(&mut cached_context.func, Function::new());
let codegened_func = crate::base::codegen_fn(tcx, cx, cached_func, module, instance);
crate::base::compile_fn(cx, cached_context, module, codegened_func);
});
}
extern "C" fn clif_jit_fn(
instance_ptr: *const Instance<'static>,
trampoline_ptr: *const u8,
@ -271,15 +287,7 @@ fn jit_fn(instance_ptr: *const Instance<'static>, trampoline_ptr: *const u8) ->
false,
Symbol::intern("dummy_cgu_name"),
);
tcx.sess.time("codegen fn", || {
crate::base::codegen_and_compile_fn(
tcx,
&mut cx,
&mut Context::new(),
jit_module,
instance,
)
});
codegen_and_compile_fn(tcx, &mut cx, &mut Context::new(), jit_module, instance);
assert!(cx.global_asm.is_empty());
jit_module.finalize_definitions().unwrap();

15
compiler/rustc_codegen_cranelift/src/driver/mod.rs
View File

@ -17,7 +17,7 @@ fn predefine_mono_items<'tcx>(
module: &mut dyn Module,
mono_items: &[(MonoItem<'tcx>, (RLinkage, Visibility))],
) {
tcx.sess.time("predefine functions", || {
tcx.prof.generic_activity("predefine functions").run(|| {
let is_compiler_builtins = tcx.is_compiler_builtins(LOCAL_CRATE);
for &(mono_item, (linkage, visibility)) in mono_items {
match mono_item {
@ -39,16 +39,3 @@ fn predefine_mono_items<'tcx>(
}
});
}
fn time<R>(tcx: TyCtxt<'_>, display: bool, name: &'static str, f: impl FnOnce() -> R) -> R {
if display {
println!("[{:<30}: {}] start", tcx.crate_name(LOCAL_CRATE), name);
let before = std::time::Instant::now();
let res = tcx.sess.time(name, f);
let after = std::time::Instant::now();
println!("[{:<30}: {}] end time: {:?}", tcx.crate_name(LOCAL_CRATE), name, after - before);
res
} else {
tcx.sess.time(name, f)
}
}

43
compiler/rustc_codegen_cranelift/src/global_asm.rs
View File

@ -7,7 +7,7 @@
use std::sync::Arc;
use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
use rustc_hir::ItemId;
use rustc_hir::{InlineAsmOperand, ItemId};
use rustc_session::config::{OutputFilenames, OutputType};
use crate::prelude::*;
@ -23,7 +23,46 @@ pub(crate) fn codegen_global_asm_item(tcx: TyCtxt<'_>, global_asm: &mut String,
for piece in asm.template {
match *piece {
InlineAsmTemplatePiece::String(ref s) => global_asm.push_str(s),
InlineAsmTemplatePiece::Placeholder { .. } => todo!(),
InlineAsmTemplatePiece::Placeholder { operand_idx, modifier: _, span: op_sp } => {
match asm.operands[operand_idx].0 {
InlineAsmOperand::Const { ref anon_const } => {
let const_value =
tcx.const_eval_poly(anon_const.def_id.to_def_id()).unwrap_or_else(
|_| span_bug!(op_sp, "asm const cannot be resolved"),
);
let ty = tcx.typeck_body(anon_const.body).node_type(anon_const.hir_id);
let string = rustc_codegen_ssa::common::asm_const_to_str(
tcx,
op_sp,
const_value,
RevealAllLayoutCx(tcx).layout_of(ty),
);
global_asm.push_str(&string);
}
InlineAsmOperand::SymFn { anon_const } => {
let ty = tcx.typeck_body(anon_const.body).node_type(anon_const.hir_id);
let instance = match ty.kind() {
&ty::FnDef(def_id, substs) => Instance::new(def_id, substs),
_ => span_bug!(op_sp, "asm sym is not a function"),
};
let symbol = tcx.symbol_name(instance);
// FIXME handle the case where the function was made private to the
// current codegen unit
global_asm.push_str(symbol.name);
}
InlineAsmOperand::SymStatic { path: _, def_id } => {
let instance = Instance::mono(tcx, def_id).polymorphize(tcx);
let symbol = tcx.symbol_name(instance);
global_asm.push_str(symbol.name);
}
InlineAsmOperand::In { .. }
| InlineAsmOperand::Out { .. }
| InlineAsmOperand::InOut { .. }
| InlineAsmOperand::SplitInOut { .. } => {
span_bug!(op_sp, "invalid operand type for global_asm!")
}
}
}
}
}
global_asm.push_str("\n.att_syntax\n\n");

187
compiler/rustc_codegen_cranelift/src/inline_asm.rs
View File

@ -9,9 +9,33 @@
use rustc_span::sym;
use rustc_target::asm::*;
enum CInlineAsmOperand<'tcx> {
In {
reg: InlineAsmRegOrRegClass,
value: CValue<'tcx>,
},
Out {
reg: InlineAsmRegOrRegClass,
late: bool,
place: Option<CPlace<'tcx>>,
},
InOut {
reg: InlineAsmRegOrRegClass,
_late: bool,
in_value: CValue<'tcx>,
out_place: Option<CPlace<'tcx>>,
},
Const {
value: String,
},
Symbol {
symbol: String,
},
}
pub(crate) fn codegen_inline_asm<'tcx>(
fx: &mut FunctionCx<'_, '_, 'tcx>,
_span: Span,
span: Span,
template: &[InlineAsmTemplatePiece],
operands: &[InlineAsmOperand<'tcx>],
options: InlineAsmOptions,
@ -198,6 +222,81 @@ pub(crate) fn codegen_inline_asm<'tcx>(
}
}
let operands = operands
.into_iter()
.map(|operand| match *operand {
InlineAsmOperand::In { reg, ref value } => {
CInlineAsmOperand::In { reg, value: crate::base::codegen_operand(fx, value) }
}
InlineAsmOperand::Out { reg, late, ref place } => CInlineAsmOperand::Out {
reg,
late,
place: place.map(|place| crate::base::codegen_place(fx, place)),
},
InlineAsmOperand::InOut { reg, late, ref in_value, ref out_place } => {
CInlineAsmOperand::InOut {
reg,
_late: late,
in_value: crate::base::codegen_operand(fx, in_value),
out_place: out_place.map(|place| crate::base::codegen_place(fx, place)),
}
}
InlineAsmOperand::Const { ref value } => {
let (const_value, ty) = crate::constant::eval_mir_constant(fx, &*value)
.unwrap_or_else(|| span_bug!(span, "asm const cannot be resolved"));
let value = rustc_codegen_ssa::common::asm_const_to_str(
fx.tcx,
span,
const_value,
fx.layout_of(ty),
);
CInlineAsmOperand::Const { value }
}
InlineAsmOperand::SymFn { ref value } => {
let literal = fx.monomorphize(value.literal);
if let ty::FnDef(def_id, substs) = *literal.ty().kind() {
let instance = ty::Instance::resolve_for_fn_ptr(
fx.tcx,
ty::ParamEnv::reveal_all(),
def_id,
substs,
)
.unwrap();
let symbol = fx.tcx.symbol_name(instance);
// Pass a wrapper rather than the function itself as the function itself may not
// be exported from the main codegen unit and may thus be unreachable from the
// object file created by an external assembler.
let inline_asm_index = fx.cx.inline_asm_index.get();
fx.cx.inline_asm_index.set(inline_asm_index + 1);
let wrapper_name = format!(
"__inline_asm_{}_wrapper_n{}",
fx.cx.cgu_name.as_str().replace('.', "__").replace('-', "_"),
inline_asm_index
);
let sig =
get_function_sig(fx.tcx, fx.target_config.default_call_conv, instance);
create_wrapper_function(
fx.module,
&mut fx.cx.unwind_context,
sig,
&wrapper_name,
symbol.name,
);
CInlineAsmOperand::Symbol { symbol: wrapper_name }
} else {
span_bug!(span, "invalid type for asm sym (fn)");
}
}
InlineAsmOperand::SymStatic { def_id } => {
assert!(fx.tcx.is_static(def_id));
let instance = Instance::mono(fx.tcx, def_id).polymorphize(fx.tcx);
CInlineAsmOperand::Symbol { symbol: fx.tcx.symbol_name(instance).name.to_owned() }
}
})
.collect::<Vec<_>>();
let mut inputs = Vec::new();
let mut outputs = Vec::new();
@ -206,7 +305,7 @@ pub(crate) fn codegen_inline_asm<'tcx>(
arch: fx.tcx.sess.asm_arch.unwrap(),
enclosing_def_id: fx.instance.def_id(),
template,
operands,
operands: &operands,
options,
registers: Vec::new(),
stack_slots_clobber: Vec::new(),
@ -229,36 +328,22 @@ pub(crate) fn codegen_inline_asm<'tcx>(
fx.cx.global_asm.push_str(&generated_asm);
for (i, operand) in operands.iter().enumerate() {
match *operand {
InlineAsmOperand::In { reg: _, ref value } => {
inputs.push((
asm_gen.stack_slots_input[i].unwrap(),
crate::base::codegen_operand(fx, value).load_scalar(fx),
));
match operand {
CInlineAsmOperand::In { reg: _, value } => {
inputs.push((asm_gen.stack_slots_input[i].unwrap(), value.load_scalar(fx)));
}
InlineAsmOperand::Out { reg: _, late: _, place } => {
CInlineAsmOperand::Out { reg: _, late: _, place } => {
if let Some(place) = place {
outputs.push((
asm_gen.stack_slots_output[i].unwrap(),
crate::base::codegen_place(fx, place),
));
outputs.push((asm_gen.stack_slots_output[i].unwrap(), place.clone()));
}
}
InlineAsmOperand::InOut { reg: _, late: _, ref in_value, out_place } => {
inputs.push((
asm_gen.stack_slots_input[i].unwrap(),
crate::base::codegen_operand(fx, in_value).load_scalar(fx),
));
CInlineAsmOperand::InOut { reg: _, _late: _, in_value, out_place } => {
inputs.push((asm_gen.stack_slots_input[i].unwrap(), in_value.load_scalar(fx)));
if let Some(out_place) = out_place {
outputs.push((
asm_gen.stack_slots_output[i].unwrap(),
crate::base::codegen_place(fx, out_place),
));
outputs.push((asm_gen.stack_slots_output[i].unwrap(), out_place.clone()));
}
}
InlineAsmOperand::Const { value: _ } => todo!(),
InlineAsmOperand::SymFn { value: _ } => todo!(),
InlineAsmOperand::SymStatic { def_id: _ } => todo!(),
CInlineAsmOperand::Const { value: _ } | CInlineAsmOperand::Symbol { symbol: _ } => {}
}
}
@ -280,7 +365,7 @@ struct InlineAssemblyGenerator<'a, 'tcx> {
arch: InlineAsmArch,
enclosing_def_id: DefId,
template: &'a [InlineAsmTemplatePiece],
operands: &'a [InlineAsmOperand<'tcx>],
operands: &'a [CInlineAsmOperand<'tcx>],
options: InlineAsmOptions,
registers: Vec<Option<InlineAsmReg>>,
stack_slots_clobber: Vec<Option<Size>>,
@ -304,18 +389,20 @@ fn allocate_registers(&mut self) {
// Add explicit registers to the allocated set.
for (i, operand) in self.operands.iter().enumerate() {
match *operand {
InlineAsmOperand::In { reg: InlineAsmRegOrRegClass::Reg(reg), .. } => {
CInlineAsmOperand::In { reg: InlineAsmRegOrRegClass::Reg(reg), .. } => {
regs[i] = Some(reg);
allocated.entry(reg).or_default().0 = true;
}
InlineAsmOperand::Out {
reg: InlineAsmRegOrRegClass::Reg(reg), late: true, ..
CInlineAsmOperand::Out {
reg: InlineAsmRegOrRegClass::Reg(reg),
late: true,
..
} => {
regs[i] = Some(reg);
allocated.entry(reg).or_default().1 = true;
}
InlineAsmOperand::Out { reg: InlineAsmRegOrRegClass::Reg(reg), .. }
| InlineAsmOperand::InOut { reg: InlineAsmRegOrRegClass::Reg(reg), .. } => {
CInlineAsmOperand::Out { reg: InlineAsmRegOrRegClass::Reg(reg), .. }
| CInlineAsmOperand::InOut { reg: InlineAsmRegOrRegClass::Reg(reg), .. } => {
regs[i] = Some(reg);
allocated.insert(reg, (true, true));
}
@ -326,12 +413,12 @@ fn allocate_registers(&mut self) {
// Allocate out/inout/inlateout registers first because they are more constrained.
for (i, operand) in self.operands.iter().enumerate() {
match *operand {
InlineAsmOperand::Out {
CInlineAsmOperand::Out {
reg: InlineAsmRegOrRegClass::RegClass(class),
late: false,
..
}
| InlineAsmOperand::InOut {
| CInlineAsmOperand::InOut {
reg: InlineAsmRegOrRegClass::RegClass(class), ..
} => {
let mut alloc_reg = None;
@ -360,7 +447,7 @@ fn allocate_registers(&mut self) {
// Allocate in/lateout.
for (i, operand) in self.operands.iter().enumerate() {
match *operand {
InlineAsmOperand::In { reg: InlineAsmRegOrRegClass::RegClass(class), .. } => {
CInlineAsmOperand::In { reg: InlineAsmRegOrRegClass::RegClass(class), .. } => {
let mut alloc_reg = None;
for &reg in &map[&class] {
let mut used = false;
@ -380,7 +467,7 @@ fn allocate_registers(&mut self) {
regs[i] = Some(reg);
allocated.entry(reg).or_default().0 = true;
}
InlineAsmOperand::Out {
CInlineAsmOperand::Out {
reg: InlineAsmRegOrRegClass::RegClass(class),
late: true,
..
@ -455,7 +542,7 @@ fn allocate_stack_slots(&mut self) {
// Allocate stack slots for inout
for (i, operand) in self.operands.iter().enumerate() {
match *operand {
InlineAsmOperand::InOut { reg, out_place: Some(_), .. } => {
CInlineAsmOperand::InOut { reg, out_place: Some(_), .. } => {
let slot = new_slot(reg.reg_class());
slots_input[i] = Some(slot);
slots_output[i] = Some(slot);
@ -470,8 +557,8 @@ fn allocate_stack_slots(&mut self) {
// Allocate stack slots for input
for (i, operand) in self.operands.iter().enumerate() {
match *operand {
InlineAsmOperand::In { reg, .. }
| InlineAsmOperand::InOut { reg, out_place: None, .. } => {
CInlineAsmOperand::In { reg, .. }
| CInlineAsmOperand::InOut { reg, out_place: None, .. } => {
slots_input[i] = Some(new_slot(reg.reg_class()));
}
_ => (),
@ -487,7 +574,7 @@ fn allocate_stack_slots(&mut self) {
// Allocate stack slots for output
for (i, operand) in self.operands.iter().enumerate() {
match *operand {
InlineAsmOperand::Out { reg, place: Some(_), .. } => {
CInlineAsmOperand::Out { reg, place: Some(_), .. } => {
slots_output[i] = Some(new_slot(reg.reg_class()));
}
_ => (),
@ -549,13 +636,23 @@ fn generate_asm_wrapper(&self, asm_name: &str) -> String {
generated_asm.push_str(s);
}
InlineAsmTemplatePiece::Placeholder { operand_idx, modifier, span: _ } => {
if self.options.contains(InlineAsmOptions::ATT_SYNTAX) {
generated_asm.push('%');
match self.operands[*operand_idx] {
CInlineAsmOperand::In { .. }
| CInlineAsmOperand::Out { .. }
| CInlineAsmOperand::InOut { .. } => {
if self.options.contains(InlineAsmOptions::ATT_SYNTAX) {
generated_asm.push('%');
}
self.registers[*operand_idx]
.unwrap()
.emit(&mut generated_asm, self.arch, *modifier)
.unwrap();
}
CInlineAsmOperand::Const { ref value } => {
generated_asm.push_str(value);
}
CInlineAsmOperand::Symbol { ref symbol } => generated_asm.push_str(symbol),
}
self.registers[*operand_idx]
.unwrap()
.emit(&mut generated_asm, self.arch, *modifier)
.unwrap();
}
}
}

31
compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
View File

@ -218,22 +218,6 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
let intrinsic = fx.tcx.item_name(instance.def_id());
let substs = instance.substs;
let target = if let Some(target) = target {
target
} else {
// Insert non returning intrinsics here
match intrinsic {
sym::abort => {
fx.bcx.ins().trap(TrapCode::User(0));
}
sym::transmute => {
crate::base::codegen_panic(fx, "Transmuting to uninhabited type.", source_info);
}
_ => unimplemented!("unsupported intrinsic {}", intrinsic),
}
return;
};
if intrinsic.as_str().starts_with("simd_") {
self::simd::codegen_simd_intrinsic_call(
fx,
@ -241,11 +225,11 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
substs,
args,
destination,
target,
target.expect("target for simd intrinsic"),
source_info.span,
);
} else if codegen_float_intrinsic_call(fx, intrinsic, args, destination) {
let ret_block = fx.get_block(target);
let ret_block = fx.get_block(target.expect("target for float intrinsic"));
fx.bcx.ins().jump(ret_block, &[]);
} else {
codegen_regular_intrinsic_call(
@ -255,7 +239,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
substs,
args,
destination,
Some(target),
target,
source_info,
);
}
@ -382,6 +366,10 @@ fn codegen_regular_intrinsic_call<'tcx>(
let usize_layout = fx.layout_of(fx.tcx.types.usize);
match intrinsic {
sym::abort => {
fx.bcx.ins().trap(TrapCode::User(0));
return;
}
sym::likely | sym::unlikely => {
intrinsic_args!(fx, args => (a); intrinsic);
@ -579,6 +567,11 @@ fn codegen_regular_intrinsic_call<'tcx>(
sym::transmute => {
intrinsic_args!(fx, args => (from); intrinsic);
if ret.layout().abi.is_uninhabited() {
crate::base::codegen_panic(fx, "Transmuting to uninhabited type.", source_info);
return;
}
ret.write_cvalue_transmute(fx, from);
}
sym::write_bytes | sym::volatile_set_memory => {

11
compiler/rustc_codegen_cranelift/src/value_and_place.rs
View File

@ -588,10 +588,13 @@ fn transmute_value<'tcx>(
return;
}
CPlaceInner::VarPair(_local, var1, var2) => {
let (ptr, meta) = from.force_stack(fx);
assert!(meta.is_none());
let (data1, data2) =
CValue(CValueInner::ByRef(ptr, None), dst_layout).load_scalar_pair(fx);
let (data1, data2) = if from.layout().ty == dst_layout.ty {
CValue(from.0, dst_layout).load_scalar_pair(fx)
} else {
let (ptr, meta) = from.force_stack(fx);
assert!(meta.is_none());
CValue(CValueInner::ByRef(ptr, None), dst_layout).load_scalar_pair(fx)
};
let (dst_ty1, dst_ty2) = fx.clif_pair_type(self.layout().ty).unwrap();
transmute_value(fx, var1, data1, dst_ty1);
transmute_value(fx, var2, data2, dst_ty2);