diff --git a/Readme.md b/Readme.md index 8a2db5a43ec..1e84c7fa365 100644 --- a/Readme.md +++ b/Readme.md @@ -52,9 +52,7 @@ configuration options. ## Not yet supported * Inline assembly ([no cranelift support](https://github.com/bytecodealliance/wasmtime/issues/1041)) - * On Linux there is support for invoking an external assembler for `global_asm!` and `asm!`. - `llvm_asm!` will remain unimplemented forever. `asm!` doesn't yet support reg classes. You - have to specify specific registers instead. + * On UNIX there is support for invoking an external assembler for `global_asm!` and `asm!`. * SIMD ([tracked here](https://github.com/bjorn3/rustc_codegen_cranelift/issues/171), some basic things work) ## License diff --git a/example/mini_core_hello_world.rs b/example/mini_core_hello_world.rs index 7e9cbe1bba5..e83be3a3df5 100644 --- a/example/mini_core_hello_world.rs +++ b/example/mini_core_hello_world.rs @@ -321,7 +321,7 @@ fn main() { #[cfg(not(any(jit, windows)))] test_tls(); - #[cfg(all(not(jit), target_arch = "x86_64", target_os = "linux"))] + #[cfg(all(not(jit), target_arch = "x86_64", any(target_os = "linux", target_os = "darwin")))] unsafe { global_asm_test(); } @@ -343,7 +343,7 @@ fn main() { } } -#[cfg(all(not(jit), target_arch = "x86_64", target_os = "linux"))] +#[cfg(all(not(jit), target_arch = "x86_64", any(target_os = "linux", target_os = "darwin")))] extern "C" { fn global_asm_test(); } @@ -358,6 +358,16 @@ global_asm! { " } +#[cfg(all(not(jit), target_arch = "x86_64", target_os = "darwin"))] +global_asm! { + " + .global _global_asm_test + _global_asm_test: + // comment that would normally be removed by LLVM + ret + " +} + #[repr(C)] enum c_void { _1, diff --git a/src/base.rs b/src/base.rs index 2c04cf47268..c68d33465bc 100644 --- a/src/base.rs +++ b/src/base.rs @@ -5,6 +5,7 @@ use rustc_index::vec::IndexVec; use rustc_middle::ty::adjustment::PointerCast; use rustc_middle::ty::layout::FnAbiOf; use rustc_middle::ty::print::with_no_trimmed_paths; +use rustc_middle::ty::SymbolName; use indexmap::IndexSet; @@ -12,17 +13,42 @@ use crate::constant::ConstantCx; use crate::prelude::*; use crate::pretty_clif::CommentWriter; -pub(crate) fn codegen_fn<'tcx>( +struct CodegenedFunction<'tcx> { + instance: Instance<'tcx>, + symbol_name: SymbolName<'tcx>, + func_id: FuncId, + func: Function, + clif_comments: CommentWriter, + source_info_set: IndexSet, + local_map: IndexVec>, +} + +pub(crate) fn codegen_and_compile_fn<'tcx>( cx: &mut crate::CodegenCx<'tcx>, + cached_context: &mut Context, module: &mut dyn Module, instance: Instance<'tcx>, ) { let tcx = cx.tcx; - let _inst_guard = crate::PrintOnPanic(|| format!("{:?} {}", instance, tcx.symbol_name(instance).name)); + + let cached_func = std::mem::replace(&mut cached_context.func, Function::new()); + let codegened_func = codegen_fn(cx, cached_func, module, instance); + + compile_fn(cx, cached_context, module, codegened_func); +} + +fn codegen_fn<'tcx>( + cx: &mut crate::CodegenCx<'tcx>, + cached_func: Function, + module: &mut dyn Module, + instance: Instance<'tcx>, +) -> CodegenedFunction<'tcx> { debug_assert!(!instance.substs.needs_infer()); + let tcx = cx.tcx; + let mir = tcx.instance_mir(instance.def); let _mir_guard = crate::PrintOnPanic(|| { let mut buf = Vec::new(); @@ -38,11 +64,10 @@ pub(crate) fn codegen_fn<'tcx>( let sig = get_function_sig(tcx, module.isa().triple(), instance); let func_id = module.declare_function(symbol_name.name, Linkage::Local, &sig).unwrap(); - cx.cached_context.clear(); - // Make the FunctionBuilder let mut func_ctx = FunctionBuilderContext::new(); - let mut func = std::mem::replace(&mut cx.cached_context.func, Function::new()); + let mut func = cached_func; + func.clear(); func.name = ExternalName::user(0, func_id.as_u32()); func.signature = sig; func.collect_debug_info(); @@ -82,27 +107,7 @@ pub(crate) fn codegen_fn<'tcx>( next_ssa_var: 0, }; - let arg_uninhabited = fx - .mir - .args_iter() - .any(|arg| fx.layout_of(fx.monomorphize(fx.mir.local_decls[arg].ty)).abi.is_uninhabited()); - - if !crate::constant::check_constants(&mut fx) { - fx.bcx.append_block_params_for_function_params(fx.block_map[START_BLOCK]); - fx.bcx.switch_to_block(fx.block_map[START_BLOCK]); - // compilation should have been aborted - fx.bcx.ins().trap(TrapCode::UnreachableCodeReached); - } else if arg_uninhabited { - fx.bcx.append_block_params_for_function_params(fx.block_map[START_BLOCK]); - fx.bcx.switch_to_block(fx.block_map[START_BLOCK]); - fx.bcx.ins().trap(TrapCode::UnreachableCodeReached); - } else { - tcx.sess.time("codegen clif ir", || { - tcx.sess - .time("codegen prelude", || crate::abi::codegen_fn_prelude(&mut fx, start_block)); - codegen_fn_content(&mut fx); - }); - } + tcx.sess.time("codegen clif ir", || codegen_fn_body(&mut fx, start_block)); // Recover all necessary data from fx, before accessing func will prevent future access to it. let instance = fx.instance; @@ -124,36 +129,31 @@ pub(crate) fn codegen_fn<'tcx>( // Verify function verify_func(tcx, &clif_comments, &func); - compile_fn( - cx, - module, + CodegenedFunction { instance, - symbol_name.name, + symbol_name, func_id, func, clif_comments, source_info_set, local_map, - ); + } } fn compile_fn<'tcx>( cx: &mut crate::CodegenCx<'tcx>, + cached_context: &mut Context, module: &mut dyn Module, - instance: Instance<'tcx>, - symbol_name: &str, - func_id: FuncId, - func: Function, - mut clif_comments: CommentWriter, - source_info_set: IndexSet, - local_map: IndexVec>, + codegened_func: CodegenedFunction<'tcx>, ) { let tcx = cx.tcx; + let mut clif_comments = codegened_func.clif_comments; + // Store function in context - let context = &mut cx.cached_context; + let context = cached_context; context.clear(); - context.func = func; + context.func = codegened_func.func; // If the return block is not reachable, then the SSA builder may have inserted an `iconst.i128` // instruction, which doesn't have an encoding. @@ -170,7 +170,7 @@ fn compile_fn<'tcx>( crate::optimize::optimize_function( tcx, module.isa(), - instance, + codegened_func.instance, context, &mut clif_comments, ); @@ -206,7 +206,7 @@ fn compile_fn<'tcx>( // Define function tcx.sess.time("define function", || { context.want_disasm = crate::pretty_clif::should_write_ir(tcx); - module.define_function(func_id, context).unwrap(); + module.define_function(codegened_func.func_id, context).unwrap(); }); // Write optimized function to file for debugging @@ -214,7 +214,7 @@ fn compile_fn<'tcx>( tcx, "opt", module.isa(), - instance, + codegened_func.instance, &context.func, &clif_comments, ); @@ -222,7 +222,7 @@ fn compile_fn<'tcx>( if let Some(disasm) = &context.mach_compile_result.as_ref().unwrap().disasm { crate::pretty_clif::write_ir_file( tcx, - || format!("{}.vcode", tcx.symbol_name(instance).name), + || format!("{}.vcode", tcx.symbol_name(codegened_func.instance).name), |file| file.write_all(disasm.as_bytes()), ) } @@ -234,16 +234,16 @@ fn compile_fn<'tcx>( tcx.sess.time("generate debug info", || { if let Some(debug_context) = debug_context { debug_context.define_function( - instance, - func_id, - symbol_name, + codegened_func.instance, + codegened_func.func_id, + codegened_func.symbol_name.name, isa, context, - &source_info_set, - local_map, + &codegened_func.source_info_set, + codegened_func.local_map, ); } - unwind_context.add_function(func_id, &context, isa); + unwind_context.add_function(codegened_func.func_id, &context, isa); }); } @@ -269,7 +269,27 @@ pub(crate) fn verify_func( }); } -fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) { +fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) { + if !crate::constant::check_constants(fx) { + fx.bcx.append_block_params_for_function_params(fx.block_map[START_BLOCK]); + fx.bcx.switch_to_block(fx.block_map[START_BLOCK]); + // compilation should have been aborted + fx.bcx.ins().trap(TrapCode::UnreachableCodeReached); + return; + } + + let arg_uninhabited = fx + .mir + .args_iter() + .any(|arg| fx.layout_of(fx.monomorphize(fx.mir.local_decls[arg].ty)).abi.is_uninhabited()); + if arg_uninhabited { + fx.bcx.append_block_params_for_function_params(fx.block_map[START_BLOCK]); + fx.bcx.switch_to_block(fx.block_map[START_BLOCK]); + fx.bcx.ins().trap(TrapCode::UnreachableCodeReached); + return; + } + fx.tcx.sess.time("codegen prelude", || crate::abi::codegen_fn_prelude(fx, start_block)); + for (bb, bb_data) in fx.mir.basic_blocks().iter_enumerated() { let block = fx.get_block(bb); fx.bcx.switch_to_block(block); diff --git a/src/driver/aot.rs b/src/driver/aot.rs index 3cd1ef5639e..9d819e3995b 100644 --- a/src/driver/aot.rs +++ b/src/driver/aot.rs @@ -1,25 +1,31 @@ //! The AOT driver uses [`cranelift_object`] to write object files suitable for linking into a //! standalone executable. +use std::fs::File; use std::path::PathBuf; +use std::sync::Arc; -use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece}; use rustc_codegen_ssa::back::metadata::create_compressed_metadata_file; use rustc_codegen_ssa::{CodegenResults, CompiledModule, CrateInfo, ModuleKind}; +use rustc_data_structures::profiling::SelfProfilerRef; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_metadata::EncodedMetadata; use rustc_middle::dep_graph::{WorkProduct, WorkProductId}; use rustc_middle::mir::mono::{CodegenUnit, MonoItem}; use rustc_session::cgu_reuse_tracker::CguReuse; -use rustc_session::config::{DebugInfo, OutputType}; +use rustc_session::config::{DebugInfo, OutputFilenames, OutputType}; use rustc_session::Session; -use cranelift_codegen::isa::TargetIsa; use cranelift_object::{ObjectBuilder, ObjectModule}; +use crate::global_asm::GlobalAsmConfig; use crate::{prelude::*, BackendConfig}; -struct ModuleCodegenResult(CompiledModule, Option<(WorkProductId, WorkProduct)>); +struct ModuleCodegenResult { + module_regular: CompiledModule, + module_global_asm: Option, + existing_work_product: Option<(WorkProductId, WorkProduct)>, +} impl HashStable for ModuleCodegenResult { fn hash_stable(&self, _: &mut HCX, _: &mut StableHasher) { @@ -27,7 +33,79 @@ impl HashStable for ModuleCodegenResult { } } -fn make_module(sess: &Session, isa: Box, name: String) -> ObjectModule { +pub(crate) struct OngoingCodegen { + modules: Vec>, + allocator_module: Option, + metadata_module: Option, + metadata: EncodedMetadata, + crate_info: CrateInfo, +} + +impl OngoingCodegen { + pub(crate) fn join( + self, + sess: &Session, + backend_config: &BackendConfig, + ) -> (CodegenResults, FxHashMap) { + let mut work_products = FxHashMap::default(); + let mut modules = vec![]; + + for module_codegen_result in self.modules { + let module_codegen_result = match module_codegen_result { + Ok(module_codegen_result) => module_codegen_result, + Err(err) => sess.fatal(&err), + }; + let ModuleCodegenResult { module_regular, module_global_asm, existing_work_product } = + module_codegen_result; + + if let Some((work_product_id, work_product)) = existing_work_product { + work_products.insert(work_product_id, work_product); + } else { + let work_product = if backend_config.disable_incr_cache { + None + } else if let Some(module_global_asm) = &module_global_asm { + rustc_incremental::copy_cgu_workproduct_to_incr_comp_cache_dir( + sess, + &module_regular.name, + &[ + ("o", &module_regular.object.as_ref().unwrap()), + ("asm.o", &module_global_asm.object.as_ref().unwrap()), + ], + ) + } else { + rustc_incremental::copy_cgu_workproduct_to_incr_comp_cache_dir( + sess, + &module_regular.name, + &[("o", &module_regular.object.as_ref().unwrap())], + ) + }; + if let Some((work_product_id, work_product)) = work_product { + work_products.insert(work_product_id, work_product); + } + } + + modules.push(module_regular); + if let Some(module_global_asm) = module_global_asm { + modules.push(module_global_asm); + } + } + + ( + CodegenResults { + modules, + allocator_module: self.allocator_module, + metadata_module: self.metadata_module, + metadata: self.metadata, + crate_info: self.crate_info, + }, + work_products, + ) + } +} + +fn make_module(sess: &Session, backend_config: &BackendConfig, name: String) -> ObjectModule { + let isa = crate::build_isa(sess, backend_config); + let mut builder = ObjectBuilder::new(isa, name + ".o", cranelift_module::default_libcall_names()).unwrap(); // Unlike cg_llvm, cg_clif defaults to disabling -Zfunction-sections. For cg_llvm binary size @@ -37,15 +115,15 @@ fn make_module(sess: &Session, isa: Box, name: String) -> ObjectM ObjectModule::new(builder) } -fn emit_module( - tcx: TyCtxt<'_>, - backend_config: &BackendConfig, +fn emit_cgu( + output_filenames: &OutputFilenames, + prof: &SelfProfilerRef, name: String, - kind: ModuleKind, module: ObjectModule, debug: Option>, unwind_context: UnwindContext, -) -> ModuleCodegenResult { + global_asm_object_file: Option, +) -> Result { let mut product = module.finish(); if let Some(mut debug) = debug { @@ -54,71 +132,117 @@ fn emit_module( unwind_context.emit(&mut product); - let tmp_file = tcx.output_filenames(()).temp_path(OutputType::Object, Some(&name)); - let obj = product.object.write().unwrap(); + let module_regular = + emit_module(output_filenames, prof, product.object, ModuleKind::Regular, name.clone())?; - tcx.sess.prof.artifact_size("object_file", name.clone(), obj.len().try_into().unwrap()); + Ok(ModuleCodegenResult { + module_regular, + module_global_asm: global_asm_object_file.map(|global_asm_object_file| CompiledModule { + name: format!("{name}.asm"), + kind: ModuleKind::Regular, + object: Some(global_asm_object_file), + dwarf_object: None, + bytecode: None, + }), + existing_work_product: None, + }) +} - if let Err(err) = std::fs::write(&tmp_file, obj) { - tcx.sess.fatal(&format!("error writing object file: {}", err)); - } - - let work_product = if backend_config.disable_incr_cache { - None - } else { - rustc_incremental::copy_cgu_workproduct_to_incr_comp_cache_dir( - tcx.sess, - &name, - &[("o", &tmp_file)], - ) +fn emit_module( + output_filenames: &OutputFilenames, + prof: &SelfProfilerRef, + object: cranelift_object::object::write::Object<'_>, + kind: ModuleKind, + name: String, +) -> Result { + let tmp_file = output_filenames.temp_path(OutputType::Object, Some(&name)); + let mut file = match File::create(&tmp_file) { + Ok(file) => file, + Err(err) => return Err(format!("error creating object file: {}", err)), }; - ModuleCodegenResult( - CompiledModule { name, kind, object: Some(tmp_file), dwarf_object: None, bytecode: None }, - work_product, - ) + if let Err(err) = object.write_stream(&mut file) { + return Err(format!("error writing object file: {}", err)); + } + + prof.artifact_size("object_file", &*name, file.metadata().unwrap().len()); + + Ok(CompiledModule { name, kind, object: Some(tmp_file), dwarf_object: None, bytecode: None }) } fn reuse_workproduct_for_cgu( tcx: TyCtxt<'_>, cgu: &CodegenUnit<'_>, - work_products: &mut FxHashMap, -) -> CompiledModule { +) -> Result { let work_product = cgu.previous_work_product(tcx); - let obj_out = tcx.output_filenames(()).temp_path(OutputType::Object, Some(cgu.name().as_str())); - let source_file = rustc_incremental::in_incr_comp_dir_sess( + let obj_out_regular = + tcx.output_filenames(()).temp_path(OutputType::Object, Some(cgu.name().as_str())); + let source_file_regular = rustc_incremental::in_incr_comp_dir_sess( &tcx.sess, &work_product.saved_files.get("o").expect("no saved object file in work product"), ); - if let Err(err) = rustc_fs_util::link_or_copy(&source_file, &obj_out) { - tcx.sess.err(&format!( + + if let Err(err) = rustc_fs_util::link_or_copy(&source_file_regular, &obj_out_regular) { + return Err(format!( "unable to copy {} to {}: {}", - source_file.display(), - obj_out.display(), + source_file_regular.display(), + obj_out_regular.display(), err )); } + let obj_out_global_asm = + crate::global_asm::add_file_stem_postfix(obj_out_regular.clone(), ".asm"); + let has_global_asm = if let Some(asm_o) = work_product.saved_files.get("asm.o") { + let source_file_global_asm = rustc_incremental::in_incr_comp_dir_sess(&tcx.sess, asm_o); + if let Err(err) = rustc_fs_util::link_or_copy(&source_file_global_asm, &obj_out_global_asm) + { + return Err(format!( + "unable to copy {} to {}: {}", + source_file_regular.display(), + obj_out_regular.display(), + err + )); + } + true + } else { + false + }; - work_products.insert(cgu.work_product_id(), work_product); - - CompiledModule { - name: cgu.name().to_string(), - kind: ModuleKind::Regular, - object: Some(obj_out), - dwarf_object: None, - bytecode: None, - } + Ok(ModuleCodegenResult { + module_regular: CompiledModule { + name: cgu.name().to_string(), + kind: ModuleKind::Regular, + object: Some(obj_out_regular), + dwarf_object: None, + bytecode: None, + }, + module_global_asm: if has_global_asm { + Some(CompiledModule { + name: cgu.name().to_string(), + kind: ModuleKind::Regular, + object: Some(obj_out_global_asm), + dwarf_object: None, + bytecode: None, + }) + } else { + None + }, + existing_work_product: Some((cgu.work_product_id(), work_product)), + }) } fn module_codegen( tcx: TyCtxt<'_>, - (backend_config, cgu_name): (BackendConfig, rustc_span::Symbol), -) -> ModuleCodegenResult { + (backend_config, global_asm_config, cgu_name): ( + BackendConfig, + Arc, + rustc_span::Symbol, + ), +) -> Result { let cgu = tcx.codegen_unit(cgu_name); let mono_items = cgu.items_in_deterministic_order(tcx); - let isa = crate::build_isa(tcx.sess, &backend_config); - let mut module = make_module(tcx.sess, isa, cgu_name.as_str().to_string()); + let mut module = make_module(tcx.sess, &backend_config, cgu_name.as_str().to_string()); let mut cx = crate::CodegenCx::new( tcx, @@ -128,32 +252,22 @@ fn module_codegen( cgu_name, ); super::predefine_mono_items(tcx, &mut module, &mono_items); + let mut cached_context = Context::new(); for (mono_item, _) in mono_items { match mono_item { MonoItem::Fn(inst) => { - cx.tcx - .sess - .time("codegen fn", || crate::base::codegen_fn(&mut cx, &mut module, inst)); + cx.tcx.sess.time("codegen fn", || { + crate::base::codegen_and_compile_fn( + &mut cx, + &mut cached_context, + &mut module, + inst, + ) + }); } MonoItem::Static(def_id) => crate::constant::codegen_static(tcx, &mut module, def_id), MonoItem::GlobalAsm(item_id) => { - let item = cx.tcx.hir().item(item_id); - if let rustc_hir::ItemKind::GlobalAsm(asm) = item.kind { - if !asm.options.contains(InlineAsmOptions::ATT_SYNTAX) { - cx.global_asm.push_str("\n.intel_syntax noprefix\n"); - } else { - cx.global_asm.push_str("\n.att_syntax\n"); - } - for piece in asm.template { - match *piece { - InlineAsmTemplatePiece::String(ref s) => cx.global_asm.push_str(s), - InlineAsmTemplatePiece::Placeholder { .. } => todo!(), - } - } - cx.global_asm.push_str("\n.att_syntax\n\n"); - } else { - bug!("Expected GlobalAsm found {:?}", item); - } + crate::global_asm::codegen_global_asm_item(tcx, &mut cx.global_asm, item_id); } } } @@ -165,23 +279,28 @@ fn module_codegen( cgu.is_primary(), ); + let global_asm_object_file = match crate::global_asm::compile_global_asm( + &global_asm_config, + cgu.name().as_str(), + &cx.global_asm, + ) { + Ok(global_asm_object_file) => global_asm_object_file, + Err(err) => tcx.sess.fatal(&err), + }; + let debug_context = cx.debug_context; let unwind_context = cx.unwind_context; - let codegen_result = tcx.sess.time("write object file", || { - emit_module( - tcx, - &backend_config, + tcx.sess.time("write object file", || { + emit_cgu( + &global_asm_config.output_filenames, + &tcx.sess.prof, cgu.name().as_str().to_string(), - ModuleKind::Regular, module, debug_context, unwind_context, + global_asm_object_file, ) - }); - - codegen_global_asm(tcx, cgu.name().as_str(), &cx.global_asm); - - codegen_result + }) } pub(crate) fn run_aot( @@ -189,9 +308,7 @@ pub(crate) fn run_aot( backend_config: BackendConfig, metadata: EncodedMetadata, need_metadata_module: bool, -) -> Box<(CodegenResults, FxHashMap)> { - let mut work_products = FxHashMap::default(); - +) -> Box { let cgus = if tcx.sess.opts.output_types.should_codegen() { tcx.collect_and_partition_mono_items(()).1 } else { @@ -206,62 +323,59 @@ pub(crate) fn run_aot( } } + let global_asm_config = Arc::new(crate::global_asm::GlobalAsmConfig::new(tcx)); + let modules = super::time(tcx, backend_config.display_cg_time, "codegen mono items", || { cgus.iter() .map(|cgu| { - let cgu_reuse = determine_cgu_reuse(tcx, cgu); + let cgu_reuse = if backend_config.disable_incr_cache { + CguReuse::No + } else { + determine_cgu_reuse(tcx, cgu) + }; tcx.sess.cgu_reuse_tracker.set_actual_reuse(cgu.name().as_str(), cgu_reuse); match cgu_reuse { - _ if backend_config.disable_incr_cache => {} - CguReuse::No => {} - CguReuse::PreLto => { - return reuse_workproduct_for_cgu(tcx, &*cgu, &mut work_products); + CguReuse::No => { + let dep_node = cgu.codegen_dep_node(tcx); + tcx.dep_graph + .with_task( + dep_node, + tcx, + (backend_config.clone(), global_asm_config.clone(), cgu.name()), + module_codegen, + Some(rustc_middle::dep_graph::hash_result), + ) + .0 } + CguReuse::PreLto => reuse_workproduct_for_cgu(tcx, &*cgu), CguReuse::PostLto => unreachable!(), } - - let dep_node = cgu.codegen_dep_node(tcx); - let (ModuleCodegenResult(module, work_product), _) = tcx.dep_graph.with_task( - dep_node, - tcx, - (backend_config.clone(), cgu.name()), - module_codegen, - Some(rustc_middle::dep_graph::hash_result), - ); - - if let Some((id, product)) = work_product { - work_products.insert(id, product); - } - - module }) .collect::>() }); tcx.sess.abort_if_errors(); - let isa = crate::build_isa(tcx.sess, &backend_config); - let mut allocator_module = make_module(tcx.sess, isa, "allocator_shim".to_string()); - assert_eq!(pointer_ty(tcx), allocator_module.target_config().pointer_type()); + let mut allocator_module = make_module(tcx.sess, &backend_config, "allocator_shim".to_string()); let mut allocator_unwind_context = UnwindContext::new(allocator_module.isa(), true); let created_alloc_shim = crate::allocator::codegen(tcx, &mut allocator_module, &mut allocator_unwind_context); let allocator_module = if created_alloc_shim { - let ModuleCodegenResult(module, work_product) = emit_module( - tcx, - &backend_config, - "allocator_shim".to_string(), + let mut product = allocator_module.finish(); + allocator_unwind_context.emit(&mut product); + + match emit_module( + tcx.output_filenames(()), + &tcx.sess.prof, + product.object, ModuleKind::Allocator, - allocator_module, - None, - allocator_unwind_context, - ); - if let Some((id, product)) = work_product { - work_products.insert(id, product); + "allocator_shim".to_owned(), + ) { + Ok(allocator_module) => Some(allocator_module), + Err(err) => tcx.sess.fatal(err), } - Some(module) } else { None }; @@ -308,102 +422,13 @@ pub(crate) fn run_aot( } .to_owned(); - Box::new(( - CodegenResults { - modules, - allocator_module, - metadata_module, - metadata, - crate_info: CrateInfo::new(tcx, target_cpu), - }, - work_products, - )) -} - -fn codegen_global_asm(tcx: TyCtxt<'_>, cgu_name: &str, global_asm: &str) { - use std::io::Write; - use std::process::{Command, Stdio}; - - if global_asm.is_empty() { - return; - } - - if cfg!(not(feature = "inline_asm")) - || tcx.sess.target.is_like_osx - || tcx.sess.target.is_like_windows - { - if global_asm.contains("__rust_probestack") { - return; - } - - // FIXME fix linker error on macOS - if cfg!(not(feature = "inline_asm")) { - tcx.sess.fatal( - "asm! and global_asm! support is disabled while compiling rustc_codegen_cranelift", - ); - } else { - tcx.sess.fatal("asm! and global_asm! are not yet supported on macOS and Windows"); - } - } - - let assembler = crate::toolchain::get_toolchain_binary(tcx.sess, "as"); - let linker = crate::toolchain::get_toolchain_binary(tcx.sess, "ld"); - - // Remove all LLVM style comments - let global_asm = global_asm - .lines() - .map(|line| if let Some(index) = line.find("//") { &line[0..index] } else { line }) - .collect::>() - .join("\n"); - - let output_object_file = tcx.output_filenames(()).temp_path(OutputType::Object, Some(cgu_name)); - - // Assemble `global_asm` - let global_asm_object_file = add_file_stem_postfix(output_object_file.clone(), ".asm"); - let mut child = Command::new(assembler) - .arg("-o") - .arg(&global_asm_object_file) - .stdin(Stdio::piped()) - .spawn() - .expect("Failed to spawn `as`."); - child.stdin.take().unwrap().write_all(global_asm.as_bytes()).unwrap(); - let status = child.wait().expect("Failed to wait for `as`."); - if !status.success() { - tcx.sess.fatal(&format!("Failed to assemble `{}`", global_asm)); - } - - // Link the global asm and main object file together - let main_object_file = add_file_stem_postfix(output_object_file.clone(), ".main"); - std::fs::rename(&output_object_file, &main_object_file).unwrap(); - let status = Command::new(linker) - .arg("-r") // Create a new object file - .arg("-o") - .arg(output_object_file) - .arg(&main_object_file) - .arg(&global_asm_object_file) - .status() - .unwrap(); - if !status.success() { - tcx.sess.fatal(&format!( - "Failed to link `{}` and `{}` together", - main_object_file.display(), - global_asm_object_file.display(), - )); - } - - std::fs::remove_file(global_asm_object_file).unwrap(); - std::fs::remove_file(main_object_file).unwrap(); -} - -fn add_file_stem_postfix(mut path: PathBuf, postfix: &str) -> PathBuf { - let mut new_filename = path.file_stem().unwrap().to_owned(); - new_filename.push(postfix); - if let Some(extension) = path.extension() { - new_filename.push("."); - new_filename.push(extension); - } - path.set_file_name(new_filename); - path + Box::new(OngoingCodegen { + modules, + allocator_module, + metadata_module, + metadata, + crate_info: CrateInfo::new(tcx, target_cpu), + }) } // Adapted from https://github.com/rust-lang/rust/blob/303d8aff6092709edd4dbd35b1c88e9aa40bf6d8/src/librustc_codegen_ssa/base.rs#L922-L953 diff --git a/src/driver/jit.rs b/src/driver/jit.rs index a56a9100059..1b046d7ec6e 100644 --- a/src/driver/jit.rs +++ b/src/driver/jit.rs @@ -111,6 +111,7 @@ pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { &backend_config, matches!(backend_config.codegen_mode, CodegenMode::JitLazy), ); + let mut cached_context = Context::new(); let (_, cgus) = tcx.collect_and_partition_mono_items(()); let mono_items = cgus @@ -129,10 +130,17 @@ pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { CodegenMode::Aot => unreachable!(), CodegenMode::Jit => { cx.tcx.sess.time("codegen fn", || { - crate::base::codegen_fn(&mut cx, &mut jit_module, inst) + crate::base::codegen_and_compile_fn( + &mut cx, + &mut cached_context, + &mut jit_module, + inst, + ) }); } - CodegenMode::JitLazy => codegen_shim(&mut cx, &mut jit_module, inst), + CodegenMode::JitLazy => { + codegen_shim(&mut cx, &mut cached_context, &mut jit_module, inst) + } }, MonoItem::Static(def_id) => { crate::constant::codegen_static(tcx, &mut jit_module, def_id); @@ -259,7 +267,14 @@ fn jit_fn(instance_ptr: *const Instance<'static>, trampoline_ptr: *const u8) -> false, Symbol::intern("dummy_cgu_name"), ); - tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, jit_module, instance)); + tcx.sess.time("codegen fn", || { + crate::base::codegen_and_compile_fn( + &mut cx, + &mut Context::new(), + jit_module, + instance, + ) + }); assert!(cx.global_asm.is_empty()); jit_module.finalize_definitions(); @@ -334,7 +349,12 @@ fn load_imported_symbols_for_jit( imported_symbols } -fn codegen_shim<'tcx>(cx: &mut CodegenCx<'tcx>, module: &mut JITModule, inst: Instance<'tcx>) { +fn codegen_shim<'tcx>( + cx: &mut CodegenCx<'tcx>, + cached_context: &mut Context, + module: &mut JITModule, + inst: Instance<'tcx>, +) { let tcx = cx.tcx; let pointer_type = module.target_config().pointer_type(); @@ -357,8 +377,9 @@ fn codegen_shim<'tcx>(cx: &mut CodegenCx<'tcx>, module: &mut JITModule, inst: In ) .unwrap(); - cx.cached_context.clear(); - let trampoline = &mut cx.cached_context.func; + let context = cached_context; + context.clear(); + let trampoline = &mut context.func; trampoline.signature = sig.clone(); let mut builder_ctx = FunctionBuilderContext::new(); @@ -381,5 +402,5 @@ fn codegen_shim<'tcx>(cx: &mut CodegenCx<'tcx>, module: &mut JITModule, inst: In let ret_vals = trampoline_builder.func.dfg.inst_results(call_inst).to_vec(); trampoline_builder.ins().return_(&ret_vals); - module.define_function(func_id, &mut cx.cached_context).unwrap(); + module.define_function(func_id, context).unwrap(); } diff --git a/src/global_asm.rs b/src/global_asm.rs new file mode 100644 index 00000000000..dcbcaba30fe --- /dev/null +++ b/src/global_asm.rs @@ -0,0 +1,114 @@ +//! The AOT driver uses [`cranelift_object`] to write object files suitable for linking into a +//! standalone executable. + +use std::io::Write; +use std::path::PathBuf; +use std::process::{Command, Stdio}; +use std::sync::Arc; + +use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece}; +use rustc_hir::ItemId; +use rustc_session::config::{OutputFilenames, OutputType}; + +use crate::prelude::*; + +pub(crate) fn codegen_global_asm_item(tcx: TyCtxt<'_>, global_asm: &mut String, item_id: ItemId) { + let item = tcx.hir().item(item_id); + if let rustc_hir::ItemKind::GlobalAsm(asm) = item.kind { + if !asm.options.contains(InlineAsmOptions::ATT_SYNTAX) { + global_asm.push_str("\n.intel_syntax noprefix\n"); + } else { + global_asm.push_str("\n.att_syntax\n"); + } + for piece in asm.template { + match *piece { + InlineAsmTemplatePiece::String(ref s) => global_asm.push_str(s), + InlineAsmTemplatePiece::Placeholder { .. } => todo!(), + } + } + global_asm.push_str("\n.att_syntax\n\n"); + } else { + bug!("Expected GlobalAsm found {:?}", item); + } +} + +#[derive(Debug)] +pub(crate) struct GlobalAsmConfig { + asm_enabled: bool, + assembler: PathBuf, + pub(crate) output_filenames: Arc, +} + +impl GlobalAsmConfig { + pub(crate) fn new(tcx: TyCtxt<'_>) -> Self { + let asm_enabled = cfg!(feature = "inline_asm") && !tcx.sess.target.is_like_windows; + + GlobalAsmConfig { + asm_enabled, + assembler: crate::toolchain::get_toolchain_binary(tcx.sess, "as"), + output_filenames: tcx.output_filenames(()).clone(), + } + } +} + +pub(crate) fn compile_global_asm( + config: &GlobalAsmConfig, + cgu_name: &str, + global_asm: &str, +) -> Result, String> { + if global_asm.is_empty() { + return Ok(None); + } + + if !config.asm_enabled { + if global_asm.contains("__rust_probestack") { + return Ok(None); + } + + // FIXME fix linker error on macOS + if cfg!(not(feature = "inline_asm")) { + return Err( + "asm! and global_asm! support is disabled while compiling rustc_codegen_cranelift" + .to_owned(), + ); + } else { + return Err("asm! and global_asm! are not yet supported on Windows".to_owned()); + } + } + + // Remove all LLVM style comments + let global_asm = global_asm + .lines() + .map(|line| if let Some(index) = line.find("//") { &line[0..index] } else { line }) + .collect::>() + .join("\n"); + + let output_object_file = config.output_filenames.temp_path(OutputType::Object, Some(cgu_name)); + + // Assemble `global_asm` + let global_asm_object_file = add_file_stem_postfix(output_object_file.clone(), ".asm"); + let mut child = Command::new(&config.assembler) + .arg("-o") + .arg(&global_asm_object_file) + .stdin(Stdio::piped()) + .spawn() + .expect("Failed to spawn `as`."); + child.stdin.take().unwrap().write_all(global_asm.as_bytes()).unwrap(); + let status = child.wait().expect("Failed to wait for `as`."); + if !status.success() { + return Err(format!("Failed to assemble `{}`", global_asm)); + } + + Ok(Some(global_asm_object_file)) +} + +pub(crate) fn add_file_stem_postfix(mut path: PathBuf, postfix: &str) -> PathBuf { + let mut new_filename = path.file_stem().unwrap().to_owned(); + new_filename.push(postfix); + if let Some(extension) = path.extension() { + new_filename.push("."); + new_filename.push(extension); + } + path.set_file_name(new_filename); + path +} diff --git a/src/lib.rs b/src/lib.rs index bb0793b1deb..909f4f00f1e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -25,7 +25,7 @@ extern crate rustc_target; extern crate rustc_driver; use std::any::Any; -use std::cell::Cell; +use std::cell::{Cell, RefCell}; use rustc_codegen_ssa::traits::CodegenBackend; use rustc_codegen_ssa::CodegenResults; @@ -56,6 +56,7 @@ mod constant; mod debuginfo; mod discriminant; mod driver; +mod global_asm; mod inline_asm; mod intrinsics; mod linkage; @@ -123,7 +124,6 @@ struct CodegenCx<'tcx> { tcx: TyCtxt<'tcx>, global_asm: String, inline_asm_index: Cell, - cached_context: Context, debug_context: Option>, unwind_context: UnwindContext, cgu_name: Symbol, @@ -150,7 +150,6 @@ impl<'tcx> CodegenCx<'tcx> { tcx, global_asm: String::new(), inline_asm_index: Cell::new(0), - cached_context: Context::new(), debug_context, unwind_context, cgu_name, @@ -159,7 +158,7 @@ impl<'tcx> CodegenCx<'tcx> { } pub struct CraneliftCodegenBackend { - pub config: Option, + pub config: RefCell>, } impl CodegenBackend for CraneliftCodegenBackend { @@ -169,6 +168,13 @@ impl CodegenBackend for CraneliftCodegenBackend { Lto::No | Lto::ThinLocal => {} Lto::Thin | Lto::Fat => sess.warn("LTO is not supported. You may get a linker error."), } + + let mut config = self.config.borrow_mut(); + if config.is_none() { + let new_config = BackendConfig::from_opts(&sess.opts.cg.llvm_args) + .unwrap_or_else(|err| sess.fatal(&err)); + *config = Some(new_config); + } } fn target_features(&self, _sess: &Session, _allow_unstable: bool) -> Vec { @@ -186,15 +192,7 @@ impl CodegenBackend for CraneliftCodegenBackend { need_metadata_module: bool, ) -> Box { tcx.sess.abort_if_errors(); - let config = if let Some(config) = self.config.clone() { - config - } else { - if !tcx.sess.unstable_options() && !tcx.sess.opts.cg.llvm_args.is_empty() { - tcx.sess.fatal("`-Z unstable-options` must be passed to allow configuring cg_clif"); - } - BackendConfig::from_opts(&tcx.sess.opts.cg.llvm_args) - .unwrap_or_else(|err| tcx.sess.fatal(&err)) - }; + let config = self.config.borrow().clone().unwrap(); match config.codegen_mode { CodegenMode::Aot => driver::aot::run_aot(tcx, config, metadata, need_metadata_module), CodegenMode::Jit | CodegenMode::JitLazy => { @@ -210,12 +208,13 @@ impl CodegenBackend for CraneliftCodegenBackend { fn join_codegen( &self, ongoing_codegen: Box, - _sess: &Session, + sess: &Session, _outputs: &OutputFilenames, ) -> Result<(CodegenResults, FxHashMap), ErrorGuaranteed> { - Ok(*ongoing_codegen - .downcast::<(CodegenResults, FxHashMap)>() - .unwrap()) + Ok(ongoing_codegen + .downcast::() + .unwrap() + .join(sess, self.config.borrow().as_ref().unwrap())) } fn link( @@ -312,5 +311,5 @@ fn build_isa(sess: &Session, backend_config: &BackendConfig) -> Box Box { - Box::new(CraneliftCodegenBackend { config: None }) + Box::new(CraneliftCodegenBackend { config: RefCell::new(None) }) } diff --git a/src/toolchain.rs b/src/toolchain.rs index f86236ef3ea..b6b465e1f4e 100644 --- a/src/toolchain.rs +++ b/src/toolchain.rs @@ -8,10 +8,8 @@ use rustc_session::Session; /// Tries to infer the path of a binary for the target toolchain from the linker name. pub(crate) fn get_toolchain_binary(sess: &Session, tool: &str) -> PathBuf { let (mut linker, _linker_flavor) = linker_and_flavor(sess); - let linker_file_name = linker - .file_name() - .and_then(|name| name.to_str()) - .unwrap_or_else(|| sess.fatal("couldn't extract file name from specified linker")); + let linker_file_name = + linker.file_name().unwrap().to_str().expect("linker filename should be valid UTF-8"); if linker_file_name == "ld.lld" { if tool != "ld" {