From a7443a61ab2581ec9cdd3be08807ce599a5d56b8 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Wed, 10 Aug 2022 15:06:17 +0000 Subject: [PATCH 01/17] Move some code into codegen_fn_content --- src/base.rs | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/src/base.rs b/src/base.rs index 2c04cf47268..439b636031b 100644 --- a/src/base.rs +++ b/src/base.rs @@ -82,27 +82,7 @@ pub(crate) fn codegen_fn<'tcx>( next_ssa_var: 0, }; - let arg_uninhabited = fx - .mir - .args_iter() - .any(|arg| fx.layout_of(fx.monomorphize(fx.mir.local_decls[arg].ty)).abi.is_uninhabited()); - - if !crate::constant::check_constants(&mut fx) { - fx.bcx.append_block_params_for_function_params(fx.block_map[START_BLOCK]); - fx.bcx.switch_to_block(fx.block_map[START_BLOCK]); - // compilation should have been aborted - fx.bcx.ins().trap(TrapCode::UnreachableCodeReached); - } else if arg_uninhabited { - fx.bcx.append_block_params_for_function_params(fx.block_map[START_BLOCK]); - fx.bcx.switch_to_block(fx.block_map[START_BLOCK]); - fx.bcx.ins().trap(TrapCode::UnreachableCodeReached); - } else { - tcx.sess.time("codegen clif ir", || { - tcx.sess - .time("codegen prelude", || crate::abi::codegen_fn_prelude(&mut fx, start_block)); - codegen_fn_content(&mut fx); - }); - } + tcx.sess.time("codegen clif ir", || codegen_fn_body(&mut fx, start_block)); // Recover all necessary data from fx, before accessing func will prevent future access to it. let instance = fx.instance; @@ -269,7 +249,27 @@ pub(crate) fn verify_func( }); } -fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) { +fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) { + if !crate::constant::check_constants(fx) { + fx.bcx.append_block_params_for_function_params(fx.block_map[START_BLOCK]); + fx.bcx.switch_to_block(fx.block_map[START_BLOCK]); + // compilation should have been aborted + fx.bcx.ins().trap(TrapCode::UnreachableCodeReached); + return; + } + + let arg_uninhabited = fx + .mir + .args_iter() + .any(|arg| fx.layout_of(fx.monomorphize(fx.mir.local_decls[arg].ty)).abi.is_uninhabited()); + if arg_uninhabited { + fx.bcx.append_block_params_for_function_params(fx.block_map[START_BLOCK]); + fx.bcx.switch_to_block(fx.block_map[START_BLOCK]); + fx.bcx.ins().trap(TrapCode::UnreachableCodeReached); + return; + } + fx.tcx.sess.time("codegen prelude", || crate::abi::codegen_fn_prelude(fx, start_block)); + for (bb, bb_data) in fx.mir.basic_blocks().iter_enumerated() { let block = fx.get_block(bb); fx.bcx.switch_to_block(block); From a10da0f76898eca1ebd398748b98b205f4284297 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Wed, 10 Aug 2022 18:29:46 +0000 Subject: [PATCH 02/17] Split non-compile parts of codegen_fn out into a separate function The new codegen_and_compile_fn function only calls codegen_fn and then compile_fn. This makes it possible for both parts to be called separately by the driver. --- src/base.rs | 67 +++++++++++++++++++++++++++++------------------ src/driver/aot.rs | 6 ++--- src/driver/jit.rs | 6 +++-- 3 files changed, 49 insertions(+), 30 deletions(-) diff --git a/src/base.rs b/src/base.rs index 439b636031b..8e3f905166b 100644 --- a/src/base.rs +++ b/src/base.rs @@ -5,6 +5,7 @@ use rustc_middle::ty::adjustment::PointerCast; use rustc_middle::ty::layout::FnAbiOf; use rustc_middle::ty::print::with_no_trimmed_paths; +use rustc_middle::ty::SymbolName; use indexmap::IndexSet; @@ -12,17 +13,39 @@ use crate::prelude::*; use crate::pretty_clif::CommentWriter; -pub(crate) fn codegen_fn<'tcx>( +struct CodegenedFunction<'tcx> { + instance: Instance<'tcx>, + symbol_name: SymbolName<'tcx>, + func_id: FuncId, + func: Function, + clif_comments: CommentWriter, + source_info_set: IndexSet, + local_map: IndexVec>, +} + +pub(crate) fn codegen_and_compile_fn<'tcx>( cx: &mut crate::CodegenCx<'tcx>, module: &mut dyn Module, instance: Instance<'tcx>, ) { let tcx = cx.tcx; - let _inst_guard = crate::PrintOnPanic(|| format!("{:?} {}", instance, tcx.symbol_name(instance).name)); + + let codegened_func = codegen_fn(cx, module, instance); + + compile_fn(cx, module, codegened_func); +} + +fn codegen_fn<'tcx>( + cx: &mut crate::CodegenCx<'tcx>, + module: &mut dyn Module, + instance: Instance<'tcx>, +) -> CodegenedFunction<'tcx> { debug_assert!(!instance.substs.needs_infer()); + let tcx = cx.tcx; + let mir = tcx.instance_mir(instance.def); let _mir_guard = crate::PrintOnPanic(|| { let mut buf = Vec::new(); @@ -104,36 +127,30 @@ pub(crate) fn codegen_fn<'tcx>( // Verify function verify_func(tcx, &clif_comments, &func); - compile_fn( - cx, - module, + CodegenedFunction { instance, - symbol_name.name, + symbol_name, func_id, func, clif_comments, source_info_set, local_map, - ); + } } fn compile_fn<'tcx>( cx: &mut crate::CodegenCx<'tcx>, module: &mut dyn Module, - instance: Instance<'tcx>, - symbol_name: &str, - func_id: FuncId, - func: Function, - mut clif_comments: CommentWriter, - source_info_set: IndexSet, - local_map: IndexVec>, + codegened_func: CodegenedFunction<'tcx>, ) { let tcx = cx.tcx; + let mut clif_comments = codegened_func.clif_comments; + // Store function in context let context = &mut cx.cached_context; context.clear(); - context.func = func; + context.func = codegened_func.func; // If the return block is not reachable, then the SSA builder may have inserted an `iconst.i128` // instruction, which doesn't have an encoding. @@ -150,7 +167,7 @@ fn compile_fn<'tcx>( crate::optimize::optimize_function( tcx, module.isa(), - instance, + codegened_func.instance, context, &mut clif_comments, ); @@ -186,7 +203,7 @@ fn compile_fn<'tcx>( // Define function tcx.sess.time("define function", || { context.want_disasm = crate::pretty_clif::should_write_ir(tcx); - module.define_function(func_id, context).unwrap(); + module.define_function(codegened_func.func_id, context).unwrap(); }); // Write optimized function to file for debugging @@ -194,7 +211,7 @@ fn compile_fn<'tcx>( tcx, "opt", module.isa(), - instance, + codegened_func.instance, &context.func, &clif_comments, ); @@ -202,7 +219,7 @@ fn compile_fn<'tcx>( if let Some(disasm) = &context.mach_compile_result.as_ref().unwrap().disasm { crate::pretty_clif::write_ir_file( tcx, - || format!("{}.vcode", tcx.symbol_name(instance).name), + || format!("{}.vcode", tcx.symbol_name(codegened_func.instance).name), |file| file.write_all(disasm.as_bytes()), ) } @@ -214,16 +231,16 @@ fn compile_fn<'tcx>( tcx.sess.time("generate debug info", || { if let Some(debug_context) = debug_context { debug_context.define_function( - instance, - func_id, - symbol_name, + codegened_func.instance, + codegened_func.func_id, + codegened_func.symbol_name.name, isa, context, - &source_info_set, - local_map, + &codegened_func.source_info_set, + codegened_func.local_map, ); } - unwind_context.add_function(func_id, &context, isa); + unwind_context.add_function(codegened_func.func_id, &context, isa); }); } diff --git a/src/driver/aot.rs b/src/driver/aot.rs index 3cd1ef5639e..802e8ebd6f6 100644 --- a/src/driver/aot.rs +++ b/src/driver/aot.rs @@ -131,9 +131,9 @@ fn module_codegen( for (mono_item, _) in mono_items { match mono_item { MonoItem::Fn(inst) => { - cx.tcx - .sess - .time("codegen fn", || crate::base::codegen_fn(&mut cx, &mut module, inst)); + cx.tcx.sess.time("codegen fn", || { + crate::base::codegen_and_compile_fn(&mut cx, &mut module, inst) + }); } MonoItem::Static(def_id) => crate::constant::codegen_static(tcx, &mut module, def_id), MonoItem::GlobalAsm(item_id) => { diff --git a/src/driver/jit.rs b/src/driver/jit.rs index a56a9100059..a7ea2b182ab 100644 --- a/src/driver/jit.rs +++ b/src/driver/jit.rs @@ -129,7 +129,7 @@ pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { CodegenMode::Aot => unreachable!(), CodegenMode::Jit => { cx.tcx.sess.time("codegen fn", || { - crate::base::codegen_fn(&mut cx, &mut jit_module, inst) + crate::base::codegen_and_compile_fn(&mut cx, &mut jit_module, inst) }); } CodegenMode::JitLazy => codegen_shim(&mut cx, &mut jit_module, inst), @@ -259,7 +259,9 @@ fn jit_fn(instance_ptr: *const Instance<'static>, trampoline_ptr: *const u8) -> false, Symbol::intern("dummy_cgu_name"), ); - tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, jit_module, instance)); + tcx.sess.time("codegen fn", || { + crate::base::codegen_and_compile_fn(&mut cx, jit_module, instance) + }); assert!(cx.global_asm.is_empty()); jit_module.finalize_definitions(); From 8a336a2ae1ebcbafb47d5c9c8b30ea956aaa58f9 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Wed, 10 Aug 2022 18:47:05 +0000 Subject: [PATCH 03/17] Move cached_context out of CodegenCx --- src/base.rs | 15 +++++++++------ src/driver/aot.rs | 8 +++++++- src/driver/jit.rs | 33 ++++++++++++++++++++++++++------- src/lib.rs | 2 -- 4 files changed, 42 insertions(+), 16 deletions(-) diff --git a/src/base.rs b/src/base.rs index 8e3f905166b..c68d33465bc 100644 --- a/src/base.rs +++ b/src/base.rs @@ -25,6 +25,7 @@ struct CodegenedFunction<'tcx> { pub(crate) fn codegen_and_compile_fn<'tcx>( cx: &mut crate::CodegenCx<'tcx>, + cached_context: &mut Context, module: &mut dyn Module, instance: Instance<'tcx>, ) { @@ -32,13 +33,15 @@ pub(crate) fn codegen_and_compile_fn<'tcx>( let _inst_guard = crate::PrintOnPanic(|| format!("{:?} {}", instance, tcx.symbol_name(instance).name)); - let codegened_func = codegen_fn(cx, module, instance); + let cached_func = std::mem::replace(&mut cached_context.func, Function::new()); + let codegened_func = codegen_fn(cx, cached_func, module, instance); - compile_fn(cx, module, codegened_func); + compile_fn(cx, cached_context, module, codegened_func); } fn codegen_fn<'tcx>( cx: &mut crate::CodegenCx<'tcx>, + cached_func: Function, module: &mut dyn Module, instance: Instance<'tcx>, ) -> CodegenedFunction<'tcx> { @@ -61,11 +64,10 @@ fn codegen_fn<'tcx>( let sig = get_function_sig(tcx, module.isa().triple(), instance); let func_id = module.declare_function(symbol_name.name, Linkage::Local, &sig).unwrap(); - cx.cached_context.clear(); - // Make the FunctionBuilder let mut func_ctx = FunctionBuilderContext::new(); - let mut func = std::mem::replace(&mut cx.cached_context.func, Function::new()); + let mut func = cached_func; + func.clear(); func.name = ExternalName::user(0, func_id.as_u32()); func.signature = sig; func.collect_debug_info(); @@ -140,6 +142,7 @@ fn codegen_fn<'tcx>( fn compile_fn<'tcx>( cx: &mut crate::CodegenCx<'tcx>, + cached_context: &mut Context, module: &mut dyn Module, codegened_func: CodegenedFunction<'tcx>, ) { @@ -148,7 +151,7 @@ fn compile_fn<'tcx>( let mut clif_comments = codegened_func.clif_comments; // Store function in context - let context = &mut cx.cached_context; + let context = cached_context; context.clear(); context.func = codegened_func.func; diff --git a/src/driver/aot.rs b/src/driver/aot.rs index 802e8ebd6f6..6aa28637943 100644 --- a/src/driver/aot.rs +++ b/src/driver/aot.rs @@ -128,11 +128,17 @@ fn module_codegen( cgu_name, ); super::predefine_mono_items(tcx, &mut module, &mono_items); + let mut cached_context = Context::new(); for (mono_item, _) in mono_items { match mono_item { MonoItem::Fn(inst) => { cx.tcx.sess.time("codegen fn", || { - crate::base::codegen_and_compile_fn(&mut cx, &mut module, inst) + crate::base::codegen_and_compile_fn( + &mut cx, + &mut cached_context, + &mut module, + inst, + ) }); } MonoItem::Static(def_id) => crate::constant::codegen_static(tcx, &mut module, def_id), diff --git a/src/driver/jit.rs b/src/driver/jit.rs index a7ea2b182ab..1b046d7ec6e 100644 --- a/src/driver/jit.rs +++ b/src/driver/jit.rs @@ -111,6 +111,7 @@ pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { &backend_config, matches!(backend_config.codegen_mode, CodegenMode::JitLazy), ); + let mut cached_context = Context::new(); let (_, cgus) = tcx.collect_and_partition_mono_items(()); let mono_items = cgus @@ -129,10 +130,17 @@ pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { CodegenMode::Aot => unreachable!(), CodegenMode::Jit => { cx.tcx.sess.time("codegen fn", || { - crate::base::codegen_and_compile_fn(&mut cx, &mut jit_module, inst) + crate::base::codegen_and_compile_fn( + &mut cx, + &mut cached_context, + &mut jit_module, + inst, + ) }); } - CodegenMode::JitLazy => codegen_shim(&mut cx, &mut jit_module, inst), + CodegenMode::JitLazy => { + codegen_shim(&mut cx, &mut cached_context, &mut jit_module, inst) + } }, MonoItem::Static(def_id) => { crate::constant::codegen_static(tcx, &mut jit_module, def_id); @@ -260,7 +268,12 @@ fn jit_fn(instance_ptr: *const Instance<'static>, trampoline_ptr: *const u8) -> Symbol::intern("dummy_cgu_name"), ); tcx.sess.time("codegen fn", || { - crate::base::codegen_and_compile_fn(&mut cx, jit_module, instance) + crate::base::codegen_and_compile_fn( + &mut cx, + &mut Context::new(), + jit_module, + instance, + ) }); assert!(cx.global_asm.is_empty()); @@ -336,7 +349,12 @@ fn load_imported_symbols_for_jit( imported_symbols } -fn codegen_shim<'tcx>(cx: &mut CodegenCx<'tcx>, module: &mut JITModule, inst: Instance<'tcx>) { +fn codegen_shim<'tcx>( + cx: &mut CodegenCx<'tcx>, + cached_context: &mut Context, + module: &mut JITModule, + inst: Instance<'tcx>, +) { let tcx = cx.tcx; let pointer_type = module.target_config().pointer_type(); @@ -359,8 +377,9 @@ fn codegen_shim<'tcx>(cx: &mut CodegenCx<'tcx>, module: &mut JITModule, inst: In ) .unwrap(); - cx.cached_context.clear(); - let trampoline = &mut cx.cached_context.func; + let context = cached_context; + context.clear(); + let trampoline = &mut context.func; trampoline.signature = sig.clone(); let mut builder_ctx = FunctionBuilderContext::new(); @@ -383,5 +402,5 @@ fn codegen_shim<'tcx>(cx: &mut CodegenCx<'tcx>, module: &mut JITModule, inst: In let ret_vals = trampoline_builder.func.dfg.inst_results(call_inst).to_vec(); trampoline_builder.ins().return_(&ret_vals); - module.define_function(func_id, &mut cx.cached_context).unwrap(); + module.define_function(func_id, context).unwrap(); } diff --git a/src/lib.rs b/src/lib.rs index bb0793b1deb..a3f8cc4dfa3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -123,7 +123,6 @@ struct CodegenCx<'tcx> { tcx: TyCtxt<'tcx>, global_asm: String, inline_asm_index: Cell, - cached_context: Context, debug_context: Option>, unwind_context: UnwindContext, cgu_name: Symbol, @@ -150,7 +149,6 @@ fn new( tcx, global_asm: String::new(), inline_asm_index: Cell::new(0), - cached_context: Context::new(), debug_context, unwind_context, cgu_name, From 07bcd111f8d3b60dbc3722215c78f25372a11c6f Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Thu, 11 Aug 2022 13:38:07 +0000 Subject: [PATCH 04/17] Return ModuleCodegenResult from reuse_workproduct_for_cgu --- src/driver/aot.rs | 60 ++++++++++++++++++++++++----------------------- 1 file changed, 31 insertions(+), 29 deletions(-) diff --git a/src/driver/aot.rs b/src/driver/aot.rs index 6aa28637943..6f1732f9707 100644 --- a/src/driver/aot.rs +++ b/src/driver/aot.rs @@ -79,11 +79,7 @@ fn emit_module( ) } -fn reuse_workproduct_for_cgu( - tcx: TyCtxt<'_>, - cgu: &CodegenUnit<'_>, - work_products: &mut FxHashMap, -) -> CompiledModule { +fn reuse_workproduct_for_cgu(tcx: TyCtxt<'_>, cgu: &CodegenUnit<'_>) -> ModuleCodegenResult { let work_product = cgu.previous_work_product(tcx); let obj_out = tcx.output_filenames(()).temp_path(OutputType::Object, Some(cgu.name().as_str())); let source_file = rustc_incremental::in_incr_comp_dir_sess( @@ -99,15 +95,16 @@ fn reuse_workproduct_for_cgu( )); } - work_products.insert(cgu.work_product_id(), work_product); - - CompiledModule { - name: cgu.name().to_string(), - kind: ModuleKind::Regular, - object: Some(obj_out), - dwarf_object: None, - bytecode: None, - } + ModuleCodegenResult( + CompiledModule { + name: cgu.name().to_string(), + kind: ModuleKind::Regular, + object: Some(obj_out), + dwarf_object: None, + bytecode: None, + }, + Some((cgu.work_product_id(), work_product)), + ) } fn module_codegen( @@ -215,26 +212,31 @@ pub(crate) fn run_aot( let modules = super::time(tcx, backend_config.display_cg_time, "codegen mono items", || { cgus.iter() .map(|cgu| { - let cgu_reuse = determine_cgu_reuse(tcx, cgu); + let cgu_reuse = if backend_config.disable_incr_cache { + CguReuse::No + } else { + determine_cgu_reuse(tcx, cgu) + }; tcx.sess.cgu_reuse_tracker.set_actual_reuse(cgu.name().as_str(), cgu_reuse); - match cgu_reuse { - _ if backend_config.disable_incr_cache => {} - CguReuse::No => {} - CguReuse::PreLto => { - return reuse_workproduct_for_cgu(tcx, &*cgu, &mut work_products); + let module_codegen_result = match cgu_reuse { + CguReuse::No => { + let dep_node = cgu.codegen_dep_node(tcx); + tcx.dep_graph + .with_task( + dep_node, + tcx, + (backend_config.clone(), cgu.name()), + module_codegen, + Some(rustc_middle::dep_graph::hash_result), + ) + .0 } + CguReuse::PreLto => reuse_workproduct_for_cgu(tcx, &*cgu), CguReuse::PostLto => unreachable!(), - } + }; - let dep_node = cgu.codegen_dep_node(tcx); - let (ModuleCodegenResult(module, work_product), _) = tcx.dep_graph.with_task( - dep_node, - tcx, - (backend_config.clone(), cgu.name()), - module_codegen, - Some(rustc_middle::dep_graph::hash_result), - ); + let ModuleCodegenResult(module, work_product) = module_codegen_result; if let Some((id, product)) = work_product { work_products.insert(id, product); From c5adc96532205a12c94c1407e6b6b35f7c7a2b64 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Thu, 11 Aug 2022 13:49:08 +0000 Subject: [PATCH 05/17] Introduce OngoingCodegen type --- src/driver/aot.rs | 67 ++++++++++++++++++++++++++++++++--------------- src/lib.rs | 4 +-- 2 files changed, 47 insertions(+), 24 deletions(-) diff --git a/src/driver/aot.rs b/src/driver/aot.rs index 6f1732f9707..c417de04ab4 100644 --- a/src/driver/aot.rs +++ b/src/driver/aot.rs @@ -27,6 +27,41 @@ fn hash_stable(&self, _: &mut HCX, _: &mut StableHasher) { } } +pub(crate) struct OngoingCodegen { + modules: Vec, + allocator_module: Option, + metadata_module: Option, + metadata: EncodedMetadata, + crate_info: CrateInfo, + work_products: FxHashMap, +} + +impl OngoingCodegen { + pub(crate) fn join(self) -> (CodegenResults, FxHashMap) { + let mut work_products = self.work_products; + let mut modules = vec![]; + + for module_codegen_result in self.modules { + let ModuleCodegenResult(module, work_product) = module_codegen_result; + if let Some((work_product_id, work_product)) = work_product { + work_products.insert(work_product_id, work_product); + } + modules.push(module); + } + + ( + CodegenResults { + modules, + allocator_module: self.allocator_module, + metadata_module: self.metadata_module, + metadata: self.metadata, + crate_info: self.crate_info, + }, + work_products, + ) + } +} + fn make_module(sess: &Session, isa: Box, name: String) -> ObjectModule { let mut builder = ObjectBuilder::new(isa, name + ".o", cranelift_module::default_libcall_names()).unwrap(); @@ -192,9 +227,7 @@ pub(crate) fn run_aot( backend_config: BackendConfig, metadata: EncodedMetadata, need_metadata_module: bool, -) -> Box<(CodegenResults, FxHashMap)> { - let mut work_products = FxHashMap::default(); - +) -> Box { let cgus = if tcx.sess.opts.output_types.should_codegen() { tcx.collect_and_partition_mono_items(()).1 } else { @@ -219,7 +252,7 @@ pub(crate) fn run_aot( }; tcx.sess.cgu_reuse_tracker.set_actual_reuse(cgu.name().as_str(), cgu_reuse); - let module_codegen_result = match cgu_reuse { + match cgu_reuse { CguReuse::No => { let dep_node = cgu.codegen_dep_node(tcx); tcx.dep_graph @@ -234,21 +267,15 @@ pub(crate) fn run_aot( } CguReuse::PreLto => reuse_workproduct_for_cgu(tcx, &*cgu), CguReuse::PostLto => unreachable!(), - }; - - let ModuleCodegenResult(module, work_product) = module_codegen_result; - - if let Some((id, product)) = work_product { - work_products.insert(id, product); } - - module }) .collect::>() }); tcx.sess.abort_if_errors(); + let mut work_products = FxHashMap::default(); + let isa = crate::build_isa(tcx.sess, &backend_config); let mut allocator_module = make_module(tcx.sess, isa, "allocator_shim".to_string()); assert_eq!(pointer_ty(tcx), allocator_module.target_config().pointer_type()); @@ -316,16 +343,14 @@ pub(crate) fn run_aot( } .to_owned(); - Box::new(( - CodegenResults { - modules, - allocator_module, - metadata_module, - metadata, - crate_info: CrateInfo::new(tcx, target_cpu), - }, + Box::new(OngoingCodegen { + modules, + allocator_module, + metadata_module, + metadata, + crate_info: CrateInfo::new(tcx, target_cpu), work_products, - )) + }) } fn codegen_global_asm(tcx: TyCtxt<'_>, cgu_name: &str, global_asm: &str) { diff --git a/src/lib.rs b/src/lib.rs index a3f8cc4dfa3..49d10012c4f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -211,9 +211,7 @@ fn join_codegen( _sess: &Session, _outputs: &OutputFilenames, ) -> Result<(CodegenResults, FxHashMap), ErrorGuaranteed> { - Ok(*ongoing_codegen - .downcast::<(CodegenResults, FxHashMap)>() - .unwrap()) + Ok(ongoing_codegen.downcast::().unwrap().join()) } fn link( From 7cc97ebcbb3fe637e2467f2e4fc45d94dae013e8 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Fri, 12 Aug 2022 09:11:47 +0000 Subject: [PATCH 06/17] Extract global_asm module --- src/driver/aot.rs | 109 +------------------------------------------ src/global_asm.rs | 116 ++++++++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 1 + 3 files changed, 119 insertions(+), 107 deletions(-) create mode 100644 src/global_asm.rs diff --git a/src/driver/aot.rs b/src/driver/aot.rs index c417de04ab4..6482dce2746 100644 --- a/src/driver/aot.rs +++ b/src/driver/aot.rs @@ -1,9 +1,6 @@ //! The AOT driver uses [`cranelift_object`] to write object files suitable for linking into a //! standalone executable. -use std::path::PathBuf; - -use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece}; use rustc_codegen_ssa::back::metadata::create_compressed_metadata_file; use rustc_codegen_ssa::{CodegenResults, CompiledModule, CrateInfo, ModuleKind}; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; @@ -175,23 +172,7 @@ fn module_codegen( } MonoItem::Static(def_id) => crate::constant::codegen_static(tcx, &mut module, def_id), MonoItem::GlobalAsm(item_id) => { - let item = cx.tcx.hir().item(item_id); - if let rustc_hir::ItemKind::GlobalAsm(asm) = item.kind { - if !asm.options.contains(InlineAsmOptions::ATT_SYNTAX) { - cx.global_asm.push_str("\n.intel_syntax noprefix\n"); - } else { - cx.global_asm.push_str("\n.att_syntax\n"); - } - for piece in asm.template { - match *piece { - InlineAsmTemplatePiece::String(ref s) => cx.global_asm.push_str(s), - InlineAsmTemplatePiece::Placeholder { .. } => todo!(), - } - } - cx.global_asm.push_str("\n.att_syntax\n\n"); - } else { - bug!("Expected GlobalAsm found {:?}", item); - } + crate::global_asm::codegen_global_asm_item(tcx, &mut cx.global_asm, item_id); } } } @@ -217,7 +198,7 @@ fn module_codegen( ) }); - codegen_global_asm(tcx, cgu.name().as_str(), &cx.global_asm); + crate::global_asm::compile_global_asm(tcx, cgu.name().as_str(), &cx.global_asm); codegen_result } @@ -353,92 +334,6 @@ pub(crate) fn run_aot( }) } -fn codegen_global_asm(tcx: TyCtxt<'_>, cgu_name: &str, global_asm: &str) { - use std::io::Write; - use std::process::{Command, Stdio}; - - if global_asm.is_empty() { - return; - } - - if cfg!(not(feature = "inline_asm")) - || tcx.sess.target.is_like_osx - || tcx.sess.target.is_like_windows - { - if global_asm.contains("__rust_probestack") { - return; - } - - // FIXME fix linker error on macOS - if cfg!(not(feature = "inline_asm")) { - tcx.sess.fatal( - "asm! and global_asm! support is disabled while compiling rustc_codegen_cranelift", - ); - } else { - tcx.sess.fatal("asm! and global_asm! are not yet supported on macOS and Windows"); - } - } - - let assembler = crate::toolchain::get_toolchain_binary(tcx.sess, "as"); - let linker = crate::toolchain::get_toolchain_binary(tcx.sess, "ld"); - - // Remove all LLVM style comments - let global_asm = global_asm - .lines() - .map(|line| if let Some(index) = line.find("//") { &line[0..index] } else { line }) - .collect::>() - .join("\n"); - - let output_object_file = tcx.output_filenames(()).temp_path(OutputType::Object, Some(cgu_name)); - - // Assemble `global_asm` - let global_asm_object_file = add_file_stem_postfix(output_object_file.clone(), ".asm"); - let mut child = Command::new(assembler) - .arg("-o") - .arg(&global_asm_object_file) - .stdin(Stdio::piped()) - .spawn() - .expect("Failed to spawn `as`."); - child.stdin.take().unwrap().write_all(global_asm.as_bytes()).unwrap(); - let status = child.wait().expect("Failed to wait for `as`."); - if !status.success() { - tcx.sess.fatal(&format!("Failed to assemble `{}`", global_asm)); - } - - // Link the global asm and main object file together - let main_object_file = add_file_stem_postfix(output_object_file.clone(), ".main"); - std::fs::rename(&output_object_file, &main_object_file).unwrap(); - let status = Command::new(linker) - .arg("-r") // Create a new object file - .arg("-o") - .arg(output_object_file) - .arg(&main_object_file) - .arg(&global_asm_object_file) - .status() - .unwrap(); - if !status.success() { - tcx.sess.fatal(&format!( - "Failed to link `{}` and `{}` together", - main_object_file.display(), - global_asm_object_file.display(), - )); - } - - std::fs::remove_file(global_asm_object_file).unwrap(); - std::fs::remove_file(main_object_file).unwrap(); -} - -fn add_file_stem_postfix(mut path: PathBuf, postfix: &str) -> PathBuf { - let mut new_filename = path.file_stem().unwrap().to_owned(); - new_filename.push(postfix); - if let Some(extension) = path.extension() { - new_filename.push("."); - new_filename.push(extension); - } - path.set_file_name(new_filename); - path -} - // Adapted from https://github.com/rust-lang/rust/blob/303d8aff6092709edd4dbd35b1c88e9aa40bf6d8/src/librustc_codegen_ssa/base.rs#L922-L953 fn determine_cgu_reuse<'tcx>(tcx: TyCtxt<'tcx>, cgu: &CodegenUnit<'tcx>) -> CguReuse { if !tcx.dep_graph.is_fully_enabled() { diff --git a/src/global_asm.rs b/src/global_asm.rs new file mode 100644 index 00000000000..5962a86a686 --- /dev/null +++ b/src/global_asm.rs @@ -0,0 +1,116 @@ +//! The AOT driver uses [`cranelift_object`] to write object files suitable for linking into a +//! standalone executable. + +use std::path::PathBuf; + +use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece}; +use rustc_hir::ItemId; +use rustc_session::config::OutputType; + +use crate::prelude::*; + +pub(crate) fn codegen_global_asm_item(tcx: TyCtxt<'_>, global_asm: &mut String, item_id: ItemId) { + let item = tcx.hir().item(item_id); + if let rustc_hir::ItemKind::GlobalAsm(asm) = item.kind { + if !asm.options.contains(InlineAsmOptions::ATT_SYNTAX) { + global_asm.push_str("\n.intel_syntax noprefix\n"); + } else { + global_asm.push_str("\n.att_syntax\n"); + } + for piece in asm.template { + match *piece { + InlineAsmTemplatePiece::String(ref s) => global_asm.push_str(s), + InlineAsmTemplatePiece::Placeholder { .. } => todo!(), + } + } + global_asm.push_str("\n.att_syntax\n\n"); + } else { + bug!("Expected GlobalAsm found {:?}", item); + } +} + +pub(crate) fn compile_global_asm(tcx: TyCtxt<'_>, cgu_name: &str, global_asm: &str) { + use std::io::Write; + use std::process::{Command, Stdio}; + + if global_asm.is_empty() { + return; + } + + if cfg!(not(feature = "inline_asm")) + || tcx.sess.target.is_like_osx + || tcx.sess.target.is_like_windows + { + if global_asm.contains("__rust_probestack") { + return; + } + + // FIXME fix linker error on macOS + if cfg!(not(feature = "inline_asm")) { + tcx.sess.fatal( + "asm! and global_asm! support is disabled while compiling rustc_codegen_cranelift", + ); + } else { + tcx.sess.fatal("asm! and global_asm! are not yet supported on macOS and Windows"); + } + } + + let assembler = crate::toolchain::get_toolchain_binary(tcx.sess, "as"); + let linker = crate::toolchain::get_toolchain_binary(tcx.sess, "ld"); + + // Remove all LLVM style comments + let global_asm = global_asm + .lines() + .map(|line| if let Some(index) = line.find("//") { &line[0..index] } else { line }) + .collect::>() + .join("\n"); + + let output_object_file = tcx.output_filenames(()).temp_path(OutputType::Object, Some(cgu_name)); + + // Assemble `global_asm` + let global_asm_object_file = add_file_stem_postfix(output_object_file.clone(), ".asm"); + let mut child = Command::new(assembler) + .arg("-o") + .arg(&global_asm_object_file) + .stdin(Stdio::piped()) + .spawn() + .expect("Failed to spawn `as`."); + child.stdin.take().unwrap().write_all(global_asm.as_bytes()).unwrap(); + let status = child.wait().expect("Failed to wait for `as`."); + if !status.success() { + tcx.sess.fatal(&format!("Failed to assemble `{}`", global_asm)); + } + + // Link the global asm and main object file together + let main_object_file = add_file_stem_postfix(output_object_file.clone(), ".main"); + std::fs::rename(&output_object_file, &main_object_file).unwrap(); + let status = Command::new(linker) + .arg("-r") // Create a new object file + .arg("-o") + .arg(output_object_file) + .arg(&main_object_file) + .arg(&global_asm_object_file) + .status() + .unwrap(); + if !status.success() { + tcx.sess.fatal(&format!( + "Failed to link `{}` and `{}` together", + main_object_file.display(), + global_asm_object_file.display(), + )); + } + + std::fs::remove_file(global_asm_object_file).unwrap(); + std::fs::remove_file(main_object_file).unwrap(); +} + +fn add_file_stem_postfix(mut path: PathBuf, postfix: &str) -> PathBuf { + let mut new_filename = path.file_stem().unwrap().to_owned(); + new_filename.push(postfix); + if let Some(extension) = path.extension() { + new_filename.push("."); + new_filename.push(extension); + } + path.set_file_name(new_filename); + path +} diff --git a/src/lib.rs b/src/lib.rs index 49d10012c4f..6ea160d26ce 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -56,6 +56,7 @@ mod debuginfo; mod discriminant; mod driver; +mod global_asm; mod inline_asm; mod intrinsics; mod linkage; From 066f844fff7b6bf227c375a293fe15af88cf85ac Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Fri, 12 Aug 2022 09:28:41 +0000 Subject: [PATCH 07/17] Move some sess.fatal calls out of compile_global_asm --- src/driver/aot.rs | 5 ++++- src/global_asm.rs | 43 +++++++++++++++++++++++++++++-------------- src/toolchain.rs | 6 ++---- 3 files changed, 35 insertions(+), 19 deletions(-) diff --git a/src/driver/aot.rs b/src/driver/aot.rs index 6482dce2746..0816ebc4599 100644 --- a/src/driver/aot.rs +++ b/src/driver/aot.rs @@ -198,7 +198,10 @@ fn module_codegen( ) }); - crate::global_asm::compile_global_asm(tcx, cgu.name().as_str(), &cx.global_asm); + match crate::global_asm::compile_global_asm(tcx, cgu.name().as_str(), &cx.global_asm) { + Ok(()) => {} + Err(err) => tcx.sess.fatal(&err.to_string()), + } codegen_result } diff --git a/src/global_asm.rs b/src/global_asm.rs index 5962a86a686..5cd7abfdfb5 100644 --- a/src/global_asm.rs +++ b/src/global_asm.rs @@ -1,7 +1,9 @@ //! The AOT driver uses [`cranelift_object`] to write object files suitable for linking into a //! standalone executable. +use std::io::{self, Write}; use std::path::PathBuf; +use std::process::{Command, Stdio}; use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece}; use rustc_hir::ItemId; @@ -29,12 +31,13 @@ pub(crate) fn codegen_global_asm_item(tcx: TyCtxt<'_>, global_asm: &mut String, } } -pub(crate) fn compile_global_asm(tcx: TyCtxt<'_>, cgu_name: &str, global_asm: &str) { - use std::io::Write; - use std::process::{Command, Stdio}; - +pub(crate) fn compile_global_asm( + tcx: TyCtxt<'_>, + cgu_name: &str, + global_asm: &str, +) -> io::Result<()> { if global_asm.is_empty() { - return; + return Ok(()); } if cfg!(not(feature = "inline_asm")) @@ -42,16 +45,20 @@ pub(crate) fn compile_global_asm(tcx: TyCtxt<'_>, cgu_name: &str, global_asm: &s || tcx.sess.target.is_like_windows { if global_asm.contains("__rust_probestack") { - return; + return Ok(()); } // FIXME fix linker error on macOS if cfg!(not(feature = "inline_asm")) { - tcx.sess.fatal( + return Err(io::Error::new( + io::ErrorKind::Unsupported, "asm! and global_asm! support is disabled while compiling rustc_codegen_cranelift", - ); + )); } else { - tcx.sess.fatal("asm! and global_asm! are not yet supported on macOS and Windows"); + return Err(io::Error::new( + io::ErrorKind::Unsupported, + "asm! and global_asm! are not yet supported on macOS and Windows", + )); } } @@ -78,7 +85,10 @@ pub(crate) fn compile_global_asm(tcx: TyCtxt<'_>, cgu_name: &str, global_asm: &s child.stdin.take().unwrap().write_all(global_asm.as_bytes()).unwrap(); let status = child.wait().expect("Failed to wait for `as`."); if !status.success() { - tcx.sess.fatal(&format!("Failed to assemble `{}`", global_asm)); + return Err(io::Error::new( + io::ErrorKind::Other, + format!("Failed to assemble `{}`", global_asm), + )); } // Link the global asm and main object file together @@ -93,15 +103,20 @@ pub(crate) fn compile_global_asm(tcx: TyCtxt<'_>, cgu_name: &str, global_asm: &s .status() .unwrap(); if !status.success() { - tcx.sess.fatal(&format!( - "Failed to link `{}` and `{}` together", - main_object_file.display(), - global_asm_object_file.display(), + return Err(io::Error::new( + io::ErrorKind::Other, + format!( + "Failed to link `{}` and `{}` together", + main_object_file.display(), + global_asm_object_file.display(), + ), )); } std::fs::remove_file(global_asm_object_file).unwrap(); std::fs::remove_file(main_object_file).unwrap(); + + Ok(()) } fn add_file_stem_postfix(mut path: PathBuf, postfix: &str) -> PathBuf { diff --git a/src/toolchain.rs b/src/toolchain.rs index f86236ef3ea..b6b465e1f4e 100644 --- a/src/toolchain.rs +++ b/src/toolchain.rs @@ -8,10 +8,8 @@ /// Tries to infer the path of a binary for the target toolchain from the linker name. pub(crate) fn get_toolchain_binary(sess: &Session, tool: &str) -> PathBuf { let (mut linker, _linker_flavor) = linker_and_flavor(sess); - let linker_file_name = linker - .file_name() - .and_then(|name| name.to_str()) - .unwrap_or_else(|| sess.fatal("couldn't extract file name from specified linker")); + let linker_file_name = + linker.file_name().unwrap().to_str().expect("linker filename should be valid UTF-8"); if linker_file_name == "ld.lld" { if tool != "ld" { From 48b312f04a89c72cf6db2cbb08bbc7fe6fce9bdb Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Fri, 12 Aug 2022 09:55:59 +0000 Subject: [PATCH 08/17] Don't take TyCtxt as argument for compile_global_asm This allows it to be executed on a background thread. --- src/driver/aot.rs | 21 ++++++++++--- src/global_asm.rs | 77 +++++++++++++++++++++++++++-------------------- 2 files changed, 61 insertions(+), 37 deletions(-) diff --git a/src/driver/aot.rs b/src/driver/aot.rs index 0816ebc4599..0ca634affb4 100644 --- a/src/driver/aot.rs +++ b/src/driver/aot.rs @@ -1,6 +1,8 @@ //! The AOT driver uses [`cranelift_object`] to write object files suitable for linking into a //! standalone executable. +use std::sync::Arc; + use rustc_codegen_ssa::back::metadata::create_compressed_metadata_file; use rustc_codegen_ssa::{CodegenResults, CompiledModule, CrateInfo, ModuleKind}; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; @@ -14,6 +16,7 @@ use cranelift_codegen::isa::TargetIsa; use cranelift_object::{ObjectBuilder, ObjectModule}; +use crate::global_asm::GlobalAsmConfig; use crate::{prelude::*, BackendConfig}; struct ModuleCodegenResult(CompiledModule, Option<(WorkProductId, WorkProduct)>); @@ -141,7 +144,11 @@ fn reuse_workproduct_for_cgu(tcx: TyCtxt<'_>, cgu: &CodegenUnit<'_>) -> ModuleCo fn module_codegen( tcx: TyCtxt<'_>, - (backend_config, cgu_name): (BackendConfig, rustc_span::Symbol), + (backend_config, global_asm_config, cgu_name): ( + BackendConfig, + Arc, + rustc_span::Symbol, + ), ) -> ModuleCodegenResult { let cgu = tcx.codegen_unit(cgu_name); let mono_items = cgu.items_in_deterministic_order(tcx); @@ -198,9 +205,13 @@ fn module_codegen( ) }); - match crate::global_asm::compile_global_asm(tcx, cgu.name().as_str(), &cx.global_asm) { + match crate::global_asm::compile_global_asm( + &global_asm_config, + cgu.name().as_str(), + &cx.global_asm, + ) { Ok(()) => {} - Err(err) => tcx.sess.fatal(&err.to_string()), + Err(err) => tcx.sess.fatal(&err), } codegen_result @@ -226,6 +237,8 @@ pub(crate) fn run_aot( } } + let global_asm_config = Arc::new(crate::global_asm::GlobalAsmConfig::new(tcx)); + let modules = super::time(tcx, backend_config.display_cg_time, "codegen mono items", || { cgus.iter() .map(|cgu| { @@ -243,7 +256,7 @@ pub(crate) fn run_aot( .with_task( dep_node, tcx, - (backend_config.clone(), cgu.name()), + (backend_config.clone(), global_asm_config.clone(), cgu.name()), module_codegen, Some(rustc_middle::dep_graph::hash_result), ) diff --git a/src/global_asm.rs b/src/global_asm.rs index 5cd7abfdfb5..14288e99242 100644 --- a/src/global_asm.rs +++ b/src/global_asm.rs @@ -1,13 +1,14 @@ //! The AOT driver uses [`cranelift_object`] to write object files suitable for linking into a //! standalone executable. -use std::io::{self, Write}; +use std::io::Write; use std::path::PathBuf; use std::process::{Command, Stdio}; +use std::sync::Arc; use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece}; use rustc_hir::ItemId; -use rustc_session::config::OutputType; +use rustc_session::config::{OutputFilenames, OutputType}; use crate::prelude::*; @@ -31,40 +32,56 @@ pub(crate) fn codegen_global_asm_item(tcx: TyCtxt<'_>, global_asm: &mut String, } } +#[derive(Debug)] +pub(crate) struct GlobalAsmConfig { + asm_enabled: bool, + assembler: PathBuf, + linker: PathBuf, + output_filenames: Arc, +} + +impl GlobalAsmConfig { + pub(crate) fn new(tcx: TyCtxt<'_>) -> Self { + let asm_enabled = cfg!(feature = "inline_asm") + && !tcx.sess.target.is_like_osx + && !tcx.sess.target.is_like_windows; + + GlobalAsmConfig { + asm_enabled, + assembler: crate::toolchain::get_toolchain_binary(tcx.sess, "as"), + linker: crate::toolchain::get_toolchain_binary(tcx.sess, "ld"), + output_filenames: tcx.output_filenames(()).clone(), + } + } +} + pub(crate) fn compile_global_asm( - tcx: TyCtxt<'_>, + config: &GlobalAsmConfig, cgu_name: &str, global_asm: &str, -) -> io::Result<()> { +) -> Result<(), String> { if global_asm.is_empty() { return Ok(()); } - if cfg!(not(feature = "inline_asm")) - || tcx.sess.target.is_like_osx - || tcx.sess.target.is_like_windows - { + if !config.asm_enabled { if global_asm.contains("__rust_probestack") { return Ok(()); } // FIXME fix linker error on macOS if cfg!(not(feature = "inline_asm")) { - return Err(io::Error::new( - io::ErrorKind::Unsupported, - "asm! and global_asm! support is disabled while compiling rustc_codegen_cranelift", - )); + return Err( + "asm! and global_asm! support is disabled while compiling rustc_codegen_cranelift" + .to_owned(), + ); } else { - return Err(io::Error::new( - io::ErrorKind::Unsupported, - "asm! and global_asm! are not yet supported on macOS and Windows", - )); + return Err( + "asm! and global_asm! are not yet supported on macOS and Windows".to_owned() + ); } } - let assembler = crate::toolchain::get_toolchain_binary(tcx.sess, "as"); - let linker = crate::toolchain::get_toolchain_binary(tcx.sess, "ld"); - // Remove all LLVM style comments let global_asm = global_asm .lines() @@ -72,11 +89,11 @@ pub(crate) fn compile_global_asm( .collect::>() .join("\n"); - let output_object_file = tcx.output_filenames(()).temp_path(OutputType::Object, Some(cgu_name)); + let output_object_file = config.output_filenames.temp_path(OutputType::Object, Some(cgu_name)); // Assemble `global_asm` let global_asm_object_file = add_file_stem_postfix(output_object_file.clone(), ".asm"); - let mut child = Command::new(assembler) + let mut child = Command::new(&config.assembler) .arg("-o") .arg(&global_asm_object_file) .stdin(Stdio::piped()) @@ -85,16 +102,13 @@ pub(crate) fn compile_global_asm( child.stdin.take().unwrap().write_all(global_asm.as_bytes()).unwrap(); let status = child.wait().expect("Failed to wait for `as`."); if !status.success() { - return Err(io::Error::new( - io::ErrorKind::Other, - format!("Failed to assemble `{}`", global_asm), - )); + return Err(format!("Failed to assemble `{}`", global_asm)); } // Link the global asm and main object file together let main_object_file = add_file_stem_postfix(output_object_file.clone(), ".main"); std::fs::rename(&output_object_file, &main_object_file).unwrap(); - let status = Command::new(linker) + let status = Command::new(&config.linker) .arg("-r") // Create a new object file .arg("-o") .arg(output_object_file) @@ -103,13 +117,10 @@ pub(crate) fn compile_global_asm( .status() .unwrap(); if !status.success() { - return Err(io::Error::new( - io::ErrorKind::Other, - format!( - "Failed to link `{}` and `{}` together", - main_object_file.display(), - global_asm_object_file.display(), - ), + return Err(format!( + "Failed to link `{}` and `{}` together", + main_object_file.display(), + global_asm_object_file.display(), )); } From e45f6000a0bd46d4b7580db59c86f3d30adbc270 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Fri, 12 Aug 2022 12:27:47 +0000 Subject: [PATCH 09/17] Remove the partial linking hack for global asm support --- src/driver/aot.rs | 102 +++++++++++++++++++++++++++++++++++++--------- src/global_asm.rs | 34 +++------------- 2 files changed, 87 insertions(+), 49 deletions(-) diff --git a/src/driver/aot.rs b/src/driver/aot.rs index 0ca634affb4..4122ce18224 100644 --- a/src/driver/aot.rs +++ b/src/driver/aot.rs @@ -1,6 +1,7 @@ //! The AOT driver uses [`cranelift_object`] to write object files suitable for linking into a //! standalone executable. +use std::path::PathBuf; use std::sync::Arc; use rustc_codegen_ssa::back::metadata::create_compressed_metadata_file; @@ -19,7 +20,11 @@ use crate::global_asm::GlobalAsmConfig; use crate::{prelude::*, BackendConfig}; -struct ModuleCodegenResult(CompiledModule, Option<(WorkProductId, WorkProduct)>); +struct ModuleCodegenResult( + CompiledModule, + Option, + Option<(WorkProductId, WorkProduct)>, +); impl HashStable for ModuleCodegenResult { fn hash_stable(&self, _: &mut HCX, _: &mut StableHasher) { @@ -42,11 +47,15 @@ pub(crate) fn join(self) -> (CodegenResults, FxHashMap>, unwind_context: UnwindContext, + global_asm_object_file: Option, ) -> ModuleCodegenResult { let mut product = module.finish(); @@ -100,6 +110,12 @@ fn emit_module( let work_product = if backend_config.disable_incr_cache { None + } else if let Some(global_asm_object_file) = &global_asm_object_file { + rustc_incremental::copy_cgu_workproduct_to_incr_comp_cache_dir( + tcx.sess, + &name, + &[("o", &tmp_file), ("asm.o", global_asm_object_file)], + ) } else { rustc_incremental::copy_cgu_workproduct_to_incr_comp_cache_dir( tcx.sess, @@ -109,35 +125,78 @@ fn emit_module( }; ModuleCodegenResult( - CompiledModule { name, kind, object: Some(tmp_file), dwarf_object: None, bytecode: None }, + CompiledModule { + name: name.clone(), + kind, + object: Some(tmp_file), + dwarf_object: None, + bytecode: None, + }, + global_asm_object_file.map(|global_asm_object_file| CompiledModule { + name: format!("{name}.asm"), + kind, + object: Some(global_asm_object_file), + dwarf_object: None, + bytecode: None, + }), work_product, ) } fn reuse_workproduct_for_cgu(tcx: TyCtxt<'_>, cgu: &CodegenUnit<'_>) -> ModuleCodegenResult { let work_product = cgu.previous_work_product(tcx); - let obj_out = tcx.output_filenames(()).temp_path(OutputType::Object, Some(cgu.name().as_str())); - let source_file = rustc_incremental::in_incr_comp_dir_sess( + let obj_out_regular = + tcx.output_filenames(()).temp_path(OutputType::Object, Some(cgu.name().as_str())); + let source_file_regular = rustc_incremental::in_incr_comp_dir_sess( &tcx.sess, &work_product.saved_files.get("o").expect("no saved object file in work product"), ); - if let Err(err) = rustc_fs_util::link_or_copy(&source_file, &obj_out) { + + if let Err(err) = rustc_fs_util::link_or_copy(&source_file_regular, &obj_out_regular) { tcx.sess.err(&format!( "unable to copy {} to {}: {}", - source_file.display(), - obj_out.display(), + source_file_regular.display(), + obj_out_regular.display(), err )); } + let obj_out_global_asm = + crate::global_asm::add_file_stem_postfix(obj_out_regular.clone(), ".asm"); + let has_global_asm = if let Some(asm_o) = work_product.saved_files.get("asm.o") { + let source_file_global_asm = rustc_incremental::in_incr_comp_dir_sess(&tcx.sess, asm_o); + if let Err(err) = rustc_fs_util::link_or_copy(&source_file_global_asm, &obj_out_global_asm) + { + tcx.sess.err(&format!( + "unable to copy {} to {}: {}", + source_file_regular.display(), + obj_out_regular.display(), + err + )); + } + true + } else { + false + }; ModuleCodegenResult( CompiledModule { name: cgu.name().to_string(), kind: ModuleKind::Regular, - object: Some(obj_out), + object: Some(obj_out_regular), dwarf_object: None, bytecode: None, }, + if has_global_asm { + Some(CompiledModule { + name: cgu.name().to_string(), + kind: ModuleKind::Regular, + object: Some(obj_out_global_asm), + dwarf_object: None, + bytecode: None, + }) + } else { + None + }, Some((cgu.work_product_id(), work_product)), ) } @@ -191,6 +250,15 @@ fn module_codegen( cgu.is_primary(), ); + let global_asm_object_file = match crate::global_asm::compile_global_asm( + &global_asm_config, + cgu.name().as_str(), + &cx.global_asm, + ) { + Ok(global_asm_object_file) => global_asm_object_file, + Err(err) => tcx.sess.fatal(&err), + }; + let debug_context = cx.debug_context; let unwind_context = cx.unwind_context; let codegen_result = tcx.sess.time("write object file", || { @@ -202,18 +270,10 @@ fn module_codegen( module, debug_context, unwind_context, + global_asm_object_file, ) }); - match crate::global_asm::compile_global_asm( - &global_asm_config, - cgu.name().as_str(), - &cx.global_asm, - ) { - Ok(()) => {} - Err(err) => tcx.sess.fatal(&err), - } - codegen_result } @@ -281,7 +341,7 @@ pub(crate) fn run_aot( crate::allocator::codegen(tcx, &mut allocator_module, &mut allocator_unwind_context); let allocator_module = if created_alloc_shim { - let ModuleCodegenResult(module, work_product) = emit_module( + let ModuleCodegenResult(module, module_global_asm, work_product) = emit_module( tcx, &backend_config, "allocator_shim".to_string(), @@ -289,7 +349,9 @@ pub(crate) fn run_aot( allocator_module, None, allocator_unwind_context, + None, ); + assert!(module_global_asm.is_none()); if let Some((id, product)) = work_product { work_products.insert(id, product); } diff --git a/src/global_asm.rs b/src/global_asm.rs index 14288e99242..8e711988f81 100644 --- a/src/global_asm.rs +++ b/src/global_asm.rs @@ -36,7 +36,6 @@ pub(crate) fn codegen_global_asm_item(tcx: TyCtxt<'_>, global_asm: &mut String, pub(crate) struct GlobalAsmConfig { asm_enabled: bool, assembler: PathBuf, - linker: PathBuf, output_filenames: Arc, } @@ -49,7 +48,6 @@ pub(crate) fn new(tcx: TyCtxt<'_>) -> Self { GlobalAsmConfig { asm_enabled, assembler: crate::toolchain::get_toolchain_binary(tcx.sess, "as"), - linker: crate::toolchain::get_toolchain_binary(tcx.sess, "ld"), output_filenames: tcx.output_filenames(()).clone(), } } @@ -59,14 +57,14 @@ pub(crate) fn compile_global_asm( config: &GlobalAsmConfig, cgu_name: &str, global_asm: &str, -) -> Result<(), String> { +) -> Result, String> { if global_asm.is_empty() { - return Ok(()); + return Ok(None); } if !config.asm_enabled { if global_asm.contains("__rust_probestack") { - return Ok(()); + return Ok(None); } // FIXME fix linker error on macOS @@ -105,32 +103,10 @@ pub(crate) fn compile_global_asm( return Err(format!("Failed to assemble `{}`", global_asm)); } - // Link the global asm and main object file together - let main_object_file = add_file_stem_postfix(output_object_file.clone(), ".main"); - std::fs::rename(&output_object_file, &main_object_file).unwrap(); - let status = Command::new(&config.linker) - .arg("-r") // Create a new object file - .arg("-o") - .arg(output_object_file) - .arg(&main_object_file) - .arg(&global_asm_object_file) - .status() - .unwrap(); - if !status.success() { - return Err(format!( - "Failed to link `{}` and `{}` together", - main_object_file.display(), - global_asm_object_file.display(), - )); - } - - std::fs::remove_file(global_asm_object_file).unwrap(); - std::fs::remove_file(main_object_file).unwrap(); - - Ok(()) + Ok(Some(global_asm_object_file)) } -fn add_file_stem_postfix(mut path: PathBuf, postfix: &str) -> PathBuf { +pub(crate) fn add_file_stem_postfix(mut path: PathBuf, postfix: &str) -> PathBuf { let mut new_filename = path.file_stem().unwrap().to_owned(); new_filename.push(postfix); if let Some(extension) = path.extension() { From f76ca2247998bff4e10b73fcb464a0a83edbfeb0 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Fri, 12 Aug 2022 12:30:24 +0000 Subject: [PATCH 10/17] Enable inline asm on macOS --- Readme.md | 4 +--- example/mini_core_hello_world.rs | 14 ++++++++++++-- src/global_asm.rs | 8 ++------ 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/Readme.md b/Readme.md index 8a2db5a43ec..1e84c7fa365 100644 --- a/Readme.md +++ b/Readme.md @@ -52,9 +52,7 @@ configuration options. ## Not yet supported * Inline assembly ([no cranelift support](https://github.com/bytecodealliance/wasmtime/issues/1041)) - * On Linux there is support for invoking an external assembler for `global_asm!` and `asm!`. - `llvm_asm!` will remain unimplemented forever. `asm!` doesn't yet support reg classes. You - have to specify specific registers instead. + * On UNIX there is support for invoking an external assembler for `global_asm!` and `asm!`. * SIMD ([tracked here](https://github.com/bjorn3/rustc_codegen_cranelift/issues/171), some basic things work) ## License diff --git a/example/mini_core_hello_world.rs b/example/mini_core_hello_world.rs index 7e9cbe1bba5..e83be3a3df5 100644 --- a/example/mini_core_hello_world.rs +++ b/example/mini_core_hello_world.rs @@ -321,7 +321,7 @@ struct ExternTypeWrapper { #[cfg(not(any(jit, windows)))] test_tls(); - #[cfg(all(not(jit), target_arch = "x86_64", target_os = "linux"))] + #[cfg(all(not(jit), target_arch = "x86_64", any(target_os = "linux", target_os = "darwin")))] unsafe { global_asm_test(); } @@ -343,7 +343,7 @@ fn main() { } } -#[cfg(all(not(jit), target_arch = "x86_64", target_os = "linux"))] +#[cfg(all(not(jit), target_arch = "x86_64", any(target_os = "linux", target_os = "darwin")))] extern "C" { fn global_asm_test(); } @@ -358,6 +358,16 @@ fn main() { " } +#[cfg(all(not(jit), target_arch = "x86_64", target_os = "darwin"))] +global_asm! { + " + .global _global_asm_test + _global_asm_test: + // comment that would normally be removed by LLVM + ret + " +} + #[repr(C)] enum c_void { _1, diff --git a/src/global_asm.rs b/src/global_asm.rs index 8e711988f81..917a6fff727 100644 --- a/src/global_asm.rs +++ b/src/global_asm.rs @@ -41,9 +41,7 @@ pub(crate) struct GlobalAsmConfig { impl GlobalAsmConfig { pub(crate) fn new(tcx: TyCtxt<'_>) -> Self { - let asm_enabled = cfg!(feature = "inline_asm") - && !tcx.sess.target.is_like_osx - && !tcx.sess.target.is_like_windows; + let asm_enabled = cfg!(feature = "inline_asm") && !tcx.sess.target.is_like_windows; GlobalAsmConfig { asm_enabled, @@ -74,9 +72,7 @@ pub(crate) fn compile_global_asm( .to_owned(), ); } else { - return Err( - "asm! and global_asm! are not yet supported on macOS and Windows".to_owned() - ); + return Err("asm! and global_asm! are not yet supported on Windows".to_owned()); } } From db7d8a811d646cc9f30eb550d2aed7ff3530bb40 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Fri, 12 Aug 2022 13:03:18 +0000 Subject: [PATCH 11/17] Give fields of ModuleCodegenResult names --- src/driver/aot.rs | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/src/driver/aot.rs b/src/driver/aot.rs index 4122ce18224..3bbd286b3d7 100644 --- a/src/driver/aot.rs +++ b/src/driver/aot.rs @@ -20,11 +20,11 @@ use crate::global_asm::GlobalAsmConfig; use crate::{prelude::*, BackendConfig}; -struct ModuleCodegenResult( - CompiledModule, - Option, - Option<(WorkProductId, WorkProduct)>, -); +struct ModuleCodegenResult { + module_regular: CompiledModule, + module_global_asm: Option, + work_product: Option<(WorkProductId, WorkProduct)>, +} impl HashStable for ModuleCodegenResult { fn hash_stable(&self, _: &mut HCX, _: &mut StableHasher) { @@ -47,7 +47,7 @@ pub(crate) fn join(self) -> (CodegenResults, FxHashMap, cgu: &CodegenUnit<'_>) -> ModuleCodegenResult { @@ -178,15 +178,15 @@ fn reuse_workproduct_for_cgu(tcx: TyCtxt<'_>, cgu: &CodegenUnit<'_>) -> ModuleCo false }; - ModuleCodegenResult( - CompiledModule { + ModuleCodegenResult { + module_regular: CompiledModule { name: cgu.name().to_string(), kind: ModuleKind::Regular, object: Some(obj_out_regular), dwarf_object: None, bytecode: None, }, - if has_global_asm { + module_global_asm: if has_global_asm { Some(CompiledModule { name: cgu.name().to_string(), kind: ModuleKind::Regular, @@ -197,8 +197,8 @@ fn reuse_workproduct_for_cgu(tcx: TyCtxt<'_>, cgu: &CodegenUnit<'_>) -> ModuleCo } else { None }, - Some((cgu.work_product_id(), work_product)), - ) + work_product: Some((cgu.work_product_id(), work_product)), + } } fn module_codegen( @@ -341,7 +341,7 @@ pub(crate) fn run_aot( crate::allocator::codegen(tcx, &mut allocator_module, &mut allocator_unwind_context); let allocator_module = if created_alloc_shim { - let ModuleCodegenResult(module, module_global_asm, work_product) = emit_module( + let ModuleCodegenResult { module_regular, module_global_asm, work_product } = emit_module( tcx, &backend_config, "allocator_shim".to_string(), @@ -355,7 +355,7 @@ pub(crate) fn run_aot( if let Some((id, product)) = work_product { work_products.insert(id, product); } - Some(module) + Some(module_regular) } else { None }; From d3512b1d8e04ab8ab78ea5111a177605d13dfff1 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Fri, 12 Aug 2022 13:15:51 +0000 Subject: [PATCH 12/17] Don't attempt to do incr comp for the allocator shim The allocator shim doesn't get reused and the allocator shim is just under 2kb, so reusing it is likely more expensive than regenerating it. --- src/driver/aot.rs | 40 +++++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/src/driver/aot.rs b/src/driver/aot.rs index 3bbd286b3d7..ee89d0701aa 100644 --- a/src/driver/aot.rs +++ b/src/driver/aot.rs @@ -38,12 +38,11 @@ pub(crate) struct OngoingCodegen { metadata_module: Option, metadata: EncodedMetadata, crate_info: CrateInfo, - work_products: FxHashMap, } impl OngoingCodegen { pub(crate) fn join(self) -> (CodegenResults, FxHashMap) { - let mut work_products = self.work_products; + let mut work_products = FxHashMap::default(); let mut modules = vec![]; for module_codegen_result in self.modules { @@ -331,8 +330,6 @@ pub(crate) fn run_aot( tcx.sess.abort_if_errors(); - let mut work_products = FxHashMap::default(); - let isa = crate::build_isa(tcx.sess, &backend_config); let mut allocator_module = make_module(tcx.sess, isa, "allocator_shim".to_string()); assert_eq!(pointer_ty(tcx), allocator_module.target_config().pointer_type()); @@ -341,21 +338,27 @@ pub(crate) fn run_aot( crate::allocator::codegen(tcx, &mut allocator_module, &mut allocator_unwind_context); let allocator_module = if created_alloc_shim { - let ModuleCodegenResult { module_regular, module_global_asm, work_product } = emit_module( - tcx, - &backend_config, - "allocator_shim".to_string(), - ModuleKind::Allocator, - allocator_module, - None, - allocator_unwind_context, - None, - ); - assert!(module_global_asm.is_none()); - if let Some((id, product)) = work_product { - work_products.insert(id, product); + let name = "allocator_shim".to_owned(); + + let mut product = allocator_module.finish(); + allocator_unwind_context.emit(&mut product); + + let tmp_file = tcx.output_filenames(()).temp_path(OutputType::Object, Some(&name)); + let obj = product.object.write().unwrap(); + + tcx.sess.prof.artifact_size("object_file", &*name, obj.len().try_into().unwrap()); + + if let Err(err) = std::fs::write(&tmp_file, obj) { + tcx.sess.fatal(&format!("error writing object file: {}", err)); } - Some(module_regular) + + Some(CompiledModule { + name, + kind: ModuleKind::Allocator, + object: Some(tmp_file), + dwarf_object: None, + bytecode: None, + }) } else { None }; @@ -408,7 +411,6 @@ pub(crate) fn run_aot( metadata_module, metadata, crate_info: CrateInfo::new(tcx, target_cpu), - work_products, }) } From ab7c706306c65527b5b8b9620e34a65d852cd279 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Fri, 12 Aug 2022 18:40:48 +0000 Subject: [PATCH 13/17] Move build_isa call into make_module --- src/driver/aot.rs | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/driver/aot.rs b/src/driver/aot.rs index ee89d0701aa..415631ba3e6 100644 --- a/src/driver/aot.rs +++ b/src/driver/aot.rs @@ -14,7 +14,6 @@ use rustc_session::config::{DebugInfo, OutputType}; use rustc_session::Session; -use cranelift_codegen::isa::TargetIsa; use cranelift_object::{ObjectBuilder, ObjectModule}; use crate::global_asm::GlobalAsmConfig; @@ -70,7 +69,9 @@ pub(crate) fn join(self) -> (CodegenResults, FxHashMap, name: String) -> ObjectModule { +fn make_module(sess: &Session, backend_config: &BackendConfig, name: String) -> ObjectModule { + let isa = crate::build_isa(sess, backend_config); + let mut builder = ObjectBuilder::new(isa, name + ".o", cranelift_module::default_libcall_names()).unwrap(); // Unlike cg_llvm, cg_clif defaults to disabling -Zfunction-sections. For cg_llvm binary size @@ -211,8 +212,7 @@ fn module_codegen( let cgu = tcx.codegen_unit(cgu_name); let mono_items = cgu.items_in_deterministic_order(tcx); - let isa = crate::build_isa(tcx.sess, &backend_config); - let mut module = make_module(tcx.sess, isa, cgu_name.as_str().to_string()); + let mut module = make_module(tcx.sess, &backend_config, cgu_name.as_str().to_string()); let mut cx = crate::CodegenCx::new( tcx, @@ -330,9 +330,7 @@ pub(crate) fn run_aot( tcx.sess.abort_if_errors(); - let isa = crate::build_isa(tcx.sess, &backend_config); - let mut allocator_module = make_module(tcx.sess, isa, "allocator_shim".to_string()); - assert_eq!(pointer_ty(tcx), allocator_module.target_config().pointer_type()); + let mut allocator_module = make_module(tcx.sess, &backend_config, "allocator_shim".to_string()); let mut allocator_unwind_context = UnwindContext::new(allocator_module.isa(), true); let created_alloc_shim = crate::allocator::codegen(tcx, &mut allocator_module, &mut allocator_unwind_context); From 6206c4e927595afcaef0496025512282f847f645 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Fri, 12 Aug 2022 18:55:39 +0000 Subject: [PATCH 14/17] Stream object file to disk This reduces memory usage and may improve performance slightly. --- src/driver/aot.rs | 70 ++++++++++++++++++++--------------------------- 1 file changed, 30 insertions(+), 40 deletions(-) diff --git a/src/driver/aot.rs b/src/driver/aot.rs index 415631ba3e6..ff6310bd8af 100644 --- a/src/driver/aot.rs +++ b/src/driver/aot.rs @@ -1,6 +1,7 @@ //! The AOT driver uses [`cranelift_object`] to write object files suitable for linking into a //! standalone executable. +use std::fs::File; use std::path::PathBuf; use std::sync::Arc; @@ -81,11 +82,10 @@ fn make_module(sess: &Session, backend_config: &BackendConfig, name: String) -> ObjectModule::new(builder) } -fn emit_module( +fn emit_cgu( tcx: TyCtxt<'_>, backend_config: &BackendConfig, name: String, - kind: ModuleKind, module: ObjectModule, debug: Option>, unwind_context: UnwindContext, @@ -99,14 +99,7 @@ fn emit_module( unwind_context.emit(&mut product); - let tmp_file = tcx.output_filenames(()).temp_path(OutputType::Object, Some(&name)); - let obj = product.object.write().unwrap(); - - tcx.sess.prof.artifact_size("object_file", name.clone(), obj.len().try_into().unwrap()); - - if let Err(err) = std::fs::write(&tmp_file, obj) { - tcx.sess.fatal(&format!("error writing object file: {}", err)); - } + let module_regular = emit_module(tcx, product.object, ModuleKind::Regular, name.clone()); let work_product = if backend_config.disable_incr_cache { None @@ -114,27 +107,21 @@ fn emit_module( rustc_incremental::copy_cgu_workproduct_to_incr_comp_cache_dir( tcx.sess, &name, - &[("o", &tmp_file), ("asm.o", global_asm_object_file)], + &[("o", &module_regular.object.as_ref().unwrap()), ("asm.o", global_asm_object_file)], ) } else { rustc_incremental::copy_cgu_workproduct_to_incr_comp_cache_dir( tcx.sess, &name, - &[("o", &tmp_file)], + &[("o", &module_regular.object.as_ref().unwrap())], ) }; ModuleCodegenResult { - module_regular: CompiledModule { - name: name.clone(), - kind, - object: Some(tmp_file), - dwarf_object: None, - bytecode: None, - }, + module_regular, module_global_asm: global_asm_object_file.map(|global_asm_object_file| CompiledModule { name: format!("{name}.asm"), - kind, + kind: ModuleKind::Regular, object: Some(global_asm_object_file), dwarf_object: None, bytecode: None, @@ -143,6 +130,27 @@ fn emit_module( } } +fn emit_module( + tcx: TyCtxt<'_>, + object: cranelift_object::object::write::Object<'_>, + kind: ModuleKind, + name: String, +) -> CompiledModule { + let tmp_file = tcx.output_filenames(()).temp_path(OutputType::Object, Some(&name)); + let mut file = match File::create(&tmp_file) { + Ok(file) => file, + Err(err) => tcx.sess.fatal(&format!("error creating object file: {}", err)), + }; + + if let Err(err) = object.write_stream(&mut file) { + tcx.sess.fatal(&format!("error writing object file: {}", err)); + } + + tcx.sess.prof.artifact_size("object_file", &*name, file.metadata().unwrap().len()); + + CompiledModule { name, kind, object: Some(tmp_file), dwarf_object: None, bytecode: None } +} + fn reuse_workproduct_for_cgu(tcx: TyCtxt<'_>, cgu: &CodegenUnit<'_>) -> ModuleCodegenResult { let work_product = cgu.previous_work_product(tcx); let obj_out_regular = @@ -261,11 +269,10 @@ fn module_codegen( let debug_context = cx.debug_context; let unwind_context = cx.unwind_context; let codegen_result = tcx.sess.time("write object file", || { - emit_module( + emit_cgu( tcx, &backend_config, cgu.name().as_str().to_string(), - ModuleKind::Regular, module, debug_context, unwind_context, @@ -336,27 +343,10 @@ pub(crate) fn run_aot( crate::allocator::codegen(tcx, &mut allocator_module, &mut allocator_unwind_context); let allocator_module = if created_alloc_shim { - let name = "allocator_shim".to_owned(); - let mut product = allocator_module.finish(); allocator_unwind_context.emit(&mut product); - let tmp_file = tcx.output_filenames(()).temp_path(OutputType::Object, Some(&name)); - let obj = product.object.write().unwrap(); - - tcx.sess.prof.artifact_size("object_file", &*name, obj.len().try_into().unwrap()); - - if let Err(err) = std::fs::write(&tmp_file, obj) { - tcx.sess.fatal(&format!("error writing object file: {}", err)); - } - - Some(CompiledModule { - name, - kind: ModuleKind::Allocator, - object: Some(tmp_file), - dwarf_object: None, - bytecode: None, - }) + Some(emit_module(tcx, product.object, ModuleKind::Allocator, "allocator_shim".to_owned())) } else { None }; From c2f0b3a1bf5ed437df3e276960f64bf3c47222e0 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Fri, 12 Aug 2022 19:10:16 +0000 Subject: [PATCH 15/17] Move copy to incr comp cache to codegen join phase The copy depends on Session, which is only available on the main thread. As such the copy can't be done on future codegen threads. --- src/driver/aot.rs | 58 +++++++++++++++++++++++++++-------------------- src/lib.rs | 30 ++++++++++++------------ 2 files changed, 50 insertions(+), 38 deletions(-) diff --git a/src/driver/aot.rs b/src/driver/aot.rs index ff6310bd8af..7e5e3453834 100644 --- a/src/driver/aot.rs +++ b/src/driver/aot.rs @@ -23,7 +23,7 @@ struct ModuleCodegenResult { module_regular: CompiledModule, module_global_asm: Option, - work_product: Option<(WorkProductId, WorkProduct)>, + existing_work_product: Option<(WorkProductId, WorkProduct)>, } impl HashStable for ModuleCodegenResult { @@ -41,16 +41,44 @@ pub(crate) struct OngoingCodegen { } impl OngoingCodegen { - pub(crate) fn join(self) -> (CodegenResults, FxHashMap) { + pub(crate) fn join( + self, + sess: &Session, + backend_config: &BackendConfig, + ) -> (CodegenResults, FxHashMap) { let mut work_products = FxHashMap::default(); let mut modules = vec![]; for module_codegen_result in self.modules { - let ModuleCodegenResult { module_regular, module_global_asm, work_product } = + let ModuleCodegenResult { module_regular, module_global_asm, existing_work_product } = module_codegen_result; - if let Some((work_product_id, work_product)) = work_product { + + if let Some((work_product_id, work_product)) = existing_work_product { work_products.insert(work_product_id, work_product); + } else { + let work_product = if backend_config.disable_incr_cache { + None + } else if let Some(module_global_asm) = &module_global_asm { + rustc_incremental::copy_cgu_workproduct_to_incr_comp_cache_dir( + sess, + &module_regular.name, + &[ + ("o", &module_regular.object.as_ref().unwrap()), + ("asm.o", &module_global_asm.object.as_ref().unwrap()), + ], + ) + } else { + rustc_incremental::copy_cgu_workproduct_to_incr_comp_cache_dir( + sess, + &module_regular.name, + &[("o", &module_regular.object.as_ref().unwrap())], + ) + }; + if let Some((work_product_id, work_product)) = work_product { + work_products.insert(work_product_id, work_product); + } } + modules.push(module_regular); if let Some(module_global_asm) = module_global_asm { modules.push(module_global_asm); @@ -84,7 +112,6 @@ fn make_module(sess: &Session, backend_config: &BackendConfig, name: String) -> fn emit_cgu( tcx: TyCtxt<'_>, - backend_config: &BackendConfig, name: String, module: ObjectModule, debug: Option>, @@ -101,22 +128,6 @@ fn emit_cgu( let module_regular = emit_module(tcx, product.object, ModuleKind::Regular, name.clone()); - let work_product = if backend_config.disable_incr_cache { - None - } else if let Some(global_asm_object_file) = &global_asm_object_file { - rustc_incremental::copy_cgu_workproduct_to_incr_comp_cache_dir( - tcx.sess, - &name, - &[("o", &module_regular.object.as_ref().unwrap()), ("asm.o", global_asm_object_file)], - ) - } else { - rustc_incremental::copy_cgu_workproduct_to_incr_comp_cache_dir( - tcx.sess, - &name, - &[("o", &module_regular.object.as_ref().unwrap())], - ) - }; - ModuleCodegenResult { module_regular, module_global_asm: global_asm_object_file.map(|global_asm_object_file| CompiledModule { @@ -126,7 +137,7 @@ fn emit_cgu( dwarf_object: None, bytecode: None, }), - work_product, + existing_work_product: None, } } @@ -205,7 +216,7 @@ fn reuse_workproduct_for_cgu(tcx: TyCtxt<'_>, cgu: &CodegenUnit<'_>) -> ModuleCo } else { None }, - work_product: Some((cgu.work_product_id(), work_product)), + existing_work_product: Some((cgu.work_product_id(), work_product)), } } @@ -271,7 +282,6 @@ fn module_codegen( let codegen_result = tcx.sess.time("write object file", || { emit_cgu( tcx, - &backend_config, cgu.name().as_str().to_string(), module, debug_context, diff --git a/src/lib.rs b/src/lib.rs index 6ea160d26ce..909f4f00f1e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -25,7 +25,7 @@ extern crate rustc_driver; use std::any::Any; -use std::cell::Cell; +use std::cell::{Cell, RefCell}; use rustc_codegen_ssa::traits::CodegenBackend; use rustc_codegen_ssa::CodegenResults; @@ -158,7 +158,7 @@ fn new( } pub struct CraneliftCodegenBackend { - pub config: Option, + pub config: RefCell>, } impl CodegenBackend for CraneliftCodegenBackend { @@ -168,6 +168,13 @@ fn init(&self, sess: &Session) { Lto::No | Lto::ThinLocal => {} Lto::Thin | Lto::Fat => sess.warn("LTO is not supported. You may get a linker error."), } + + let mut config = self.config.borrow_mut(); + if config.is_none() { + let new_config = BackendConfig::from_opts(&sess.opts.cg.llvm_args) + .unwrap_or_else(|err| sess.fatal(&err)); + *config = Some(new_config); + } } fn target_features(&self, _sess: &Session, _allow_unstable: bool) -> Vec { @@ -185,15 +192,7 @@ fn codegen_crate( need_metadata_module: bool, ) -> Box { tcx.sess.abort_if_errors(); - let config = if let Some(config) = self.config.clone() { - config - } else { - if !tcx.sess.unstable_options() && !tcx.sess.opts.cg.llvm_args.is_empty() { - tcx.sess.fatal("`-Z unstable-options` must be passed to allow configuring cg_clif"); - } - BackendConfig::from_opts(&tcx.sess.opts.cg.llvm_args) - .unwrap_or_else(|err| tcx.sess.fatal(&err)) - }; + let config = self.config.borrow().clone().unwrap(); match config.codegen_mode { CodegenMode::Aot => driver::aot::run_aot(tcx, config, metadata, need_metadata_module), CodegenMode::Jit | CodegenMode::JitLazy => { @@ -209,10 +208,13 @@ fn codegen_crate( fn join_codegen( &self, ongoing_codegen: Box, - _sess: &Session, + sess: &Session, _outputs: &OutputFilenames, ) -> Result<(CodegenResults, FxHashMap), ErrorGuaranteed> { - Ok(ongoing_codegen.downcast::().unwrap().join()) + Ok(ongoing_codegen + .downcast::() + .unwrap() + .join(sess, self.config.borrow().as_ref().unwrap())) } fn link( @@ -309,5 +311,5 @@ fn build_isa(sess: &Session, backend_config: &BackendConfig) -> Box Box { - Box::new(CraneliftCodegenBackend { config: None }) + Box::new(CraneliftCodegenBackend { config: RefCell::new(None) }) } From 4c0766ce6cae86b0f340683d90a5ed02cace81f7 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Sat, 13 Aug 2022 09:03:28 +0000 Subject: [PATCH 16/17] Move error reporting out of emit_cgu Error reporting requires a Session, which isn't available on background threads. --- src/driver/aot.rs | 48 +++++++++++++++++++++++++++-------------------- 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/src/driver/aot.rs b/src/driver/aot.rs index 7e5e3453834..817ce7f7e6d 100644 --- a/src/driver/aot.rs +++ b/src/driver/aot.rs @@ -33,7 +33,7 @@ fn hash_stable(&self, _: &mut HCX, _: &mut StableHasher) { } pub(crate) struct OngoingCodegen { - modules: Vec, + modules: Vec>, allocator_module: Option, metadata_module: Option, metadata: EncodedMetadata, @@ -50,6 +50,10 @@ pub(crate) fn join( let mut modules = vec![]; for module_codegen_result in self.modules { + let module_codegen_result = match module_codegen_result { + Ok(module_codegen_result) => module_codegen_result, + Err(err) => sess.fatal(&err), + }; let ModuleCodegenResult { module_regular, module_global_asm, existing_work_product } = module_codegen_result; @@ -117,7 +121,7 @@ fn emit_cgu( debug: Option>, unwind_context: UnwindContext, global_asm_object_file: Option, -) -> ModuleCodegenResult { +) -> Result { let mut product = module.finish(); if let Some(mut debug) = debug { @@ -126,9 +130,9 @@ fn emit_cgu( unwind_context.emit(&mut product); - let module_regular = emit_module(tcx, product.object, ModuleKind::Regular, name.clone()); + let module_regular = emit_module(tcx, product.object, ModuleKind::Regular, name.clone())?; - ModuleCodegenResult { + Ok(ModuleCodegenResult { module_regular, module_global_asm: global_asm_object_file.map(|global_asm_object_file| CompiledModule { name: format!("{name}.asm"), @@ -138,7 +142,7 @@ fn emit_cgu( bytecode: None, }), existing_work_product: None, - } + }) } fn emit_module( @@ -146,23 +150,26 @@ fn emit_module( object: cranelift_object::object::write::Object<'_>, kind: ModuleKind, name: String, -) -> CompiledModule { +) -> Result { let tmp_file = tcx.output_filenames(()).temp_path(OutputType::Object, Some(&name)); let mut file = match File::create(&tmp_file) { Ok(file) => file, - Err(err) => tcx.sess.fatal(&format!("error creating object file: {}", err)), + Err(err) => return Err(format!("error creating object file: {}", err)), }; if let Err(err) = object.write_stream(&mut file) { - tcx.sess.fatal(&format!("error writing object file: {}", err)); + return Err(format!("error writing object file: {}", err)); } tcx.sess.prof.artifact_size("object_file", &*name, file.metadata().unwrap().len()); - CompiledModule { name, kind, object: Some(tmp_file), dwarf_object: None, bytecode: None } + Ok(CompiledModule { name, kind, object: Some(tmp_file), dwarf_object: None, bytecode: None }) } -fn reuse_workproduct_for_cgu(tcx: TyCtxt<'_>, cgu: &CodegenUnit<'_>) -> ModuleCodegenResult { +fn reuse_workproduct_for_cgu( + tcx: TyCtxt<'_>, + cgu: &CodegenUnit<'_>, +) -> Result { let work_product = cgu.previous_work_product(tcx); let obj_out_regular = tcx.output_filenames(()).temp_path(OutputType::Object, Some(cgu.name().as_str())); @@ -172,7 +179,7 @@ fn reuse_workproduct_for_cgu(tcx: TyCtxt<'_>, cgu: &CodegenUnit<'_>) -> ModuleCo ); if let Err(err) = rustc_fs_util::link_or_copy(&source_file_regular, &obj_out_regular) { - tcx.sess.err(&format!( + return Err(format!( "unable to copy {} to {}: {}", source_file_regular.display(), obj_out_regular.display(), @@ -185,7 +192,7 @@ fn reuse_workproduct_for_cgu(tcx: TyCtxt<'_>, cgu: &CodegenUnit<'_>) -> ModuleCo let source_file_global_asm = rustc_incremental::in_incr_comp_dir_sess(&tcx.sess, asm_o); if let Err(err) = rustc_fs_util::link_or_copy(&source_file_global_asm, &obj_out_global_asm) { - tcx.sess.err(&format!( + return Err(format!( "unable to copy {} to {}: {}", source_file_regular.display(), obj_out_regular.display(), @@ -197,7 +204,7 @@ fn reuse_workproduct_for_cgu(tcx: TyCtxt<'_>, cgu: &CodegenUnit<'_>) -> ModuleCo false }; - ModuleCodegenResult { + Ok(ModuleCodegenResult { module_regular: CompiledModule { name: cgu.name().to_string(), kind: ModuleKind::Regular, @@ -217,7 +224,7 @@ fn reuse_workproduct_for_cgu(tcx: TyCtxt<'_>, cgu: &CodegenUnit<'_>) -> ModuleCo None }, existing_work_product: Some((cgu.work_product_id(), work_product)), - } + }) } fn module_codegen( @@ -227,7 +234,7 @@ fn module_codegen( Arc, rustc_span::Symbol, ), -) -> ModuleCodegenResult { +) -> Result { let cgu = tcx.codegen_unit(cgu_name); let mono_items = cgu.items_in_deterministic_order(tcx); @@ -279,7 +286,7 @@ fn module_codegen( let debug_context = cx.debug_context; let unwind_context = cx.unwind_context; - let codegen_result = tcx.sess.time("write object file", || { + tcx.sess.time("write object file", || { emit_cgu( tcx, cgu.name().as_str().to_string(), @@ -288,9 +295,7 @@ fn module_codegen( unwind_context, global_asm_object_file, ) - }); - - codegen_result + }) } pub(crate) fn run_aot( @@ -356,7 +361,10 @@ pub(crate) fn run_aot( let mut product = allocator_module.finish(); allocator_unwind_context.emit(&mut product); - Some(emit_module(tcx, product.object, ModuleKind::Allocator, "allocator_shim".to_owned())) + match emit_module(tcx, product.object, ModuleKind::Allocator, "allocator_shim".to_owned()) { + Ok(allocator_module) => Some(allocator_module), + Err(err) => tcx.sess.fatal(err), + } } else { None }; From 9461fd2cb061c7016207c22dc77f7ad906066279 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Sat, 13 Aug 2022 12:18:41 +0000 Subject: [PATCH 17/17] Remove TyCtxt parameter from emit_cgu TyCtxt isn't available on background threads. --- src/driver/aot.rs | 27 +++++++++++++++++++-------- src/global_asm.rs | 2 +- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/driver/aot.rs b/src/driver/aot.rs index 817ce7f7e6d..9d819e3995b 100644 --- a/src/driver/aot.rs +++ b/src/driver/aot.rs @@ -7,12 +7,13 @@ use rustc_codegen_ssa::back::metadata::create_compressed_metadata_file; use rustc_codegen_ssa::{CodegenResults, CompiledModule, CrateInfo, ModuleKind}; +use rustc_data_structures::profiling::SelfProfilerRef; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_metadata::EncodedMetadata; use rustc_middle::dep_graph::{WorkProduct, WorkProductId}; use rustc_middle::mir::mono::{CodegenUnit, MonoItem}; use rustc_session::cgu_reuse_tracker::CguReuse; -use rustc_session::config::{DebugInfo, OutputType}; +use rustc_session::config::{DebugInfo, OutputFilenames, OutputType}; use rustc_session::Session; use cranelift_object::{ObjectBuilder, ObjectModule}; @@ -115,7 +116,8 @@ fn make_module(sess: &Session, backend_config: &BackendConfig, name: String) -> } fn emit_cgu( - tcx: TyCtxt<'_>, + output_filenames: &OutputFilenames, + prof: &SelfProfilerRef, name: String, module: ObjectModule, debug: Option>, @@ -130,7 +132,8 @@ fn emit_cgu( unwind_context.emit(&mut product); - let module_regular = emit_module(tcx, product.object, ModuleKind::Regular, name.clone())?; + let module_regular = + emit_module(output_filenames, prof, product.object, ModuleKind::Regular, name.clone())?; Ok(ModuleCodegenResult { module_regular, @@ -146,12 +149,13 @@ fn emit_cgu( } fn emit_module( - tcx: TyCtxt<'_>, + output_filenames: &OutputFilenames, + prof: &SelfProfilerRef, object: cranelift_object::object::write::Object<'_>, kind: ModuleKind, name: String, ) -> Result { - let tmp_file = tcx.output_filenames(()).temp_path(OutputType::Object, Some(&name)); + let tmp_file = output_filenames.temp_path(OutputType::Object, Some(&name)); let mut file = match File::create(&tmp_file) { Ok(file) => file, Err(err) => return Err(format!("error creating object file: {}", err)), @@ -161,7 +165,7 @@ fn emit_module( return Err(format!("error writing object file: {}", err)); } - tcx.sess.prof.artifact_size("object_file", &*name, file.metadata().unwrap().len()); + prof.artifact_size("object_file", &*name, file.metadata().unwrap().len()); Ok(CompiledModule { name, kind, object: Some(tmp_file), dwarf_object: None, bytecode: None }) } @@ -288,7 +292,8 @@ fn module_codegen( let unwind_context = cx.unwind_context; tcx.sess.time("write object file", || { emit_cgu( - tcx, + &global_asm_config.output_filenames, + &tcx.sess.prof, cgu.name().as_str().to_string(), module, debug_context, @@ -361,7 +366,13 @@ pub(crate) fn run_aot( let mut product = allocator_module.finish(); allocator_unwind_context.emit(&mut product); - match emit_module(tcx, product.object, ModuleKind::Allocator, "allocator_shim".to_owned()) { + match emit_module( + tcx.output_filenames(()), + &tcx.sess.prof, + product.object, + ModuleKind::Allocator, + "allocator_shim".to_owned(), + ) { Ok(allocator_module) => Some(allocator_module), Err(err) => tcx.sess.fatal(err), } diff --git a/src/global_asm.rs b/src/global_asm.rs index 917a6fff727..dcbcaba30fe 100644 --- a/src/global_asm.rs +++ b/src/global_asm.rs @@ -36,7 +36,7 @@ pub(crate) fn codegen_global_asm_item(tcx: TyCtxt<'_>, global_asm: &mut String, pub(crate) struct GlobalAsmConfig { asm_enabled: bool, assembler: PathBuf, - output_filenames: Arc, + pub(crate) output_filenames: Arc, } impl GlobalAsmConfig {