diff --git a/compiler/rustc_codegen_gcc/src/asm.rs b/compiler/rustc_codegen_gcc/src/asm.rs index 8a74c4c07e0..2af050f0c75 100644 --- a/compiler/rustc_codegen_gcc/src/asm.rs +++ b/compiler/rustc_codegen_gcc/src/asm.rs @@ -258,9 +258,14 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> { } InlineAsmOperandRef::SymFn { instance } => { + // TODO(@Amanieu): Additional mangling is needed on + // some targets to add a leading underscore (Mach-O) + // or byte count suffixes (x86 Windows). constants_len += self.tcx.symbol_name(instance).name.len(); } InlineAsmOperandRef::SymStatic { def_id } => { + // TODO(@Amanieu): Additional mangling is needed on + // some targets to add a leading underscore (Mach-O). constants_len += self.tcx.symbol_name(Instance::mono(self.tcx, def_id)).name.len(); } } @@ -412,13 +417,16 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> { } InlineAsmOperandRef::SymFn { instance } => { + // TODO(@Amanieu): Additional mangling is needed on + // some targets to add a leading underscore (Mach-O) + // or byte count suffixes (x86 Windows). let name = self.tcx.symbol_name(instance).name; template_str.push_str(name); } InlineAsmOperandRef::SymStatic { def_id } => { - // TODO(@Commeownist): This may not be sufficient for all kinds of statics. - // Some statics may need the `@plt` suffix, like thread-local vars. + // TODO(@Amanieu): Additional mangling is needed on + // some targets to add a leading underscore (Mach-O). let instance = Instance::mono(self.tcx, def_id); let name = self.tcx.symbol_name(instance).name; template_str.push_str(name); @@ -656,8 +664,8 @@ fn dummy_output_type<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, reg: InlineAsmRegCl } } -impl<'gcc, 'tcx> AsmMethods for CodegenCx<'gcc, 'tcx> { - fn codegen_global_asm(&self, template: &[InlineAsmTemplatePiece], operands: &[GlobalAsmOperandRef], options: InlineAsmOptions, _line_spans: &[Span]) { +impl<'gcc, 'tcx> AsmMethods<'tcx> for CodegenCx<'gcc, 'tcx> { + fn codegen_global_asm(&self, template: &[InlineAsmTemplatePiece], operands: &[GlobalAsmOperandRef<'tcx>], options: InlineAsmOptions, _line_spans: &[Span]) { let asm_arch = self.tcx.sess.asm_arch.unwrap(); // Default to Intel syntax on x86 @@ -690,6 +698,22 @@ impl<'gcc, 'tcx> AsmMethods for CodegenCx<'gcc, 'tcx> { // here unlike normal inline assembly. template_str.push_str(string); } + + GlobalAsmOperandRef::SymFn { instance } => { + // TODO(@Amanieu): Additional mangling is needed on + // some targets to add a leading underscore (Mach-O) + // or byte count suffixes (x86 Windows). + let name = self.tcx.symbol_name(instance).name; + template_str.push_str(name); + } + + GlobalAsmOperandRef::SymStatic { def_id } => { + // TODO(@Amanieu): Additional mangling is needed on + // some targets to add a leading underscore (Mach-O). + let instance = Instance::mono(self.tcx, def_id); + let name = self.tcx.symbol_name(instance).name; + template_str.push_str(name); + } } } } diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index 91d132eb343..dff32007918 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -312,11 +312,11 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { } } -impl AsmMethods for CodegenCx<'_, '_> { +impl<'tcx> AsmMethods<'tcx> for CodegenCx<'_, 'tcx> { fn codegen_global_asm( &self, template: &[InlineAsmTemplatePiece], - operands: &[GlobalAsmOperandRef], + operands: &[GlobalAsmOperandRef<'tcx>], options: InlineAsmOptions, _line_spans: &[Span], ) { @@ -342,6 +342,29 @@ impl AsmMethods for CodegenCx<'_, '_> { // here unlike normal inline assembly. template_str.push_str(string); } + GlobalAsmOperandRef::SymFn { instance } => { + let llval = self.get_fn(instance); + self.add_compiler_used_global(llval); + let symbol = llvm::build_string(|s| unsafe { + llvm::LLVMRustGetMangledName(llval, s); + }) + .expect("symbol is not valid UTF-8"); + template_str.push_str(&symbol); + } + GlobalAsmOperandRef::SymStatic { def_id } => { + let llval = self + .renamed_statics + .borrow() + .get(&def_id) + .copied() + .unwrap_or_else(|| self.get_static(def_id)); + self.add_compiler_used_global(llval); + let symbol = llvm::build_string(|s| unsafe { + llvm::LLVMRustGetMangledName(llval, s); + }) + .expect("symbol is not valid UTF-8"); + template_str.push_str(&symbol); + } } } } diff --git a/compiler/rustc_codegen_llvm/src/base.rs b/compiler/rustc_codegen_llvm/src/base.rs index dd3ada44389..c163c7d8692 100644 --- a/compiler/rustc_codegen_llvm/src/base.rs +++ b/compiler/rustc_codegen_llvm/src/base.rs @@ -99,15 +99,6 @@ pub fn compile_codegen_unit(tcx: TyCtxt<'_>, cgu_name: Symbol) -> (ModuleCodegen attributes::apply_to_llfn(entry, llvm::AttributePlace::Function, &attrs); } - // Run replace-all-uses-with for statics that need it - for &(old_g, new_g) in cx.statics_to_rauw().borrow().iter() { - unsafe { - let bitcast = llvm::LLVMConstPointerCast(new_g, cx.val_ty(old_g)); - llvm::LLVMReplaceAllUsesWith(old_g, bitcast); - llvm::LLVMDeleteGlobal(old_g); - } - } - // Finalize code coverage by injecting the coverage map. Note, the coverage map will // also be added to the `llvm.compiler.used` variable, created next. if cx.sess().instrument_coverage() { @@ -122,6 +113,16 @@ pub fn compile_codegen_unit(tcx: TyCtxt<'_>, cgu_name: Symbol) -> (ModuleCodegen cx.create_compiler_used_variable() } + // Run replace-all-uses-with for statics that need it. This must + // happen after the llvm.used variables are created. + for &(old_g, new_g) in cx.statics_to_rauw().borrow().iter() { + unsafe { + let bitcast = llvm::LLVMConstPointerCast(new_g, cx.val_ty(old_g)); + llvm::LLVMReplaceAllUsesWith(old_g, bitcast); + llvm::LLVMDeleteGlobal(old_g); + } + } + // Finalize debuginfo if cx.sess().opts.debuginfo != DebugInfo::None { cx.debuginfo_finalize(); diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs index cb4073528e1..4d3f3f318b8 100644 --- a/compiler/rustc_codegen_llvm/src/consts.rs +++ b/compiler/rustc_codegen_llvm/src/consts.rs @@ -412,6 +412,13 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> { llvm::LLVMRustSetLinkage(new_g, linkage); llvm::LLVMRustSetVisibility(new_g, visibility); + // The old global has had its name removed but is returned by + // get_static since it is in the instance cache. Provide an + // alternative lookup that points to the new global so that + // global_asm! can compute the correct mangled symbol name + // for the global. + self.renamed_statics.borrow_mut().insert(def_id, new_g); + // To avoid breaking any invariants, we leave around the old // global for the moment; we'll replace all references to it // with the new global later. (See base::codegen_backend.) diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 98cf873ebbd..d296ee3b42c 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -14,6 +14,7 @@ use rustc_codegen_ssa::traits::*; use rustc_data_structures::base_n; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::small_c_str::SmallCStr; +use rustc_hir::def_id::DefId; use rustc_middle::mir::mono::CodegenUnit; use rustc_middle::ty::layout::{ FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasParamEnv, LayoutError, LayoutOfHelpers, @@ -105,6 +106,12 @@ pub struct CodegenCx<'ll, 'tcx> { /// A counter that is used for generating local symbol names local_gen_sym_counter: Cell, + + /// `codegen_static` will sometimes create a second global variable with a + /// different type and clear the symbol name of the original global. + /// `global_asm!` needs to be able to find this new global so that it can + /// compute the correct mangled symbol name to insert into the asm. + pub renamed_statics: RefCell>, } pub struct TypeLowering<'ll> { @@ -436,6 +443,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { rust_try_fn: Cell::new(None), intrinsics: Default::default(), local_gen_sym_counter: Cell::new(0), + renamed_statics: Default::default(), } } diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 375b9927c86..7f533b0552a 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -2537,4 +2537,6 @@ extern "C" { remark_passes_len: usize, ); + #[allow(improper_ctypes)] + pub fn LLVMRustGetMangledName(V: &Value, out: &RustString); } diff --git a/compiler/rustc_codegen_ssa/src/mono_item.rs b/compiler/rustc_codegen_ssa/src/mono_item.rs index 5414c619dcb..5006a2157fc 100644 --- a/compiler/rustc_codegen_ssa/src/mono_item.rs +++ b/compiler/rustc_codegen_ssa/src/mono_item.rs @@ -4,7 +4,9 @@ use crate::traits::*; use rustc_hir as hir; use rustc_middle::mir::mono::MonoItem; use rustc_middle::mir::mono::{Linkage, Visibility}; +use rustc_middle::ty; use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf}; +use rustc_middle::ty::Instance; pub trait MonoItemExt<'a, 'tcx> { fn define>(&self, cx: &'a Bx::CodegenCx); @@ -56,7 +58,27 @@ impl<'a, 'tcx: 'a> MonoItemExt<'a, 'tcx> for MonoItem<'tcx> { ); GlobalAsmOperandRef::Const { string } } - _ => span_bug!(*op_sp, "invalid operand type for global_asm!"), + hir::InlineAsmOperand::SymFn { ref anon_const } => { + let ty = cx + .tcx() + .typeck_body(anon_const.body) + .node_type(anon_const.hir_id); + let instance = match ty.kind() { + &ty::FnDef(def_id, substs) => Instance::new(def_id, substs), + _ => span_bug!(*op_sp, "asm sym is not a function"), + }; + + GlobalAsmOperandRef::SymFn { instance } + } + hir::InlineAsmOperand::SymStatic { path: _, def_id } => { + GlobalAsmOperandRef::SymStatic { def_id } + } + hir::InlineAsmOperand::In { .. } + | hir::InlineAsmOperand::Out { .. } + | hir::InlineAsmOperand::InOut { .. } + | hir::InlineAsmOperand::SplitInOut { .. } => { + span_bug!(*op_sp, "invalid operand type for global_asm!") + } }) .collect(); diff --git a/compiler/rustc_codegen_ssa/src/traits/asm.rs b/compiler/rustc_codegen_ssa/src/traits/asm.rs index 11111a79744..c2ae74b18d8 100644 --- a/compiler/rustc_codegen_ssa/src/traits/asm.rs +++ b/compiler/rustc_codegen_ssa/src/traits/asm.rs @@ -36,8 +36,10 @@ pub enum InlineAsmOperandRef<'tcx, B: BackendTypes + ?Sized> { } #[derive(Debug)] -pub enum GlobalAsmOperandRef { +pub enum GlobalAsmOperandRef<'tcx> { Const { string: String }, + SymFn { instance: Instance<'tcx> }, + SymStatic { def_id: DefId }, } pub trait AsmBuilderMethods<'tcx>: BackendTypes { @@ -53,11 +55,11 @@ pub trait AsmBuilderMethods<'tcx>: BackendTypes { ); } -pub trait AsmMethods { +pub trait AsmMethods<'tcx> { fn codegen_global_asm( &self, template: &[InlineAsmTemplatePiece], - operands: &[GlobalAsmOperandRef], + operands: &[GlobalAsmOperandRef<'tcx>], options: InlineAsmOptions, line_spans: &[Span], ); diff --git a/compiler/rustc_codegen_ssa/src/traits/mod.rs b/compiler/rustc_codegen_ssa/src/traits/mod.rs index c529fbbf518..396768e0a42 100644 --- a/compiler/rustc_codegen_ssa/src/traits/mod.rs +++ b/compiler/rustc_codegen_ssa/src/traits/mod.rs @@ -60,7 +60,7 @@ pub trait CodegenMethods<'tcx>: + StaticMethods + CoverageInfoMethods<'tcx> + DebugInfoMethods<'tcx> - + AsmMethods + + AsmMethods<'tcx> + PreDefineMethods<'tcx> + HasParamEnv<'tcx> + HasTyCtxt<'tcx> @@ -76,7 +76,7 @@ impl<'tcx, T> CodegenMethods<'tcx> for T where + StaticMethods + CoverageInfoMethods<'tcx> + DebugInfoMethods<'tcx> - + AsmMethods + + AsmMethods<'tcx> + PreDefineMethods<'tcx> + HasParamEnv<'tcx> + HasTyCtxt<'tcx> diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp index 51739df067f..3ed4396d1e9 100644 --- a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp @@ -1835,3 +1835,9 @@ extern "C" void LLVMRustContextConfigureDiagnosticHandler( unwrap(C)->setDiagnosticHandler(std::make_unique( DiagnosticHandlerCallback, DiagnosticHandlerContext, RemarkAllPasses, Passes)); } + +extern "C" void LLVMRustGetMangledName(LLVMValueRef V, RustStringRef Str) { + RawRustStringOstream OS(Str); + GlobalValue *GV = unwrap(V); + Mangler().getNameWithPrefix(OS, GV, true); +} diff --git a/compiler/rustc_monomorphize/src/collector.rs b/compiler/rustc_monomorphize/src/collector.rs index 2a659a97db5..fdb14e45d98 100644 --- a/compiler/rustc_monomorphize/src/collector.rs +++ b/compiler/rustc_monomorphize/src/collector.rs @@ -435,7 +435,27 @@ fn collect_items_rec<'tcx>( // are supported. Therefore the value should not // depend on any other items. } - _ => span_bug!(*op_sp, "invalid operand type for global_asm!"), + hir::InlineAsmOperand::SymFn { anon_const } => { + let def_id = tcx.hir().body_owner_def_id(anon_const.body).to_def_id(); + if let Ok(val) = tcx.const_eval_poly(def_id) { + rustc_data_structures::stack::ensure_sufficient_stack(|| { + collect_const_value(tcx, val, &mut neighbors); + }); + } + } + hir::InlineAsmOperand::SymStatic { path: _, def_id } => { + let instance = Instance::mono(tcx, *def_id); + if should_codegen_locally(tcx, &instance) { + trace!("collecting static {:?}", def_id); + neighbors.push(dummy_spanned(MonoItem::Static(*def_id))); + } + } + hir::InlineAsmOperand::In { .. } + | hir::InlineAsmOperand::Out { .. } + | hir::InlineAsmOperand::InOut { .. } + | hir::InlineAsmOperand::SplitInOut { .. } => { + span_bug!(*op_sp, "invalid operand type for global_asm!") + } } } } else {