diff --git a/build_system/build_sysroot.rs b/build_system/build_sysroot.rs index 7791df1d53f..196ff8fda75 100644 --- a/build_system/build_sysroot.rs +++ b/build_system/build_sysroot.rs @@ -276,7 +276,7 @@ fn build_clif_sysroot_for_triple( if channel == "release" { build_cmd.arg("--release"); } - build_cmd.arg("--features").arg("compiler-builtins-no-asm backtrace panic-unwind"); + build_cmd.arg("--features").arg("backtrace panic-unwind"); build_cmd.env("CARGO_PROFILE_RELEASE_DEBUG", "true"); build_cmd.env("__CARGO_DEFAULT_LIB_METADATA", "cg_clif"); if compiler.triple.contains("apple") { diff --git a/example/mini_core_hello_world.rs b/example/mini_core_hello_world.rs index efa4be7e15a..aab20f67248 100644 --- a/example/mini_core_hello_world.rs +++ b/example/mini_core_hello_world.rs @@ -4,6 +4,7 @@ never_type, linkage, extern_types, + naked_functions, thread_local, repr_simd, raw_ref_op @@ -340,6 +341,7 @@ struct ExternTypeWrapper { ))] unsafe { global_asm_test(); + naked_test(); } // Both statics have a reference that points to the same anonymous allocation. @@ -395,6 +397,14 @@ fn stack_val_align() { " } +#[cfg(all(not(jit), not(no_unstable_features), target_arch = "x86_64"))] +#[naked] +extern "C" fn naked_test() { + unsafe { + asm!("ret", options(noreturn)); + } +} + #[repr(C)] enum c_void { _1, diff --git a/src/base.rs b/src/base.rs index 8874efadec9..5846689643f 100644 --- a/src/base.rs +++ b/src/base.rs @@ -6,6 +6,7 @@ use cranelift_module::ModuleError; use rustc_ast::InlineAsmOptions; use rustc_index::IndexVec; +use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; use rustc_middle::ty::adjustment::PointerCoercion; use rustc_middle::ty::layout::FnAbiOf; use rustc_middle::ty::print::with_no_trimmed_paths; @@ -14,6 +15,7 @@ use crate::constant::ConstantCx; use crate::debuginfo::{FunctionDebugContext, TypeDebugContext}; +use crate::inline_asm::codegen_naked_asm; use crate::prelude::*; use crate::pretty_clif::CommentWriter; @@ -32,7 +34,7 @@ pub(crate) fn codegen_fn<'tcx>( cached_func: Function, module: &mut dyn Module, instance: Instance<'tcx>, -) -> CodegenedFunction { +) -> Option { debug_assert!(!instance.args.has_infer()); let symbol_name = tcx.symbol_name(instance).name.to_string(); @@ -48,6 +50,37 @@ pub(crate) fn codegen_fn<'tcx>( String::from_utf8_lossy(&buf).into_owned() }); + if tcx.codegen_fn_attrs(instance.def_id()).flags.contains(CodegenFnAttrFlags::NAKED) { + assert_eq!(mir.basic_blocks.len(), 1); + assert!(mir.basic_blocks[START_BLOCK].statements.is_empty()); + + match &mir.basic_blocks[START_BLOCK].terminator().kind { + TerminatorKind::InlineAsm { + template, + operands, + options, + line_spans: _, + targets: _, + unwind: _, + } => { + codegen_naked_asm( + tcx, + cx, + module, + instance, + mir.basic_blocks[START_BLOCK].terminator().source_info.span, + &symbol_name, + template, + operands, + *options, + ); + } + _ => unreachable!(), + } + + return None; + } + // Declare function let sig = get_function_sig(tcx, module.target_config().default_call_conv, instance); let func_id = module.declare_function(&symbol_name, Linkage::Local, &sig).unwrap(); @@ -128,7 +161,7 @@ pub(crate) fn codegen_fn<'tcx>( // Verify function verify_func(tcx, &clif_comments, &func); - CodegenedFunction { symbol_name, func_id, func, clif_comments, func_debug_cx } + Some(CodegenedFunction { symbol_name, func_id, func, clif_comments, func_debug_cx }) } pub(crate) fn compile_fn( diff --git a/src/driver/aot.rs b/src/driver/aot.rs index 2651e56cac4..fce4690f97d 100644 --- a/src/driver/aot.rs +++ b/src/driver/aot.rs @@ -482,15 +482,16 @@ fn module_codegen( for (mono_item, _) in mono_items { match mono_item { MonoItem::Fn(inst) => { - let codegened_function = crate::base::codegen_fn( + if let Some(codegened_function) = crate::base::codegen_fn( tcx, &mut cx, &mut type_dbg, Function::new(), &mut module, inst, - ); - codegened_functions.push(codegened_function); + ) { + codegened_functions.push(codegened_function); + } } MonoItem::Static(def_id) => { let data_id = crate::constant::codegen_static(tcx, &mut module, def_id); diff --git a/src/driver/jit.rs b/src/driver/jit.rs index 6f8be03065f..4b149131b61 100644 --- a/src/driver/jit.rs +++ b/src/driver/jit.rs @@ -232,16 +232,16 @@ pub(crate) fn codegen_and_compile_fn<'tcx>( crate::PrintOnPanic(|| format!("{:?} {}", instance, tcx.symbol_name(instance).name)); let cached_func = std::mem::replace(&mut cached_context.func, Function::new()); - let codegened_func = crate::base::codegen_fn( + if let Some(codegened_func) = crate::base::codegen_fn( tcx, cx, &mut TypeDebugContext::default(), cached_func, module, instance, - ); - - crate::base::compile_fn(cx, cached_context, module, codegened_func); + ) { + crate::base::compile_fn(cx, cached_context, module, codegened_func); + } }); } diff --git a/src/driver/mod.rs b/src/driver/mod.rs index 12e90b58410..fb0eed07c19 100644 --- a/src/driver/mod.rs +++ b/src/driver/mod.rs @@ -5,6 +5,7 @@ //! [`codegen_static`]: crate::constant::codegen_static use rustc_data_structures::profiling::SelfProfilerRef; +use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; use rustc_middle::mir::mono::{MonoItem, MonoItemData}; use crate::prelude::*; @@ -33,7 +34,20 @@ fn predefine_mono_items<'tcx>( data.visibility, is_compiler_builtins, ); - module.declare_function(name, linkage, &sig).unwrap(); + let is_naked = tcx + .codegen_fn_attrs(instance.def_id()) + .flags + .contains(CodegenFnAttrFlags::NAKED); + module + .declare_function( + name, + // Naked functions are defined in a separate object + // file from the codegen unit rustc expects them to + // be defined in. + if is_naked { Linkage::Import } else { linkage }, + &sig, + ) + .unwrap(); } MonoItem::Static(_) | MonoItem::GlobalAsm(_) => {} } diff --git a/src/inline_asm.rs b/src/inline_asm.rs index f2299e933bb..2de804f5e04 100644 --- a/src/inline_asm.rs +++ b/src/inline_asm.rs @@ -169,6 +169,7 @@ pub(crate) fn codegen_inline_asm_inner<'tcx>( stack_slots_input: Vec::new(), stack_slots_output: Vec::new(), stack_slot_size: Size::from_bytes(0), + is_naked: false, }; asm_gen.allocate_registers(); asm_gen.allocate_stack_slots(); @@ -209,6 +210,121 @@ pub(crate) fn codegen_inline_asm_inner<'tcx>( call_inline_asm(fx, &asm_name, asm_gen.stack_slot_size, inputs, outputs); } +pub(crate) fn codegen_naked_asm<'tcx>( + tcx: TyCtxt<'tcx>, + cx: &mut crate::CodegenCx, + module: &mut dyn Module, + instance: Instance<'tcx>, + span: Span, + symbol_name: &str, + template: &[InlineAsmTemplatePiece], + operands: &[InlineAsmOperand<'tcx>], + options: InlineAsmOptions, +) { + // FIXME add .eh_frame unwind info directives + + let operands = operands + .iter() + .map(|operand| match *operand { + InlineAsmOperand::In { .. } + | InlineAsmOperand::Out { .. } + | InlineAsmOperand::InOut { .. } => { + span_bug!(span, "invalid operand type for naked asm") + } + InlineAsmOperand::Const { ref value } => { + let cv = instance.instantiate_mir_and_normalize_erasing_regions( + tcx, + ty::ParamEnv::reveal_all(), + ty::EarlyBinder::bind(value.const_), + ); + let const_value = cv + .eval(tcx, ty::ParamEnv::reveal_all(), value.span) + .expect("erroneous constant missed by mono item collection"); + + let value = rustc_codegen_ssa::common::asm_const_to_str( + tcx, + span, + const_value, + RevealAllLayoutCx(tcx).layout_of(cv.ty()), + ); + CInlineAsmOperand::Const { value } + } + InlineAsmOperand::SymFn { ref value } => { + if cfg!(not(feature = "inline_asm_sym")) { + tcx.dcx() + .span_err(span, "asm! and global_asm! sym operands are not yet supported"); + } + + let const_ = instance.instantiate_mir_and_normalize_erasing_regions( + tcx, + ty::ParamEnv::reveal_all(), + ty::EarlyBinder::bind(value.const_), + ); + if let ty::FnDef(def_id, args) = *const_.ty().kind() { + let instance = ty::Instance::resolve_for_fn_ptr( + tcx, + ty::ParamEnv::reveal_all(), + def_id, + args, + ) + .unwrap(); + let symbol = tcx.symbol_name(instance); + + // Pass a wrapper rather than the function itself as the function itself may not + // be exported from the main codegen unit and may thus be unreachable from the + // object file created by an external assembler. + let inline_asm_index = cx.inline_asm_index.get(); + cx.inline_asm_index.set(inline_asm_index + 1); + let wrapper_name = format!( + "__inline_asm_{}_wrapper_n{}", + cx.cgu_name.as_str().replace('.', "__").replace('-', "_"), + inline_asm_index + ); + let sig = + get_function_sig(tcx, module.target_config().default_call_conv, instance); + create_wrapper_function( + module, + &mut cx.unwind_context, + sig, + &wrapper_name, + symbol.name, + ); + + CInlineAsmOperand::Symbol { symbol: wrapper_name } + } else { + span_bug!(span, "invalid type for asm sym (fn)"); + } + } + InlineAsmOperand::SymStatic { def_id } => { + assert!(tcx.is_static(def_id)); + let instance = Instance::mono(tcx, def_id); + CInlineAsmOperand::Symbol { symbol: tcx.symbol_name(instance).name.to_owned() } + } + InlineAsmOperand::Label { .. } => { + span_bug!(span, "asm! label operands are not yet supported"); + } + }) + .collect::>(); + + let asm_gen = InlineAssemblyGenerator { + tcx, + arch: tcx.sess.asm_arch.unwrap(), + enclosing_def_id: instance.def_id(), + template, + operands: &operands, + options, + registers: Vec::new(), + stack_slots_clobber: Vec::new(), + stack_slots_input: Vec::new(), + stack_slots_output: Vec::new(), + stack_slot_size: Size::from_bytes(0), + is_naked: true, + }; + + let generated_asm = asm_gen.generate_asm_wrapper(symbol_name); + cx.global_asm.push_str(&generated_asm); +} + struct InlineAssemblyGenerator<'a, 'tcx> { tcx: TyCtxt<'tcx>, arch: InlineAsmArch, @@ -221,10 +337,13 @@ struct InlineAssemblyGenerator<'a, 'tcx> { stack_slots_input: Vec>, stack_slots_output: Vec>, stack_slot_size: Size, + is_naked: bool, } impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> { fn allocate_registers(&mut self) { + assert!(!self.is_naked); + let sess = self.tcx.sess; let map = allocatable_registers( self.arch, @@ -348,6 +467,8 @@ fn allocate_registers(&mut self) { } fn allocate_stack_slots(&mut self) { + assert!(!self.is_naked); + let mut slot_size = Size::from_bytes(0); let mut slots_clobber = vec![None; self.operands.len()]; let mut slots_input = vec![None; self.operands.len()]; @@ -468,30 +589,32 @@ fn generate_asm_wrapper(&self, asm_name: &str) -> String { if is_x86 { generated_asm.push_str(".intel_syntax noprefix\n"); } - Self::prologue(&mut generated_asm, self.arch); + if !self.is_naked { + Self::prologue(&mut generated_asm, self.arch); - // Save clobbered registers - if !self.options.contains(InlineAsmOptions::NORETURN) { + // Save clobbered registers + if !self.options.contains(InlineAsmOptions::NORETURN) { + for (reg, slot) in self + .registers + .iter() + .zip(self.stack_slots_clobber.iter().copied()) + .filter_map(|(r, s)| r.zip(s)) + { + Self::save_register(&mut generated_asm, self.arch, reg, slot); + } + } + + // Write input registers for (reg, slot) in self .registers .iter() - .zip(self.stack_slots_clobber.iter().copied()) + .zip(self.stack_slots_input.iter().copied()) .filter_map(|(r, s)| r.zip(s)) { - Self::save_register(&mut generated_asm, self.arch, reg, slot); + Self::restore_register(&mut generated_asm, self.arch, reg, slot); } } - // Write input registers - for (reg, slot) in self - .registers - .iter() - .zip(self.stack_slots_input.iter().copied()) - .filter_map(|(r, s)| r.zip(s)) - { - Self::restore_register(&mut generated_asm, self.arch, reg, slot); - } - if is_x86 && self.options.contains(InlineAsmOptions::ATT_SYNTAX) { generated_asm.push_str(".att_syntax\n"); } @@ -553,30 +676,32 @@ fn generate_asm_wrapper(&self, asm_name: &str) -> String { generated_asm.push_str(".intel_syntax noprefix\n"); } - if !self.options.contains(InlineAsmOptions::NORETURN) { - // Read output registers - for (reg, slot) in self - .registers - .iter() - .zip(self.stack_slots_output.iter().copied()) - .filter_map(|(r, s)| r.zip(s)) - { - Self::save_register(&mut generated_asm, self.arch, reg, slot); - } + if !self.is_naked { + if !self.options.contains(InlineAsmOptions::NORETURN) { + // Read output registers + for (reg, slot) in self + .registers + .iter() + .zip(self.stack_slots_output.iter().copied()) + .filter_map(|(r, s)| r.zip(s)) + { + Self::save_register(&mut generated_asm, self.arch, reg, slot); + } - // Restore clobbered registers - for (reg, slot) in self - .registers - .iter() - .zip(self.stack_slots_clobber.iter().copied()) - .filter_map(|(r, s)| r.zip(s)) - { - Self::restore_register(&mut generated_asm, self.arch, reg, slot); - } + // Restore clobbered registers + for (reg, slot) in self + .registers + .iter() + .zip(self.stack_slots_clobber.iter().copied()) + .filter_map(|(r, s)| r.zip(s)) + { + Self::restore_register(&mut generated_asm, self.arch, reg, slot); + } - Self::epilogue(&mut generated_asm, self.arch); - } else { - Self::epilogue_noreturn(&mut generated_asm, self.arch); + Self::epilogue(&mut generated_asm, self.arch); + } else { + Self::epilogue_noreturn(&mut generated_asm, self.arch); + } } if is_x86 {