936 lines
36 KiB
Rust
936 lines
36 KiB
Rust
//! Codegen of `asm!` invocations.
|
|
|
|
use std::fmt::Write;
|
|
|
|
use cranelift_codegen::isa::CallConv;
|
|
use rustc_ast::ast::{InlineAsmOptions, InlineAsmTemplatePiece};
|
|
use rustc_hir::LangItem;
|
|
use rustc_span::sym;
|
|
use rustc_target::asm::*;
|
|
use target_lexicon::BinaryFormat;
|
|
|
|
use crate::prelude::*;
|
|
|
|
pub(crate) enum CInlineAsmOperand<'tcx> {
|
|
In {
|
|
reg: InlineAsmRegOrRegClass,
|
|
value: Value,
|
|
},
|
|
Out {
|
|
reg: InlineAsmRegOrRegClass,
|
|
late: bool,
|
|
place: Option<CPlace<'tcx>>,
|
|
},
|
|
InOut {
|
|
reg: InlineAsmRegOrRegClass,
|
|
_late: bool,
|
|
in_value: Value,
|
|
out_place: Option<CPlace<'tcx>>,
|
|
},
|
|
Const {
|
|
value: String,
|
|
},
|
|
Symbol {
|
|
symbol: String,
|
|
},
|
|
}
|
|
|
|
pub(crate) fn codegen_inline_asm_terminator<'tcx>(
|
|
fx: &mut FunctionCx<'_, '_, 'tcx>,
|
|
span: Span,
|
|
template: &[InlineAsmTemplatePiece],
|
|
operands: &[InlineAsmOperand<'tcx>],
|
|
options: InlineAsmOptions,
|
|
destination: Option<mir::BasicBlock>,
|
|
) {
|
|
// Used by panic_abort on Windows, but uses a syntax which only happens to work with
|
|
// asm!() by accident and breaks with the GNU assembler as well as global_asm!() for
|
|
// the LLVM backend.
|
|
if template.len() == 1
|
|
&& template[0] == InlineAsmTemplatePiece::String("int $$0x29".to_string())
|
|
{
|
|
fx.bcx.ins().trap(TrapCode::User(1));
|
|
return;
|
|
}
|
|
|
|
let operands = operands
|
|
.iter()
|
|
.map(|operand| match *operand {
|
|
InlineAsmOperand::In { reg, ref value } => CInlineAsmOperand::In {
|
|
reg,
|
|
value: crate::base::codegen_operand(fx, value).load_scalar(fx),
|
|
},
|
|
InlineAsmOperand::Out { reg, late, ref place } => CInlineAsmOperand::Out {
|
|
reg,
|
|
late,
|
|
place: place.map(|place| crate::base::codegen_place(fx, place)),
|
|
},
|
|
InlineAsmOperand::InOut { reg, late, ref in_value, ref out_place } => {
|
|
CInlineAsmOperand::InOut {
|
|
reg,
|
|
_late: late,
|
|
in_value: crate::base::codegen_operand(fx, in_value).load_scalar(fx),
|
|
out_place: out_place.map(|place| crate::base::codegen_place(fx, place)),
|
|
}
|
|
}
|
|
InlineAsmOperand::Const { ref value } => {
|
|
let (const_value, ty) = crate::constant::eval_mir_constant(fx, value);
|
|
let value = rustc_codegen_ssa::common::asm_const_to_str(
|
|
fx.tcx,
|
|
span,
|
|
const_value,
|
|
fx.layout_of(ty),
|
|
);
|
|
CInlineAsmOperand::Const { value }
|
|
}
|
|
InlineAsmOperand::SymFn { ref value } => {
|
|
if cfg!(not(feature = "inline_asm_sym")) {
|
|
fx.tcx
|
|
.dcx()
|
|
.span_err(span, "asm! and global_asm! sym operands are not yet supported");
|
|
}
|
|
|
|
let const_ = fx.monomorphize(value.const_);
|
|
if let ty::FnDef(def_id, args) = *const_.ty().kind() {
|
|
let instance = ty::Instance::resolve_for_fn_ptr(
|
|
fx.tcx,
|
|
ty::ParamEnv::reveal_all(),
|
|
def_id,
|
|
args,
|
|
)
|
|
.unwrap();
|
|
let symbol = fx.tcx.symbol_name(instance);
|
|
|
|
// Pass a wrapper rather than the function itself as the function itself may not
|
|
// be exported from the main codegen unit and may thus be unreachable from the
|
|
// object file created by an external assembler.
|
|
let inline_asm_index = fx.cx.inline_asm_index.get();
|
|
fx.cx.inline_asm_index.set(inline_asm_index + 1);
|
|
let wrapper_name = format!(
|
|
"__inline_asm_{}_wrapper_n{}",
|
|
fx.cx.cgu_name.as_str().replace('.', "__").replace('-', "_"),
|
|
inline_asm_index
|
|
);
|
|
let sig =
|
|
get_function_sig(fx.tcx, fx.target_config.default_call_conv, instance);
|
|
create_wrapper_function(fx.module, sig, &wrapper_name, symbol.name);
|
|
|
|
CInlineAsmOperand::Symbol { symbol: wrapper_name }
|
|
} else {
|
|
span_bug!(span, "invalid type for asm sym (fn)");
|
|
}
|
|
}
|
|
InlineAsmOperand::SymStatic { def_id } => {
|
|
assert!(fx.tcx.is_static(def_id));
|
|
let instance = Instance::mono(fx.tcx, def_id);
|
|
CInlineAsmOperand::Symbol { symbol: fx.tcx.symbol_name(instance).name.to_owned() }
|
|
}
|
|
InlineAsmOperand::Label { .. } => {
|
|
span_bug!(span, "asm! label operands are not yet supported");
|
|
}
|
|
})
|
|
.collect::<Vec<_>>();
|
|
|
|
codegen_inline_asm_inner(fx, template, &operands, options);
|
|
|
|
match destination {
|
|
Some(destination) => {
|
|
let destination_block = fx.get_block(destination);
|
|
fx.bcx.ins().jump(destination_block, &[]);
|
|
}
|
|
None => {
|
|
fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
|
|
}
|
|
}
|
|
}
|
|
|
|
pub(crate) fn codegen_inline_asm_inner<'tcx>(
|
|
fx: &mut FunctionCx<'_, '_, 'tcx>,
|
|
template: &[InlineAsmTemplatePiece],
|
|
operands: &[CInlineAsmOperand<'tcx>],
|
|
options: InlineAsmOptions,
|
|
) {
|
|
// FIXME add .eh_frame unwind info directives
|
|
|
|
let mut asm_gen = InlineAssemblyGenerator {
|
|
tcx: fx.tcx,
|
|
arch: fx.tcx.sess.asm_arch.unwrap(),
|
|
enclosing_def_id: fx.instance.def_id(),
|
|
template,
|
|
operands,
|
|
options,
|
|
registers: Vec::new(),
|
|
stack_slots_clobber: Vec::new(),
|
|
stack_slots_input: Vec::new(),
|
|
stack_slots_output: Vec::new(),
|
|
stack_slot_size: Size::from_bytes(0),
|
|
is_naked: false,
|
|
};
|
|
asm_gen.allocate_registers();
|
|
asm_gen.allocate_stack_slots();
|
|
|
|
let inline_asm_index = fx.cx.inline_asm_index.get();
|
|
fx.cx.inline_asm_index.set(inline_asm_index + 1);
|
|
let asm_name = format!(
|
|
"__inline_asm_{}_n{}",
|
|
fx.cx.cgu_name.as_str().replace('.', "__").replace('-', "_"),
|
|
inline_asm_index
|
|
);
|
|
|
|
let generated_asm = asm_gen.generate_asm_wrapper(&asm_name);
|
|
fx.cx.global_asm.push_str(&generated_asm);
|
|
|
|
let mut inputs = Vec::new();
|
|
let mut outputs = Vec::new();
|
|
for (i, operand) in operands.iter().enumerate() {
|
|
match operand {
|
|
CInlineAsmOperand::In { reg: _, value } => {
|
|
inputs.push((asm_gen.stack_slots_input[i].unwrap(), *value));
|
|
}
|
|
CInlineAsmOperand::Out { reg: _, late: _, place } => {
|
|
if let Some(place) = place {
|
|
outputs.push((asm_gen.stack_slots_output[i].unwrap(), *place));
|
|
}
|
|
}
|
|
CInlineAsmOperand::InOut { reg: _, _late: _, in_value, out_place } => {
|
|
inputs.push((asm_gen.stack_slots_input[i].unwrap(), *in_value));
|
|
if let Some(out_place) = out_place {
|
|
outputs.push((asm_gen.stack_slots_output[i].unwrap(), *out_place));
|
|
}
|
|
}
|
|
CInlineAsmOperand::Const { value: _ } | CInlineAsmOperand::Symbol { symbol: _ } => {}
|
|
}
|
|
}
|
|
|
|
call_inline_asm(fx, &asm_name, asm_gen.stack_slot_size, inputs, outputs);
|
|
}
|
|
|
|
pub(crate) fn codegen_naked_asm<'tcx>(
|
|
tcx: TyCtxt<'tcx>,
|
|
cx: &mut crate::CodegenCx,
|
|
module: &mut dyn Module,
|
|
instance: Instance<'tcx>,
|
|
span: Span,
|
|
symbol_name: &str,
|
|
template: &[InlineAsmTemplatePiece],
|
|
operands: &[InlineAsmOperand<'tcx>],
|
|
options: InlineAsmOptions,
|
|
) {
|
|
// FIXME add .eh_frame unwind info directives
|
|
|
|
let operands = operands
|
|
.iter()
|
|
.map(|operand| match *operand {
|
|
InlineAsmOperand::In { .. }
|
|
| InlineAsmOperand::Out { .. }
|
|
| InlineAsmOperand::InOut { .. } => {
|
|
span_bug!(span, "invalid operand type for naked asm")
|
|
}
|
|
InlineAsmOperand::Const { ref value } => {
|
|
let cv = instance.instantiate_mir_and_normalize_erasing_regions(
|
|
tcx,
|
|
ty::ParamEnv::reveal_all(),
|
|
ty::EarlyBinder::bind(value.const_),
|
|
);
|
|
let const_value = cv
|
|
.eval(tcx, ty::ParamEnv::reveal_all(), value.span)
|
|
.expect("erroneous constant missed by mono item collection");
|
|
|
|
let value = rustc_codegen_ssa::common::asm_const_to_str(
|
|
tcx,
|
|
span,
|
|
const_value,
|
|
RevealAllLayoutCx(tcx).layout_of(cv.ty()),
|
|
);
|
|
CInlineAsmOperand::Const { value }
|
|
}
|
|
InlineAsmOperand::SymFn { ref value } => {
|
|
if cfg!(not(feature = "inline_asm_sym")) {
|
|
tcx.dcx()
|
|
.span_err(span, "asm! and global_asm! sym operands are not yet supported");
|
|
}
|
|
|
|
let const_ = instance.instantiate_mir_and_normalize_erasing_regions(
|
|
tcx,
|
|
ty::ParamEnv::reveal_all(),
|
|
ty::EarlyBinder::bind(value.const_),
|
|
);
|
|
if let ty::FnDef(def_id, args) = *const_.ty().kind() {
|
|
let instance = ty::Instance::resolve_for_fn_ptr(
|
|
tcx,
|
|
ty::ParamEnv::reveal_all(),
|
|
def_id,
|
|
args,
|
|
)
|
|
.unwrap();
|
|
let symbol = tcx.symbol_name(instance);
|
|
|
|
// Pass a wrapper rather than the function itself as the function itself may not
|
|
// be exported from the main codegen unit and may thus be unreachable from the
|
|
// object file created by an external assembler.
|
|
let inline_asm_index = cx.inline_asm_index.get();
|
|
cx.inline_asm_index.set(inline_asm_index + 1);
|
|
let wrapper_name = format!(
|
|
"__inline_asm_{}_wrapper_n{}",
|
|
cx.cgu_name.as_str().replace('.', "__").replace('-', "_"),
|
|
inline_asm_index
|
|
);
|
|
let sig =
|
|
get_function_sig(tcx, module.target_config().default_call_conv, instance);
|
|
create_wrapper_function(module, sig, &wrapper_name, symbol.name);
|
|
|
|
CInlineAsmOperand::Symbol { symbol: wrapper_name }
|
|
} else {
|
|
span_bug!(span, "invalid type for asm sym (fn)");
|
|
}
|
|
}
|
|
InlineAsmOperand::SymStatic { def_id } => {
|
|
assert!(tcx.is_static(def_id));
|
|
let instance = Instance::mono(tcx, def_id);
|
|
CInlineAsmOperand::Symbol { symbol: tcx.symbol_name(instance).name.to_owned() }
|
|
}
|
|
InlineAsmOperand::Label { .. } => {
|
|
span_bug!(span, "asm! label operands are not yet supported");
|
|
}
|
|
})
|
|
.collect::<Vec<_>>();
|
|
|
|
let asm_gen = InlineAssemblyGenerator {
|
|
tcx,
|
|
arch: tcx.sess.asm_arch.unwrap(),
|
|
enclosing_def_id: instance.def_id(),
|
|
template,
|
|
operands: &operands,
|
|
options,
|
|
registers: Vec::new(),
|
|
stack_slots_clobber: Vec::new(),
|
|
stack_slots_input: Vec::new(),
|
|
stack_slots_output: Vec::new(),
|
|
stack_slot_size: Size::from_bytes(0),
|
|
is_naked: true,
|
|
};
|
|
|
|
let generated_asm = asm_gen.generate_asm_wrapper(symbol_name);
|
|
cx.global_asm.push_str(&generated_asm);
|
|
}
|
|
|
|
struct InlineAssemblyGenerator<'a, 'tcx> {
|
|
tcx: TyCtxt<'tcx>,
|
|
arch: InlineAsmArch,
|
|
enclosing_def_id: DefId,
|
|
template: &'a [InlineAsmTemplatePiece],
|
|
operands: &'a [CInlineAsmOperand<'tcx>],
|
|
options: InlineAsmOptions,
|
|
registers: Vec<Option<InlineAsmReg>>,
|
|
stack_slots_clobber: Vec<Option<Size>>,
|
|
stack_slots_input: Vec<Option<Size>>,
|
|
stack_slots_output: Vec<Option<Size>>,
|
|
stack_slot_size: Size,
|
|
is_naked: bool,
|
|
}
|
|
|
|
impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> {
|
|
fn allocate_registers(&mut self) {
|
|
assert!(!self.is_naked);
|
|
|
|
let sess = self.tcx.sess;
|
|
let map = allocatable_registers(
|
|
self.arch,
|
|
sess.relocation_model(),
|
|
self.tcx.asm_target_features(self.enclosing_def_id),
|
|
&sess.target,
|
|
);
|
|
let mut allocated = FxHashMap::<_, (bool, bool)>::default();
|
|
let mut regs = vec![None; self.operands.len()];
|
|
|
|
// Add explicit registers to the allocated set.
|
|
for (i, operand) in self.operands.iter().enumerate() {
|
|
match *operand {
|
|
CInlineAsmOperand::In { reg: InlineAsmRegOrRegClass::Reg(reg), .. } => {
|
|
regs[i] = Some(reg);
|
|
allocated.entry(reg).or_default().0 = true;
|
|
}
|
|
CInlineAsmOperand::Out {
|
|
reg: InlineAsmRegOrRegClass::Reg(reg),
|
|
late: true,
|
|
..
|
|
} => {
|
|
regs[i] = Some(reg);
|
|
allocated.entry(reg).or_default().1 = true;
|
|
}
|
|
CInlineAsmOperand::Out { reg: InlineAsmRegOrRegClass::Reg(reg), .. }
|
|
| CInlineAsmOperand::InOut { reg: InlineAsmRegOrRegClass::Reg(reg), .. } => {
|
|
regs[i] = Some(reg);
|
|
allocated.insert(reg, (true, true));
|
|
}
|
|
_ => (),
|
|
}
|
|
}
|
|
|
|
// Allocate out/inout/inlateout registers first because they are more constrained.
|
|
for (i, operand) in self.operands.iter().enumerate() {
|
|
match *operand {
|
|
CInlineAsmOperand::Out {
|
|
reg: InlineAsmRegOrRegClass::RegClass(class),
|
|
late: false,
|
|
..
|
|
}
|
|
| CInlineAsmOperand::InOut {
|
|
reg: InlineAsmRegOrRegClass::RegClass(class), ..
|
|
} => {
|
|
let mut alloc_reg = None;
|
|
for ® in &map[&class] {
|
|
let mut used = false;
|
|
reg.overlapping_regs(|r| {
|
|
if allocated.contains_key(&r) {
|
|
used = true;
|
|
}
|
|
});
|
|
|
|
if !used {
|
|
alloc_reg = Some(reg);
|
|
break;
|
|
}
|
|
}
|
|
|
|
let reg = alloc_reg.expect("cannot allocate registers");
|
|
regs[i] = Some(reg);
|
|
allocated.insert(reg, (true, true));
|
|
}
|
|
_ => (),
|
|
}
|
|
}
|
|
|
|
// Allocate in/lateout.
|
|
for (i, operand) in self.operands.iter().enumerate() {
|
|
match *operand {
|
|
CInlineAsmOperand::In { reg: InlineAsmRegOrRegClass::RegClass(class), .. } => {
|
|
let mut alloc_reg = None;
|
|
for ® in &map[&class] {
|
|
let mut used = false;
|
|
reg.overlapping_regs(|r| {
|
|
if allocated.get(&r).copied().unwrap_or_default().0 {
|
|
used = true;
|
|
}
|
|
});
|
|
|
|
if !used {
|
|
alloc_reg = Some(reg);
|
|
break;
|
|
}
|
|
}
|
|
|
|
let reg = alloc_reg.expect("cannot allocate registers");
|
|
regs[i] = Some(reg);
|
|
allocated.entry(reg).or_default().0 = true;
|
|
}
|
|
CInlineAsmOperand::Out {
|
|
reg: InlineAsmRegOrRegClass::RegClass(class),
|
|
late: true,
|
|
..
|
|
} => {
|
|
let mut alloc_reg = None;
|
|
for ® in &map[&class] {
|
|
let mut used = false;
|
|
reg.overlapping_regs(|r| {
|
|
if allocated.get(&r).copied().unwrap_or_default().1 {
|
|
used = true;
|
|
}
|
|
});
|
|
|
|
if !used {
|
|
alloc_reg = Some(reg);
|
|
break;
|
|
}
|
|
}
|
|
|
|
let reg = alloc_reg.expect("cannot allocate registers");
|
|
regs[i] = Some(reg);
|
|
allocated.entry(reg).or_default().1 = true;
|
|
}
|
|
_ => (),
|
|
}
|
|
}
|
|
|
|
self.registers = regs;
|
|
}
|
|
|
|
fn allocate_stack_slots(&mut self) {
|
|
assert!(!self.is_naked);
|
|
|
|
let mut slot_size = Size::from_bytes(0);
|
|
let mut slots_clobber = vec![None; self.operands.len()];
|
|
let mut slots_input = vec![None; self.operands.len()];
|
|
let mut slots_output = vec![None; self.operands.len()];
|
|
|
|
let new_slot_fn = |slot_size: &mut Size, reg_class: InlineAsmRegClass| {
|
|
let reg_size =
|
|
reg_class.supported_types(self.arch).iter().map(|(ty, _)| ty.size()).max().unwrap();
|
|
let align = rustc_target::abi::Align::from_bytes(reg_size.bytes()).unwrap();
|
|
let offset = slot_size.align_to(align);
|
|
*slot_size = offset + reg_size;
|
|
offset
|
|
};
|
|
let mut new_slot = |x| new_slot_fn(&mut slot_size, x);
|
|
|
|
// Allocate stack slots for saving clobbered registers
|
|
let abi_clobber = InlineAsmClobberAbi::parse(self.arch, &self.tcx.sess.target, sym::C)
|
|
.unwrap()
|
|
.clobbered_regs();
|
|
for (i, reg) in self.registers.iter().enumerate().filter_map(|(i, r)| r.map(|r| (i, r))) {
|
|
let mut need_save = true;
|
|
// If the register overlaps with a register clobbered by function call, then
|
|
// we don't need to save it.
|
|
for r in abi_clobber {
|
|
r.overlapping_regs(|r| {
|
|
if r == reg {
|
|
need_save = false;
|
|
}
|
|
});
|
|
|
|
if !need_save {
|
|
break;
|
|
}
|
|
}
|
|
|
|
if need_save {
|
|
slots_clobber[i] = Some(new_slot(reg.reg_class()));
|
|
}
|
|
}
|
|
|
|
// Allocate stack slots for inout
|
|
for (i, operand) in self.operands.iter().enumerate() {
|
|
match *operand {
|
|
CInlineAsmOperand::InOut { reg, out_place: Some(_), .. } => {
|
|
let slot = new_slot(reg.reg_class());
|
|
slots_input[i] = Some(slot);
|
|
slots_output[i] = Some(slot);
|
|
}
|
|
_ => (),
|
|
}
|
|
}
|
|
|
|
let slot_size_before_input = slot_size;
|
|
let mut new_slot = |x| new_slot_fn(&mut slot_size, x);
|
|
|
|
// Allocate stack slots for input
|
|
for (i, operand) in self.operands.iter().enumerate() {
|
|
match *operand {
|
|
CInlineAsmOperand::In { reg, .. }
|
|
| CInlineAsmOperand::InOut { reg, out_place: None, .. } => {
|
|
slots_input[i] = Some(new_slot(reg.reg_class()));
|
|
}
|
|
_ => (),
|
|
}
|
|
}
|
|
|
|
// Reset slot size to before input so that input and output operands can overlap
|
|
// and save some memory.
|
|
let slot_size_after_input = slot_size;
|
|
slot_size = slot_size_before_input;
|
|
let mut new_slot = |x| new_slot_fn(&mut slot_size, x);
|
|
|
|
// Allocate stack slots for output
|
|
for (i, operand) in self.operands.iter().enumerate() {
|
|
match *operand {
|
|
CInlineAsmOperand::Out { reg, place: Some(_), .. } => {
|
|
slots_output[i] = Some(new_slot(reg.reg_class()));
|
|
}
|
|
_ => (),
|
|
}
|
|
}
|
|
|
|
slot_size = slot_size.max(slot_size_after_input);
|
|
|
|
self.stack_slots_clobber = slots_clobber;
|
|
self.stack_slots_input = slots_input;
|
|
self.stack_slots_output = slots_output;
|
|
self.stack_slot_size = slot_size;
|
|
}
|
|
|
|
fn generate_asm_wrapper(&self, asm_name: &str) -> String {
|
|
let binary_format = crate::target_triple(self.tcx.sess).binary_format;
|
|
|
|
let mut generated_asm = String::new();
|
|
match binary_format {
|
|
BinaryFormat::Elf => {
|
|
writeln!(generated_asm, ".globl {}", asm_name).unwrap();
|
|
writeln!(generated_asm, ".type {},@function", asm_name).unwrap();
|
|
writeln!(generated_asm, ".section .text.{},\"ax\",@progbits", asm_name).unwrap();
|
|
writeln!(generated_asm, "{}:", asm_name).unwrap();
|
|
}
|
|
BinaryFormat::Macho => {
|
|
writeln!(generated_asm, ".globl _{}", asm_name).unwrap();
|
|
writeln!(generated_asm, "_{}:", asm_name).unwrap();
|
|
}
|
|
BinaryFormat::Coff => {
|
|
writeln!(generated_asm, ".globl {}", asm_name).unwrap();
|
|
writeln!(generated_asm, "{}:", asm_name).unwrap();
|
|
}
|
|
_ => self
|
|
.tcx
|
|
.dcx()
|
|
.fatal(format!("Unsupported binary format for inline asm: {binary_format:?}")),
|
|
}
|
|
|
|
let is_x86 = matches!(self.arch, InlineAsmArch::X86 | InlineAsmArch::X86_64);
|
|
|
|
if is_x86 {
|
|
generated_asm.push_str(".intel_syntax noprefix\n");
|
|
}
|
|
if !self.is_naked {
|
|
Self::prologue(&mut generated_asm, self.arch);
|
|
|
|
// Save clobbered registers
|
|
if !self.options.contains(InlineAsmOptions::NORETURN) {
|
|
for (reg, slot) in self
|
|
.registers
|
|
.iter()
|
|
.zip(self.stack_slots_clobber.iter().copied())
|
|
.filter_map(|(r, s)| r.zip(s))
|
|
{
|
|
Self::save_register(&mut generated_asm, self.arch, reg, slot);
|
|
}
|
|
}
|
|
|
|
// Write input registers
|
|
for (reg, slot) in self
|
|
.registers
|
|
.iter()
|
|
.zip(self.stack_slots_input.iter().copied())
|
|
.filter_map(|(r, s)| r.zip(s))
|
|
{
|
|
Self::restore_register(&mut generated_asm, self.arch, reg, slot);
|
|
}
|
|
}
|
|
|
|
if is_x86 && self.options.contains(InlineAsmOptions::ATT_SYNTAX) {
|
|
generated_asm.push_str(".att_syntax\n");
|
|
}
|
|
|
|
// The actual inline asm
|
|
for piece in self.template {
|
|
match piece {
|
|
InlineAsmTemplatePiece::String(s) => {
|
|
generated_asm.push_str(s);
|
|
}
|
|
InlineAsmTemplatePiece::Placeholder { operand_idx, modifier, span: _ } => {
|
|
match self.operands[*operand_idx] {
|
|
CInlineAsmOperand::In { .. }
|
|
| CInlineAsmOperand::Out { .. }
|
|
| CInlineAsmOperand::InOut { .. } => {
|
|
if self.options.contains(InlineAsmOptions::ATT_SYNTAX) {
|
|
generated_asm.push('%');
|
|
}
|
|
|
|
let reg = self.registers[*operand_idx].unwrap();
|
|
match self.arch {
|
|
InlineAsmArch::X86_64 => match reg {
|
|
InlineAsmReg::X86(reg)
|
|
if reg as u32 >= X86InlineAsmReg::xmm0 as u32
|
|
&& reg as u32 <= X86InlineAsmReg::xmm15 as u32 =>
|
|
{
|
|
// rustc emits x0 rather than xmm0
|
|
let class = match *modifier {
|
|
None | Some('x') => "xmm",
|
|
Some('y') => "ymm",
|
|
Some('z') => "zmm",
|
|
_ => unreachable!(),
|
|
};
|
|
write!(
|
|
generated_asm,
|
|
"{class}{}",
|
|
reg as u32 - X86InlineAsmReg::xmm0 as u32
|
|
)
|
|
.unwrap();
|
|
}
|
|
_ => reg
|
|
.emit(&mut generated_asm, InlineAsmArch::X86_64, *modifier)
|
|
.unwrap(),
|
|
},
|
|
_ => reg.emit(&mut generated_asm, self.arch, *modifier).unwrap(),
|
|
}
|
|
}
|
|
CInlineAsmOperand::Const { ref value } => {
|
|
generated_asm.push_str(value);
|
|
}
|
|
CInlineAsmOperand::Symbol { ref symbol } => generated_asm.push_str(symbol),
|
|
}
|
|
}
|
|
}
|
|
}
|
|
generated_asm.push('\n');
|
|
|
|
if is_x86 && self.options.contains(InlineAsmOptions::ATT_SYNTAX) {
|
|
generated_asm.push_str(".intel_syntax noprefix\n");
|
|
}
|
|
|
|
if !self.is_naked {
|
|
if !self.options.contains(InlineAsmOptions::NORETURN) {
|
|
// Read output registers
|
|
for (reg, slot) in self
|
|
.registers
|
|
.iter()
|
|
.zip(self.stack_slots_output.iter().copied())
|
|
.filter_map(|(r, s)| r.zip(s))
|
|
{
|
|
Self::save_register(&mut generated_asm, self.arch, reg, slot);
|
|
}
|
|
|
|
// Restore clobbered registers
|
|
for (reg, slot) in self
|
|
.registers
|
|
.iter()
|
|
.zip(self.stack_slots_clobber.iter().copied())
|
|
.filter_map(|(r, s)| r.zip(s))
|
|
{
|
|
Self::restore_register(&mut generated_asm, self.arch, reg, slot);
|
|
}
|
|
|
|
Self::epilogue(&mut generated_asm, self.arch);
|
|
} else {
|
|
Self::epilogue_noreturn(&mut generated_asm, self.arch);
|
|
}
|
|
}
|
|
|
|
if is_x86 {
|
|
generated_asm.push_str(".att_syntax\n");
|
|
}
|
|
|
|
match binary_format {
|
|
BinaryFormat::Elf => {
|
|
writeln!(generated_asm, ".size {name}, .-{name}", name = asm_name).unwrap();
|
|
generated_asm.push_str(".text\n");
|
|
}
|
|
BinaryFormat::Macho | BinaryFormat::Coff => {}
|
|
_ => self
|
|
.tcx
|
|
.dcx()
|
|
.fatal(format!("Unsupported binary format for inline asm: {binary_format:?}")),
|
|
}
|
|
|
|
generated_asm.push_str("\n\n");
|
|
|
|
generated_asm
|
|
}
|
|
|
|
fn prologue(generated_asm: &mut String, arch: InlineAsmArch) {
|
|
match arch {
|
|
InlineAsmArch::X86_64 => {
|
|
generated_asm.push_str(" push rbp\n");
|
|
generated_asm.push_str(" mov rbp,rsp\n");
|
|
generated_asm.push_str(" push rbx\n"); // rbx is callee saved
|
|
// rbx is reserved by LLVM for the "base pointer", so rustc doesn't allow using it
|
|
generated_asm.push_str(" mov rbx,rdi\n");
|
|
}
|
|
InlineAsmArch::AArch64 => {
|
|
generated_asm.push_str(" stp fp, lr, [sp, #-32]!\n");
|
|
generated_asm.push_str(" mov fp, sp\n");
|
|
generated_asm.push_str(" str x19, [sp, #24]\n"); // x19 is callee saved
|
|
// x19 is reserved by LLVM for the "base pointer", so rustc doesn't allow using it
|
|
generated_asm.push_str(" mov x19, x0\n");
|
|
}
|
|
InlineAsmArch::RiscV64 => {
|
|
generated_asm.push_str(" addi sp, sp, -16\n");
|
|
generated_asm.push_str(" sd ra, 8(sp)\n");
|
|
generated_asm.push_str(" sd s1, 0(sp)\n"); // s1 is callee saved
|
|
// s1/x9 is reserved by LLVM for the "base pointer", so rustc doesn't allow using it
|
|
generated_asm.push_str(" mv s1, a0\n");
|
|
}
|
|
_ => unimplemented!("prologue for {:?}", arch),
|
|
}
|
|
}
|
|
|
|
fn epilogue(generated_asm: &mut String, arch: InlineAsmArch) {
|
|
match arch {
|
|
InlineAsmArch::X86_64 => {
|
|
generated_asm.push_str(" pop rbx\n");
|
|
generated_asm.push_str(" pop rbp\n");
|
|
generated_asm.push_str(" ret\n");
|
|
}
|
|
InlineAsmArch::AArch64 => {
|
|
generated_asm.push_str(" ldr x19, [sp, #24]\n");
|
|
generated_asm.push_str(" ldp fp, lr, [sp], #32\n");
|
|
generated_asm.push_str(" ret\n");
|
|
}
|
|
InlineAsmArch::RiscV64 => {
|
|
generated_asm.push_str(" ld s1, 0(sp)\n");
|
|
generated_asm.push_str(" ld ra, 8(sp)\n");
|
|
generated_asm.push_str(" addi sp, sp, 16\n");
|
|
generated_asm.push_str(" ret\n");
|
|
}
|
|
_ => unimplemented!("epilogue for {:?}", arch),
|
|
}
|
|
}
|
|
|
|
fn epilogue_noreturn(generated_asm: &mut String, arch: InlineAsmArch) {
|
|
match arch {
|
|
InlineAsmArch::X86_64 => {
|
|
generated_asm.push_str(" ud2\n");
|
|
}
|
|
InlineAsmArch::AArch64 => {
|
|
generated_asm.push_str(" brk #0x1\n");
|
|
}
|
|
InlineAsmArch::RiscV64 => {
|
|
generated_asm.push_str(" ebreak\n");
|
|
}
|
|
_ => unimplemented!("epilogue_noreturn for {:?}", arch),
|
|
}
|
|
}
|
|
|
|
fn save_register(
|
|
generated_asm: &mut String,
|
|
arch: InlineAsmArch,
|
|
reg: InlineAsmReg,
|
|
offset: Size,
|
|
) {
|
|
match arch {
|
|
InlineAsmArch::X86_64 => {
|
|
match reg {
|
|
InlineAsmReg::X86(reg)
|
|
if reg as u32 >= X86InlineAsmReg::xmm0 as u32
|
|
&& reg as u32 <= X86InlineAsmReg::xmm15 as u32 =>
|
|
{
|
|
// rustc emits x0 rather than xmm0
|
|
write!(generated_asm, " movups [rbx+0x{:x}], ", offset.bytes()).unwrap();
|
|
write!(generated_asm, "xmm{}", reg as u32 - X86InlineAsmReg::xmm0 as u32)
|
|
.unwrap();
|
|
}
|
|
_ => {
|
|
write!(generated_asm, " mov [rbx+0x{:x}], ", offset.bytes()).unwrap();
|
|
reg.emit(generated_asm, InlineAsmArch::X86_64, None).unwrap();
|
|
}
|
|
}
|
|
generated_asm.push('\n');
|
|
}
|
|
InlineAsmArch::AArch64 => {
|
|
generated_asm.push_str(" str ");
|
|
reg.emit(generated_asm, InlineAsmArch::AArch64, None).unwrap();
|
|
writeln!(generated_asm, ", [x19, 0x{:x}]", offset.bytes()).unwrap();
|
|
}
|
|
InlineAsmArch::RiscV64 => {
|
|
generated_asm.push_str(" sd ");
|
|
reg.emit(generated_asm, InlineAsmArch::RiscV64, None).unwrap();
|
|
writeln!(generated_asm, ", 0x{:x}(s1)", offset.bytes()).unwrap();
|
|
}
|
|
_ => unimplemented!("save_register for {:?}", arch),
|
|
}
|
|
}
|
|
|
|
fn restore_register(
|
|
generated_asm: &mut String,
|
|
arch: InlineAsmArch,
|
|
reg: InlineAsmReg,
|
|
offset: Size,
|
|
) {
|
|
match arch {
|
|
InlineAsmArch::X86_64 => {
|
|
match reg {
|
|
InlineAsmReg::X86(reg)
|
|
if reg as u32 >= X86InlineAsmReg::xmm0 as u32
|
|
&& reg as u32 <= X86InlineAsmReg::xmm15 as u32 =>
|
|
{
|
|
// rustc emits x0 rather than xmm0
|
|
write!(
|
|
generated_asm,
|
|
" movups xmm{}",
|
|
reg as u32 - X86InlineAsmReg::xmm0 as u32
|
|
)
|
|
.unwrap();
|
|
}
|
|
_ => {
|
|
generated_asm.push_str(" mov ");
|
|
reg.emit(generated_asm, InlineAsmArch::X86_64, None).unwrap()
|
|
}
|
|
}
|
|
writeln!(generated_asm, ", [rbx+0x{:x}]", offset.bytes()).unwrap();
|
|
}
|
|
InlineAsmArch::AArch64 => {
|
|
generated_asm.push_str(" ldr ");
|
|
reg.emit(generated_asm, InlineAsmArch::AArch64, None).unwrap();
|
|
writeln!(generated_asm, ", [x19, 0x{:x}]", offset.bytes()).unwrap();
|
|
}
|
|
InlineAsmArch::RiscV64 => {
|
|
generated_asm.push_str(" ld ");
|
|
reg.emit(generated_asm, InlineAsmArch::RiscV64, None).unwrap();
|
|
writeln!(generated_asm, ", 0x{:x}(s1)", offset.bytes()).unwrap();
|
|
}
|
|
_ => unimplemented!("restore_register for {:?}", arch),
|
|
}
|
|
}
|
|
}
|
|
|
|
fn call_inline_asm<'tcx>(
|
|
fx: &mut FunctionCx<'_, '_, 'tcx>,
|
|
asm_name: &str,
|
|
slot_size: Size,
|
|
inputs: Vec<(Size, Value)>,
|
|
outputs: Vec<(Size, CPlace<'tcx>)>,
|
|
) {
|
|
let stack_slot = fx.create_stack_slot(u32::try_from(slot_size.bytes()).unwrap(), 16);
|
|
|
|
let inline_asm_func = fx
|
|
.module
|
|
.declare_function(
|
|
asm_name,
|
|
Linkage::Import,
|
|
&Signature {
|
|
call_conv: CallConv::SystemV,
|
|
params: vec![AbiParam::new(fx.pointer_type)],
|
|
returns: vec![],
|
|
},
|
|
)
|
|
.unwrap();
|
|
let inline_asm_func = fx.module.declare_func_in_func(inline_asm_func, fx.bcx.func);
|
|
if fx.clif_comments.enabled() {
|
|
fx.add_comment(inline_asm_func, asm_name);
|
|
}
|
|
|
|
for (offset, value) in inputs {
|
|
stack_slot.offset(fx, i32::try_from(offset.bytes()).unwrap().into()).store(
|
|
fx,
|
|
value,
|
|
MemFlags::trusted(),
|
|
);
|
|
}
|
|
|
|
let stack_slot_addr = stack_slot.get_addr(fx);
|
|
fx.bcx.ins().call(inline_asm_func, &[stack_slot_addr]);
|
|
|
|
for (offset, place) in outputs {
|
|
let ty = if place.layout().ty.is_simd() {
|
|
let (lane_count, lane_type) = place.layout().ty.simd_size_and_type(fx.tcx);
|
|
asm_clif_type(fx, lane_type).unwrap().by(lane_count.try_into().unwrap()).unwrap()
|
|
} else {
|
|
asm_clif_type(fx, place.layout().ty).unwrap()
|
|
};
|
|
let value = stack_slot.offset(fx, i32::try_from(offset.bytes()).unwrap().into()).load(
|
|
fx,
|
|
ty,
|
|
MemFlags::trusted(),
|
|
);
|
|
place.write_cvalue(fx, CValue::by_val(value, place.layout()));
|
|
}
|
|
}
|
|
|
|
fn asm_clif_type<'tcx>(fx: &FunctionCx<'_, '_, 'tcx>, ty: Ty<'tcx>) -> Option<types::Type> {
|
|
match ty.kind() {
|
|
// Adapted from https://github.com/rust-lang/rust/blob/f3c66088610c1b80110297c2d9a8b5f9265b013f/compiler/rustc_hir_analysis/src/check/intrinsicck.rs#L136-L151
|
|
ty::Adt(adt, args) if fx.tcx.is_lang_item(adt.did(), LangItem::MaybeUninit) => {
|
|
let fields = &adt.non_enum_variant().fields;
|
|
let ty = fields[FieldIdx::from_u32(1)].ty(fx.tcx, args);
|
|
let ty::Adt(ty, args) = ty.kind() else {
|
|
unreachable!("expected first field of `MaybeUninit` to be an ADT")
|
|
};
|
|
assert!(
|
|
ty.is_manually_drop(),
|
|
"expected first field of `MaybeUninit` to be `ManuallyDrop`"
|
|
);
|
|
let fields = &ty.non_enum_variant().fields;
|
|
let ty = fields[FieldIdx::ZERO].ty(fx.tcx, args);
|
|
fx.clif_type(ty)
|
|
}
|
|
_ => fx.clif_type(ty),
|
|
}
|
|
}
|