use gccjit::{RValue, ToRValue, Type}; use rustc_ast::ast::{InlineAsmOptions, InlineAsmTemplatePiece}; use rustc_codegen_ssa::mir::operand::OperandValue; use rustc_codegen_ssa::mir::place::PlaceRef; use rustc_codegen_ssa::traits::{AsmBuilderMethods, AsmMethods, BaseTypeMethods, BuilderMethods, GlobalAsmOperandRef, InlineAsmOperandRef}; use rustc_data_structures::fx::FxHashMap; use rustc_hir::LlvmInlineAsmInner; use rustc_middle::bug; use rustc_span::Span; use rustc_target::asm::*; use crate::builder::Builder; use crate::context::CodegenCx; use crate::type_of::LayoutGccExt; impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> { fn codegen_llvm_inline_asm(&mut self, _ia: &LlvmInlineAsmInner, _outputs: Vec>>, mut _inputs: Vec>, _span: Span) -> bool { // TODO(antoyo) return true; } fn codegen_inline_asm(&mut self, template: &[InlineAsmTemplatePiece], operands: &[InlineAsmOperandRef<'tcx, Self>], options: InlineAsmOptions, _span: &[Span]) { let asm_arch = self.tcx.sess.asm_arch.unwrap(); let intel_dialect = match asm_arch { InlineAsmArch::X86 | InlineAsmArch::X86_64 if !options.contains(InlineAsmOptions::ATT_SYNTAX) => true, _ => false, }; // Collect the types of output operands // FIXME(antoyo): we do this here instead of later because of a bug in libgccjit where creating the // variable after the extended asm expression causes a segfault: // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=100380 let mut output_vars = FxHashMap::default(); let mut operand_numbers = FxHashMap::default(); let mut current_number = 0; for (idx, op) in operands.iter().enumerate() { match *op { InlineAsmOperandRef::Out { place, .. } => { let ty = match place { Some(place) => place.layout.gcc_type(self.cx, false), None => { // If the output is discarded, we don't really care what // type is used. We're just using this to tell GCC to // reserve the register. //dummy_output_type(self.cx, reg.reg_class()) // NOTE: if no output value, we should not create one (it will be a // clobber). continue; }, }; let var = self.current_func().new_local(None, ty, "output_register"); operand_numbers.insert(idx, current_number); current_number += 1; output_vars.insert(idx, var); } InlineAsmOperandRef::InOut { out_place, .. } => { let ty = match out_place { Some(place) => place.layout.gcc_type(self.cx, false), None => { // NOTE: if no output value, we should not create one. continue; }, }; operand_numbers.insert(idx, current_number); current_number += 1; let var = self.current_func().new_local(None, ty, "output_register"); output_vars.insert(idx, var); } _ => {} } } // All output operands must come before the input operands, hence the 2 loops. for (idx, op) in operands.iter().enumerate() { match *op { InlineAsmOperandRef::In { .. } | InlineAsmOperandRef::InOut { .. } => { operand_numbers.insert(idx, current_number); current_number += 1; }, _ => (), } } // Build the template string let mut template_str = String::new(); for piece in template { match *piece { InlineAsmTemplatePiece::String(ref string) => { if string.contains('%') { for c in string.chars() { if c == '%' { template_str.push_str("%%"); } else { template_str.push(c); } } } else { template_str.push_str(string) } } InlineAsmTemplatePiece::Placeholder { operand_idx, modifier, span: _ } => { match operands[operand_idx] { InlineAsmOperandRef::Out { reg, place: Some(_), .. } => { let modifier = modifier_to_gcc(asm_arch, reg.reg_class(), modifier); if let Some(modifier) = modifier { template_str.push_str(&format!("%{}{}", modifier, operand_numbers[&operand_idx])); } else { template_str.push_str(&format!("%{}", operand_numbers[&operand_idx])); } }, InlineAsmOperandRef::Out { place: None, .. } => { unimplemented!("Out None"); }, InlineAsmOperandRef::In { reg, .. } | InlineAsmOperandRef::InOut { reg, .. } => { let modifier = modifier_to_gcc(asm_arch, reg.reg_class(), modifier); if let Some(modifier) = modifier { template_str.push_str(&format!("%{}{}", modifier, operand_numbers[&operand_idx])); } else { template_str.push_str(&format!("%{}", operand_numbers[&operand_idx])); } } InlineAsmOperandRef::Const { ref string } => { // Const operands get injected directly into the template template_str.push_str(string); } InlineAsmOperandRef::SymFn { .. } | InlineAsmOperandRef::SymStatic { .. } => { unimplemented!(); // Only emit the raw symbol name //template_str.push_str(&format!("${{{}:c}}", op_idx[&operand_idx])); } } } } } let block = self.llbb(); let template_str = if intel_dialect { template_str } else { // FIXME(antoyo): this might break the "m" memory constraint: // https://stackoverflow.com/a/9347957/389119 // TODO(antoyo): only set on x86 platforms. format!(".att_syntax noprefix\n\t{}\n\t.intel_syntax noprefix", template_str) }; let extended_asm = block.add_extended_asm(None, &template_str); // Collect the types of output operands let mut output_types = vec![]; for (idx, op) in operands.iter().enumerate() { match *op { InlineAsmOperandRef::Out { reg, late, place } => { let ty = match place { Some(place) => place.layout.gcc_type(self.cx, false), None => { // If the output is discarded, we don't really care what // type is used. We're just using this to tell GCC to // reserve the register. dummy_output_type(self.cx, reg.reg_class()) }, }; output_types.push(ty); let prefix = if late { "=" } else { "=&" }; let constraint = format!("{}{}", prefix, reg_to_gcc(reg)); if place.is_some() { let var = output_vars[&idx]; extended_asm.add_output_operand(None, &constraint, var); } else { // NOTE: reg.to_string() returns the register name with quotes around it so // remove them. extended_asm.add_clobber(reg.to_string().trim_matches('"')); } } InlineAsmOperandRef::InOut { reg, late, in_value, out_place } => { let ty = match out_place { Some(out_place) => out_place.layout.gcc_type(self.cx, false), None => dummy_output_type(self.cx, reg.reg_class()) }; output_types.push(ty); // TODO(antoyo): prefix of "+" for reading and writing? let prefix = if late { "=" } else { "=&" }; let constraint = format!("{}{}", prefix, reg_to_gcc(reg)); if out_place.is_some() { let var = output_vars[&idx]; // TODO(antoyo): also specify an output operand when out_place is none: that would // be the clobber but clobbers do not support general constraint like reg; // they only support named registers. // Not sure how we can do this. And the LLVM backend does not seem to add a // clobber. extended_asm.add_output_operand(None, &constraint, var); } let constraint = reg_to_gcc(reg); extended_asm.add_input_operand(None, &constraint, in_value.immediate()); } InlineAsmOperandRef::In { reg, value } => { let constraint = reg_to_gcc(reg); extended_asm.add_input_operand(None, &constraint, value.immediate()); } _ => {} } } // Write results to outputs for (idx, op) in operands.iter().enumerate() { if let InlineAsmOperandRef::Out { place: Some(place), .. } | InlineAsmOperandRef::InOut { out_place: Some(place), .. } = *op { OperandValue::Immediate(output_vars[&idx].to_rvalue()).store(self, place); } } } } /// Converts a register class to a GCC constraint code. // TODO(antoyo): return &'static str instead? fn reg_to_gcc(reg: InlineAsmRegOrRegClass) -> String { match reg { // For vector registers LLVM wants the register name to match the type size. InlineAsmRegOrRegClass::Reg(reg) => { // TODO(antoyo): add support for vector register. let constraint = match reg.name() { "ax" => "a", "bx" => "b", "cx" => "c", "dx" => "d", "si" => "S", "di" => "D", // TODO(antoyo): for registers like r11, we have to create a register variable: https://stackoverflow.com/a/31774784/389119 // TODO(antoyo): in this case though, it's a clobber, so it should work as r11. // Recent nightly supports clobber() syntax, so update to it. It does not seem // like it's implemented yet. name => name, // FIXME(antoyo): probably wrong. }; constraint.to_string() }, InlineAsmRegOrRegClass::RegClass(reg) => match reg { InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => unimplemented!(), InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => unimplemented!(), InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg) => unimplemented!(), InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => unimplemented!(), InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg) => unimplemented!(), InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg_thumb) => unimplemented!(), InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg) | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low16) | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low8) => unimplemented!(), InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg_low16) | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low8) | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low4) => unimplemented!(), InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg) | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg) => unimplemented!(), InlineAsmRegClass::Bpf(_) => unimplemented!(), InlineAsmRegClass::Hexagon(HexagonInlineAsmRegClass::reg) => unimplemented!(), InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg) => unimplemented!(), InlineAsmRegClass::Mips(MipsInlineAsmRegClass::freg) => unimplemented!(), InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg16) => unimplemented!(), InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg32) => unimplemented!(), InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg64) => unimplemented!(), InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg) => unimplemented!(), InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => unimplemented!(), InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::freg) => unimplemented!(), InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg) => unimplemented!(), InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => unimplemented!(), InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::vreg) => unimplemented!(), InlineAsmRegClass::X86(X86InlineAsmRegClass::mmx_reg) => unimplemented!(), InlineAsmRegClass::X86(X86InlineAsmRegClass::reg) => "r", InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd) => unimplemented!(), InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_byte) => unimplemented!(), InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg) | InlineAsmRegClass::X86(X86InlineAsmRegClass::ymm_reg) => unimplemented!(), InlineAsmRegClass::X86(X86InlineAsmRegClass::x87_reg) => unimplemented!(), InlineAsmRegClass::X86(X86InlineAsmRegClass::zmm_reg) => unimplemented!(), InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => unimplemented!(), InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => unimplemented!(), InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => { bug!("GCC backend does not support SPIR-V") } InlineAsmRegClass::Err => unreachable!(), } .to_string(), } } /// Type to use for outputs that are discarded. It doesn't really matter what /// the type is, as long as it is valid for the constraint code. fn dummy_output_type<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, reg: InlineAsmRegClass) -> Type<'gcc> { match reg { InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => cx.type_i32(), InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => unimplemented!(), InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg) | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => { unimplemented!() } InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg) | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg_thumb) => cx.type_i32(), InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg) | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg_low16) => cx.type_f32(), InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg) | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low16) | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low8) => cx.type_f64(), InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg) | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low8) | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low4) => { unimplemented!() } InlineAsmRegClass::Bpf(_) => unimplemented!(), InlineAsmRegClass::Hexagon(HexagonInlineAsmRegClass::reg) => cx.type_i32(), InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg) => cx.type_i32(), InlineAsmRegClass::Mips(MipsInlineAsmRegClass::freg) => cx.type_f32(), InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg16) => cx.type_i16(), InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg32) => cx.type_i32(), InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg64) => cx.type_i64(), InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg) => cx.type_i32(), InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => cx.type_i32(), InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::freg) => cx.type_f64(), InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg) => cx.type_i32(), InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => cx.type_f32(), InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::vreg) => cx.type_f32(), InlineAsmRegClass::X86(X86InlineAsmRegClass::reg) | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd) => cx.type_i32(), InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_byte) => cx.type_i8(), InlineAsmRegClass::X86(X86InlineAsmRegClass::mmx_reg) => unimplemented!(), InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg) | InlineAsmRegClass::X86(X86InlineAsmRegClass::ymm_reg) | InlineAsmRegClass::X86(X86InlineAsmRegClass::zmm_reg) => cx.type_f32(), InlineAsmRegClass::X86(X86InlineAsmRegClass::x87_reg) => unimplemented!(), InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => cx.type_i16(), InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => cx.type_i32(), InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => { bug!("LLVM backend does not support SPIR-V") }, InlineAsmRegClass::Err => unreachable!(), } } impl<'gcc, 'tcx> AsmMethods for CodegenCx<'gcc, 'tcx> { fn codegen_global_asm(&self, template: &[InlineAsmTemplatePiece], operands: &[GlobalAsmOperandRef], options: InlineAsmOptions, _line_spans: &[Span]) { let asm_arch = self.tcx.sess.asm_arch.unwrap(); // Default to Intel syntax on x86 let intel_syntax = matches!(asm_arch, InlineAsmArch::X86 | InlineAsmArch::X86_64) && !options.contains(InlineAsmOptions::ATT_SYNTAX); // Build the template string let mut template_str = String::new(); for piece in template { match *piece { InlineAsmTemplatePiece::String(ref string) => { for line in string.lines() { // NOTE: gcc does not allow inline comment, so remove them. let line = if let Some(index) = line.rfind("//") { &line[..index] } else { line }; template_str.push_str(line); template_str.push('\n'); } }, InlineAsmTemplatePiece::Placeholder { operand_idx, modifier: _, span: _ } => { match operands[operand_idx] { GlobalAsmOperandRef::Const { ref string } => { // Const operands get injected directly into the // template. Note that we don't need to escape $ // here unlike normal inline assembly. template_str.push_str(string); } } } } } let template_str = if intel_syntax { format!("{}\n\t.intel_syntax noprefix", template_str) } else { format!(".att_syntax\n\t{}\n\t.intel_syntax noprefix", template_str) }; // NOTE: seems like gcc will put the asm in the wrong section, so set it to .text manually. let template_str = format!(".pushsection .text\n{}\n.popsection", template_str); self.context.add_top_level_asm(None, &template_str); } } fn modifier_to_gcc(arch: InlineAsmArch, reg: InlineAsmRegClass, modifier: Option) -> Option { match reg { InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => modifier, InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => modifier, InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg) | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => { unimplemented!() } InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg) | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg_thumb) => unimplemented!(), InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg) | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg_low16) => unimplemented!(), InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg) | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low16) | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low8) => unimplemented!(), InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg) | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low8) | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low4) => { unimplemented!() } InlineAsmRegClass::Bpf(_) => unimplemented!(), InlineAsmRegClass::Hexagon(_) => unimplemented!(), InlineAsmRegClass::Mips(_) => unimplemented!(), InlineAsmRegClass::Nvptx(_) => unimplemented!(), InlineAsmRegClass::PowerPC(_) => unimplemented!(), InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg) | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => unimplemented!(), InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::vreg) => unimplemented!(), InlineAsmRegClass::X86(X86InlineAsmRegClass::reg) | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd) => match modifier { None if arch == InlineAsmArch::X86_64 => Some('q'), None => Some('k'), Some('l') => Some('b'), Some('h') => Some('h'), Some('x') => Some('w'), Some('e') => Some('k'), Some('r') => Some('q'), _ => unreachable!(), }, InlineAsmRegClass::X86(X86InlineAsmRegClass::mmx_reg) => unimplemented!(), InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_byte) => unimplemented!(), InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg) | InlineAsmRegClass::X86(X86InlineAsmRegClass::ymm_reg) | InlineAsmRegClass::X86(X86InlineAsmRegClass::zmm_reg) => unimplemented!(), InlineAsmRegClass::X86(X86InlineAsmRegClass::x87_reg) => unimplemented!(), InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => unimplemented!(), InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => unimplemented!(), InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => { bug!("LLVM backend does not support SPIR-V") }, InlineAsmRegClass::Err => unreachable!(), } }