// Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT // file at the top-level directory of this distribution and at // http://rust-lang.org/COPYRIGHT. // // Licensed under the Apache License, Version 2.0 or the MIT license // , at your // option. This file may not be copied, modified, or distributed // except according to those terms. #![allow(non_upper_case_globals)] use arena::TypedArena; use intrinsics::{self, Intrinsic}; use libc; use llvm; use llvm::{SequentiallyConsistent, Acquire, Release, AtomicXchg, ValueRef, TypeKind}; use middle::subst; use middle::subst::FnSpace; use trans::adt; use trans::attributes; use trans::base::*; use trans::build::*; use trans::callee; use trans::cleanup; use trans::cleanup::CleanupMethods; use trans::common::*; use trans::consts; use trans::datum::*; use trans::debuginfo::DebugLoc; use trans::declare; use trans::expr; use trans::glue; use trans::type_of::*; use trans::type_of; use trans::machine; use trans::machine::llsize_of; use trans::type_::Type; use middle::ty::{self, Ty, HasTypeFlags}; use middle::subst::Substs; use syntax::abi::{self, RustIntrinsic}; use syntax::ast; use syntax::ptr::P; use syntax::parse::token; use std::cmp::Ordering; pub fn get_simple_intrinsic(ccx: &CrateContext, item: &ast::ForeignItem) -> Option { let name = match &*item.ident.name.as_str() { "sqrtf32" => "llvm.sqrt.f32", "sqrtf64" => "llvm.sqrt.f64", "powif32" => "llvm.powi.f32", "powif64" => "llvm.powi.f64", "sinf32" => "llvm.sin.f32", "sinf64" => "llvm.sin.f64", "cosf32" => "llvm.cos.f32", "cosf64" => "llvm.cos.f64", "powf32" => "llvm.pow.f32", "powf64" => "llvm.pow.f64", "expf32" => "llvm.exp.f32", "expf64" => "llvm.exp.f64", "exp2f32" => "llvm.exp2.f32", "exp2f64" => "llvm.exp2.f64", "logf32" => "llvm.log.f32", "logf64" => "llvm.log.f64", "log10f32" => "llvm.log10.f32", "log10f64" => "llvm.log10.f64", "log2f32" => "llvm.log2.f32", "log2f64" => "llvm.log2.f64", "fmaf32" => "llvm.fma.f32", "fmaf64" => "llvm.fma.f64", "fabsf32" => "llvm.fabs.f32", "fabsf64" => "llvm.fabs.f64", "copysignf32" => "llvm.copysign.f32", "copysignf64" => "llvm.copysign.f64", "floorf32" => "llvm.floor.f32", "floorf64" => "llvm.floor.f64", "ceilf32" => "llvm.ceil.f32", "ceilf64" => "llvm.ceil.f64", "truncf32" => "llvm.trunc.f32", "truncf64" => "llvm.trunc.f64", "rintf32" => "llvm.rint.f32", "rintf64" => "llvm.rint.f64", "nearbyintf32" => "llvm.nearbyint.f32", "nearbyintf64" => "llvm.nearbyint.f64", "roundf32" => "llvm.round.f32", "roundf64" => "llvm.round.f64", "ctpop8" => "llvm.ctpop.i8", "ctpop16" => "llvm.ctpop.i16", "ctpop32" => "llvm.ctpop.i32", "ctpop64" => "llvm.ctpop.i64", "bswap16" => "llvm.bswap.i16", "bswap32" => "llvm.bswap.i32", "bswap64" => "llvm.bswap.i64", "assume" => "llvm.assume", _ => return None }; Some(ccx.get_intrinsic(&name)) } /// Performs late verification that intrinsics are used correctly. At present, /// the only intrinsic that needs such verification is `transmute`. pub fn check_intrinsics(ccx: &CrateContext) { let mut last_failing_id = None; for transmute_restriction in ccx.tcx().transmute_restrictions.borrow().iter() { // Sometimes, a single call to transmute will push multiple // type pairs to test in order to exhaustively test the // possibility around a type parameter. If one of those fails, // there is no sense reporting errors on the others. if last_failing_id == Some(transmute_restriction.id) { continue; } debug!("transmute_restriction: {:?}", transmute_restriction); assert!(!transmute_restriction.substituted_from.has_param_types()); assert!(!transmute_restriction.substituted_to.has_param_types()); let llfromtype = type_of::sizing_type_of(ccx, transmute_restriction.substituted_from); let lltotype = type_of::sizing_type_of(ccx, transmute_restriction.substituted_to); let from_type_size = machine::llbitsize_of_real(ccx, llfromtype); let to_type_size = machine::llbitsize_of_real(ccx, lltotype); if from_type_size != to_type_size { last_failing_id = Some(transmute_restriction.id); if transmute_restriction.original_from != transmute_restriction.substituted_from { ccx.sess().span_err( transmute_restriction.span, &format!("transmute called on types with potentially different sizes: \ {} (could be {} bit{}) to {} (could be {} bit{})", transmute_restriction.original_from, from_type_size as usize, if from_type_size == 1 {""} else {"s"}, transmute_restriction.original_to, to_type_size as usize, if to_type_size == 1 {""} else {"s"})); } else { ccx.sess().span_err( transmute_restriction.span, &format!("transmute called on types with different sizes: \ {} ({} bit{}) to {} ({} bit{})", transmute_restriction.original_from, from_type_size as usize, if from_type_size == 1 {""} else {"s"}, transmute_restriction.original_to, to_type_size as usize, if to_type_size == 1 {""} else {"s"})); } } } ccx.sess().abort_if_errors(); } /// Remember to add all intrinsics here, in librustc_typeck/check/mod.rs, /// and in libcore/intrinsics.rs; if you need access to any llvm intrinsics, /// add them to librustc_trans/trans/context.rs pub fn trans_intrinsic_call<'a, 'blk, 'tcx>(mut bcx: Block<'blk, 'tcx>, node: ast::NodeId, callee_ty: Ty<'tcx>, cleanup_scope: cleanup::CustomScopeIndex, args: callee::CallArgs<'a, 'tcx>, dest: expr::Dest, substs: subst::Substs<'tcx>, call_info: NodeIdAndSpan) -> Result<'blk, 'tcx> { let fcx = bcx.fcx; let ccx = fcx.ccx; let tcx = bcx.tcx(); let _icx = push_ctxt("trans_intrinsic_call"); let ret_ty = match callee_ty.sty { ty::TyBareFn(_, ref f) => { bcx.tcx().erase_late_bound_regions(&f.sig.output()) } _ => panic!("expected bare_fn in trans_intrinsic_call") }; let foreign_item = tcx.map.expect_foreign_item(node); let name = foreign_item.ident.name.as_str(); // For `transmute` we can just trans the input expr directly into dest if name == "transmute" { let llret_ty = type_of::type_of(ccx, ret_ty.unwrap()); match args { callee::ArgExprs(arg_exprs) => { assert_eq!(arg_exprs.len(), 1); let (in_type, out_type) = (*substs.types.get(FnSpace, 0), *substs.types.get(FnSpace, 1)); let llintype = type_of::type_of(ccx, in_type); let llouttype = type_of::type_of(ccx, out_type); let in_type_size = machine::llbitsize_of_real(ccx, llintype); let out_type_size = machine::llbitsize_of_real(ccx, llouttype); // This should be caught by the intrinsicck pass assert_eq!(in_type_size, out_type_size); let nonpointer_nonaggregate = |llkind: TypeKind| -> bool { use llvm::TypeKind::*; match llkind { Half | Float | Double | X86_FP80 | FP128 | PPC_FP128 | Integer | Vector | X86_MMX => true, _ => false } }; // An approximation to which types can be directly cast via // LLVM's bitcast. This doesn't cover pointer -> pointer casts, // but does, importantly, cover SIMD types. let in_kind = llintype.kind(); let ret_kind = llret_ty.kind(); let bitcast_compatible = (nonpointer_nonaggregate(in_kind) && nonpointer_nonaggregate(ret_kind)) || { in_kind == TypeKind::Pointer && ret_kind == TypeKind::Pointer }; let dest = if bitcast_compatible { // if we're here, the type is scalar-like (a primitive, a // SIMD type or a pointer), and so can be handled as a // by-value ValueRef and can also be directly bitcast to the // target type. Doing this special case makes conversions // like `u32x4` -> `u64x2` much nicer for LLVM and so more // efficient (these are done efficiently implicitly in C // with the `__m128i` type and so this means Rust doesn't // lose out there). let expr = &*arg_exprs[0]; let datum = unpack_datum!(bcx, expr::trans(bcx, expr)); let datum = unpack_datum!(bcx, datum.to_rvalue_datum(bcx, "transmute_temp")); let val = if datum.kind.is_by_ref() { load_ty(bcx, datum.val, datum.ty) } else { from_arg_ty(bcx, datum.val, datum.ty) }; let cast_val = BitCast(bcx, val, llret_ty); match dest { expr::SaveIn(d) => { // this often occurs in a sequence like `Store(val, // d); val2 = Load(d)`, so disappears easily. Store(bcx, cast_val, d); } expr::Ignore => {} } dest } else { // The types are too complicated to do with a by-value // bitcast, so pointer cast instead. We need to cast the // dest so the types work out. let dest = match dest { expr::SaveIn(d) => expr::SaveIn(PointerCast(bcx, d, llintype.ptr_to())), expr::Ignore => expr::Ignore }; bcx = expr::trans_into(bcx, &*arg_exprs[0], dest); dest }; fcx.scopes.borrow_mut().last_mut().unwrap().drop_non_lifetime_clean(); fcx.pop_and_trans_custom_cleanup_scope(bcx, cleanup_scope); return match dest { expr::SaveIn(d) => Result::new(bcx, d), expr::Ignore => Result::new(bcx, C_undef(llret_ty.ptr_to())) }; } _ => { ccx.sess().bug("expected expr as argument for transmute"); } } } // For `move_val_init` we can evaluate the destination address // (the first argument) and then trans the source value (the // second argument) directly into the resulting destination // address. if name == "move_val_init" { if let callee::ArgExprs(ref exprs) = args { let (dest_expr, source_expr) = if exprs.len() != 2 { ccx.sess().bug("expected two exprs as arguments for `move_val_init` intrinsic"); } else { (&exprs[0], &exprs[1]) }; // evaluate destination address let dest_datum = unpack_datum!(bcx, expr::trans(bcx, dest_expr)); let dest_datum = unpack_datum!( bcx, dest_datum.to_rvalue_datum(bcx, "arg")); let dest_datum = unpack_datum!( bcx, dest_datum.to_appropriate_datum(bcx)); // `expr::trans_into(bcx, expr, dest)` is equiv to // // `trans(bcx, expr).store_to_dest(dest)`, // // which for `dest == expr::SaveIn(addr)`, is equivalent to: // // `trans(bcx, expr).store_to(bcx, addr)`. let lldest = expr::Dest::SaveIn(dest_datum.val); bcx = expr::trans_into(bcx, source_expr, lldest); let llresult = C_nil(ccx); fcx.pop_and_trans_custom_cleanup_scope(bcx, cleanup_scope); return Result::new(bcx, llresult); } else { ccx.sess().bug("expected two exprs as arguments for `move_val_init` intrinsic"); } } let call_debug_location = DebugLoc::At(call_info.id, call_info.span); // For `try` we need some custom control flow if &name[..] == "try" { if let callee::ArgExprs(ref exprs) = args { let (func, data) = if exprs.len() != 2 { ccx.sess().bug("expected two exprs as arguments for \ `try` intrinsic"); } else { (&exprs[0], &exprs[1]) }; // translate arguments let func = unpack_datum!(bcx, expr::trans(bcx, func)); let func = unpack_datum!(bcx, func.to_rvalue_datum(bcx, "func")); let data = unpack_datum!(bcx, expr::trans(bcx, data)); let data = unpack_datum!(bcx, data.to_rvalue_datum(bcx, "data")); let dest = match dest { expr::SaveIn(d) => d, expr::Ignore => alloc_ty(bcx, tcx.mk_mut_ptr(tcx.types.i8), "try_result"), }; // do the invoke bcx = try_intrinsic(bcx, func.val, data.val, dest, call_debug_location); fcx.pop_and_trans_custom_cleanup_scope(bcx, cleanup_scope); return Result::new(bcx, dest); } else { ccx.sess().bug("expected two exprs as arguments for \ `try` intrinsic"); } } // save the actual AST arguments for later (some places need to do // const-evaluation on them) let expr_arguments = match args { callee::ArgExprs(args) => Some(args), _ => None, }; // Push the arguments. let mut llargs = Vec::new(); bcx = callee::trans_args(bcx, args, callee_ty, &mut llargs, cleanup::CustomScope(cleanup_scope), false, RustIntrinsic); fcx.scopes.borrow_mut().last_mut().unwrap().drop_non_lifetime_clean(); // These are the only intrinsic functions that diverge. if name == "abort" { let llfn = ccx.get_intrinsic(&("llvm.trap")); Call(bcx, llfn, &[], None, call_debug_location); fcx.pop_and_trans_custom_cleanup_scope(bcx, cleanup_scope); Unreachable(bcx); return Result::new(bcx, C_undef(Type::nil(ccx).ptr_to())); } else if &name[..] == "unreachable" { fcx.pop_and_trans_custom_cleanup_scope(bcx, cleanup_scope); Unreachable(bcx); return Result::new(bcx, C_nil(ccx)); } let ret_ty = match ret_ty { ty::FnConverging(ret_ty) => ret_ty, ty::FnDiverging => unreachable!() }; let llret_ty = type_of::type_of(ccx, ret_ty); // Get location to store the result. If the user does // not care about the result, just make a stack slot let llresult = match dest { expr::SaveIn(d) => d, expr::Ignore => { if !type_is_zero_size(ccx, ret_ty) { alloc_ty(bcx, ret_ty, "intrinsic_result") } else { C_undef(llret_ty.ptr_to()) } } }; let simple = get_simple_intrinsic(ccx, &*foreign_item); let llval = match (simple, &*name) { (Some(llfn), _) => { Call(bcx, llfn, &llargs, None, call_debug_location) } (_, "breakpoint") => { let llfn = ccx.get_intrinsic(&("llvm.debugtrap")); Call(bcx, llfn, &[], None, call_debug_location) } (_, "size_of") => { let tp_ty = *substs.types.get(FnSpace, 0); let lltp_ty = type_of::type_of(ccx, tp_ty); C_uint(ccx, machine::llsize_of_alloc(ccx, lltp_ty)) } (_, "size_of_val") => { let tp_ty = *substs.types.get(FnSpace, 0); if !type_is_sized(tcx, tp_ty) { let (llsize, _) = glue::size_and_align_of_dst(bcx, tp_ty, llargs[1]); llsize } else { let lltp_ty = type_of::type_of(ccx, tp_ty); C_uint(ccx, machine::llsize_of_alloc(ccx, lltp_ty)) } } (_, "min_align_of") => { let tp_ty = *substs.types.get(FnSpace, 0); C_uint(ccx, type_of::align_of(ccx, tp_ty)) } (_, "min_align_of_val") => { let tp_ty = *substs.types.get(FnSpace, 0); if !type_is_sized(tcx, tp_ty) { let (_, llalign) = glue::size_and_align_of_dst(bcx, tp_ty, llargs[1]); llalign } else { C_uint(ccx, type_of::align_of(ccx, tp_ty)) } } (_, "pref_align_of") => { let tp_ty = *substs.types.get(FnSpace, 0); let lltp_ty = type_of::type_of(ccx, tp_ty); C_uint(ccx, machine::llalign_of_pref(ccx, lltp_ty)) } (_, "drop_in_place") => { let tp_ty = *substs.types.get(FnSpace, 0); let ptr = if type_is_sized(tcx, tp_ty) { llargs[0] } else { let scratch = rvalue_scratch_datum(bcx, tp_ty, "tmp"); Store(bcx, llargs[0], expr::get_dataptr(bcx, scratch.val)); Store(bcx, llargs[1], expr::get_len(bcx, scratch.val)); fcx.schedule_lifetime_end(cleanup::CustomScope(cleanup_scope), scratch.val); scratch.val }; glue::drop_ty(bcx, ptr, tp_ty, call_debug_location); C_nil(ccx) } (_, "type_name") => { let tp_ty = *substs.types.get(FnSpace, 0); let ty_name = token::intern_and_get_ident(&tp_ty.to_string()); C_str_slice(ccx, ty_name) } (_, "type_id") => { let hash = ccx.tcx().hash_crate_independent(*substs.types.get(FnSpace, 0), &ccx.link_meta().crate_hash); C_u64(ccx, hash) } (_, "init_dropped") => { let tp_ty = *substs.types.get(FnSpace, 0); if !return_type_is_void(ccx, tp_ty) { drop_done_fill_mem(bcx, llresult, tp_ty); } C_nil(ccx) } (_, "init") => { let tp_ty = *substs.types.get(FnSpace, 0); if !return_type_is_void(ccx, tp_ty) { // Just zero out the stack slot. (See comment on base::memzero for explanation) init_zero_mem(bcx, llresult, tp_ty); } C_nil(ccx) } // Effectively no-ops (_, "uninit") | (_, "forget") => { C_nil(ccx) } (_, "needs_drop") => { let tp_ty = *substs.types.get(FnSpace, 0); C_bool(ccx, bcx.fcx.type_needs_drop(tp_ty)) } (_, "offset") => { let ptr = llargs[0]; let offset = llargs[1]; InBoundsGEP(bcx, ptr, &[offset]) } (_, "arith_offset") => { let ptr = llargs[0]; let offset = llargs[1]; GEP(bcx, ptr, &[offset]) } (_, "copy_nonoverlapping") => { copy_intrinsic(bcx, false, false, *substs.types.get(FnSpace, 0), llargs[1], llargs[0], llargs[2], call_debug_location) } (_, "copy") => { copy_intrinsic(bcx, true, false, *substs.types.get(FnSpace, 0), llargs[1], llargs[0], llargs[2], call_debug_location) } (_, "write_bytes") => { memset_intrinsic(bcx, false, *substs.types.get(FnSpace, 0), llargs[0], llargs[1], llargs[2], call_debug_location) } (_, "volatile_copy_nonoverlapping_memory") => { copy_intrinsic(bcx, false, true, *substs.types.get(FnSpace, 0), llargs[0], llargs[1], llargs[2], call_debug_location) } (_, "volatile_copy_memory") => { copy_intrinsic(bcx, true, true, *substs.types.get(FnSpace, 0), llargs[0], llargs[1], llargs[2], call_debug_location) } (_, "volatile_set_memory") => { memset_intrinsic(bcx, true, *substs.types.get(FnSpace, 0), llargs[0], llargs[1], llargs[2], call_debug_location) } (_, "volatile_load") => { let tp_ty = *substs.types.get(FnSpace, 0); let ptr = to_arg_ty_ptr(bcx, llargs[0], tp_ty); let load = VolatileLoad(bcx, ptr); unsafe { llvm::LLVMSetAlignment(load, type_of::align_of(ccx, tp_ty)); } to_arg_ty(bcx, load, tp_ty) }, (_, "volatile_store") => { let tp_ty = *substs.types.get(FnSpace, 0); let ptr = to_arg_ty_ptr(bcx, llargs[0], tp_ty); let val = from_arg_ty(bcx, llargs[1], tp_ty); let store = VolatileStore(bcx, val, ptr); unsafe { llvm::LLVMSetAlignment(store, type_of::align_of(ccx, tp_ty)); } C_nil(ccx) }, (_, "ctlz8") => count_zeros_intrinsic(bcx, "llvm.ctlz.i8", llargs[0], call_debug_location), (_, "ctlz16") => count_zeros_intrinsic(bcx, "llvm.ctlz.i16", llargs[0], call_debug_location), (_, "ctlz32") => count_zeros_intrinsic(bcx, "llvm.ctlz.i32", llargs[0], call_debug_location), (_, "ctlz64") => count_zeros_intrinsic(bcx, "llvm.ctlz.i64", llargs[0], call_debug_location), (_, "cttz8") => count_zeros_intrinsic(bcx, "llvm.cttz.i8", llargs[0], call_debug_location), (_, "cttz16") => count_zeros_intrinsic(bcx, "llvm.cttz.i16", llargs[0], call_debug_location), (_, "cttz32") => count_zeros_intrinsic(bcx, "llvm.cttz.i32", llargs[0], call_debug_location), (_, "cttz64") => count_zeros_intrinsic(bcx, "llvm.cttz.i64", llargs[0], call_debug_location), (_, "i8_add_with_overflow") => with_overflow_intrinsic(bcx, "llvm.sadd.with.overflow.i8", ret_ty, llargs[0], llargs[1], call_debug_location), (_, "i16_add_with_overflow") => with_overflow_intrinsic(bcx, "llvm.sadd.with.overflow.i16", ret_ty, llargs[0], llargs[1], call_debug_location), (_, "i32_add_with_overflow") => with_overflow_intrinsic(bcx, "llvm.sadd.with.overflow.i32", ret_ty, llargs[0], llargs[1], call_debug_location), (_, "i64_add_with_overflow") => with_overflow_intrinsic(bcx, "llvm.sadd.with.overflow.i64", ret_ty, llargs[0], llargs[1], call_debug_location), (_, "u8_add_with_overflow") => with_overflow_intrinsic(bcx, "llvm.uadd.with.overflow.i8", ret_ty, llargs[0], llargs[1], call_debug_location), (_, "u16_add_with_overflow") => with_overflow_intrinsic(bcx, "llvm.uadd.with.overflow.i16", ret_ty, llargs[0], llargs[1], call_debug_location), (_, "u32_add_with_overflow") => with_overflow_intrinsic(bcx, "llvm.uadd.with.overflow.i32", ret_ty, llargs[0], llargs[1], call_debug_location), (_, "u64_add_with_overflow") => with_overflow_intrinsic(bcx, "llvm.uadd.with.overflow.i64", ret_ty, llargs[0], llargs[1], call_debug_location), (_, "i8_sub_with_overflow") => with_overflow_intrinsic(bcx, "llvm.ssub.with.overflow.i8", ret_ty, llargs[0], llargs[1], call_debug_location), (_, "i16_sub_with_overflow") => with_overflow_intrinsic(bcx, "llvm.ssub.with.overflow.i16", ret_ty, llargs[0], llargs[1], call_debug_location), (_, "i32_sub_with_overflow") => with_overflow_intrinsic(bcx, "llvm.ssub.with.overflow.i32", ret_ty, llargs[0], llargs[1], call_debug_location), (_, "i64_sub_with_overflow") => with_overflow_intrinsic(bcx, "llvm.ssub.with.overflow.i64", ret_ty, llargs[0], llargs[1], call_debug_location), (_, "u8_sub_with_overflow") => with_overflow_intrinsic(bcx, "llvm.usub.with.overflow.i8", ret_ty, llargs[0], llargs[1], call_debug_location), (_, "u16_sub_with_overflow") => with_overflow_intrinsic(bcx, "llvm.usub.with.overflow.i16", ret_ty, llargs[0], llargs[1], call_debug_location), (_, "u32_sub_with_overflow") => with_overflow_intrinsic(bcx, "llvm.usub.with.overflow.i32", ret_ty, llargs[0], llargs[1], call_debug_location), (_, "u64_sub_with_overflow") => with_overflow_intrinsic(bcx, "llvm.usub.with.overflow.i64", ret_ty, llargs[0], llargs[1], call_debug_location), (_, "i8_mul_with_overflow") => with_overflow_intrinsic(bcx, "llvm.smul.with.overflow.i8", ret_ty, llargs[0], llargs[1], call_debug_location), (_, "i16_mul_with_overflow") => with_overflow_intrinsic(bcx, "llvm.smul.with.overflow.i16", ret_ty, llargs[0], llargs[1], call_debug_location), (_, "i32_mul_with_overflow") => with_overflow_intrinsic(bcx, "llvm.smul.with.overflow.i32", ret_ty, llargs[0], llargs[1], call_debug_location), (_, "i64_mul_with_overflow") => with_overflow_intrinsic(bcx, "llvm.smul.with.overflow.i64", ret_ty, llargs[0], llargs[1], call_debug_location), (_, "u8_mul_with_overflow") => with_overflow_intrinsic(bcx, "llvm.umul.with.overflow.i8", ret_ty, llargs[0], llargs[1], call_debug_location), (_, "u16_mul_with_overflow") => with_overflow_intrinsic(bcx, "llvm.umul.with.overflow.i16", ret_ty, llargs[0], llargs[1], call_debug_location), (_, "u32_mul_with_overflow") => with_overflow_intrinsic(bcx, "llvm.umul.with.overflow.i32", ret_ty, llargs[0], llargs[1], call_debug_location), (_, "u64_mul_with_overflow") => with_overflow_intrinsic(bcx, "llvm.umul.with.overflow.i64", ret_ty, llargs[0], llargs[1], call_debug_location), (_, "unchecked_udiv") => UDiv(bcx, llargs[0], llargs[1], call_debug_location), (_, "unchecked_sdiv") => SDiv(bcx, llargs[0], llargs[1], call_debug_location), (_, "unchecked_urem") => URem(bcx, llargs[0], llargs[1], call_debug_location), (_, "unchecked_srem") => SRem(bcx, llargs[0], llargs[1], call_debug_location), (_, "overflowing_add") => Add(bcx, llargs[0], llargs[1], call_debug_location), (_, "overflowing_sub") => Sub(bcx, llargs[0], llargs[1], call_debug_location), (_, "overflowing_mul") => Mul(bcx, llargs[0], llargs[1], call_debug_location), (_, "return_address") => { if !fcx.caller_expects_out_pointer { tcx.sess.span_err(call_info.span, "invalid use of `return_address` intrinsic: function \ does not use out pointer"); C_null(Type::i8p(ccx)) } else { PointerCast(bcx, llvm::get_param(fcx.llfn, 0), Type::i8p(ccx)) } } (_, "discriminant_value") => { let val_ty = substs.types.get(FnSpace, 0); match val_ty.sty { ty::TyEnum(..) => { let repr = adt::represent_type(ccx, *val_ty); adt::trans_get_discr(bcx, &*repr, llargs[0], Some(llret_ty)) } _ => C_null(llret_ty) } } (_, name) if name.starts_with("simd_") => { generic_simd_intrinsic(bcx, name, substs, callee_ty, expr_arguments, &llargs, ret_ty, llret_ty, call_debug_location, call_info) } // This requires that atomic intrinsics follow a specific naming pattern: // "atomic_[_]", and no ordering means SeqCst (_, name) if name.starts_with("atomic_") => { let split: Vec<&str> = name.split('_').collect(); assert!(split.len() >= 2, "Atomic intrinsic not correct format"); let order = if split.len() == 2 { llvm::SequentiallyConsistent } else { match split[2] { "unordered" => llvm::Unordered, "relaxed" => llvm::Monotonic, "acq" => llvm::Acquire, "rel" => llvm::Release, "acqrel" => llvm::AcquireRelease, _ => ccx.sess().fatal("unknown ordering in atomic intrinsic") } }; match split[1] { "cxchg" => { // See include/llvm/IR/Instructions.h for their implementation // of this, I assume that it's good enough for us to use for // now. let strongest_failure_ordering = match order { llvm::NotAtomic | llvm::Unordered => ccx.sess().fatal("cmpxchg must be atomic"), llvm::Monotonic | llvm::Release => llvm::Monotonic, llvm::Acquire | llvm::AcquireRelease => llvm::Acquire, llvm::SequentiallyConsistent => llvm::SequentiallyConsistent }; let tp_ty = *substs.types.get(FnSpace, 0); let ptr = to_arg_ty_ptr(bcx, llargs[0], tp_ty); let cmp = from_arg_ty(bcx, llargs[1], tp_ty); let src = from_arg_ty(bcx, llargs[2], tp_ty); let res = AtomicCmpXchg(bcx, ptr, cmp, src, order, strongest_failure_ordering); ExtractValue(bcx, res, 0) } "load" => { let tp_ty = *substs.types.get(FnSpace, 0); let ptr = to_arg_ty_ptr(bcx, llargs[0], tp_ty); to_arg_ty(bcx, AtomicLoad(bcx, ptr, order), tp_ty) } "store" => { let tp_ty = *substs.types.get(FnSpace, 0); let ptr = to_arg_ty_ptr(bcx, llargs[0], tp_ty); let val = from_arg_ty(bcx, llargs[1], tp_ty); AtomicStore(bcx, val, ptr, order); C_nil(ccx) } "fence" => { AtomicFence(bcx, order, llvm::CrossThread); C_nil(ccx) } "singlethreadfence" => { AtomicFence(bcx, order, llvm::SingleThread); C_nil(ccx) } // These are all AtomicRMW ops op => { let atom_op = match op { "xchg" => llvm::AtomicXchg, "xadd" => llvm::AtomicAdd, "xsub" => llvm::AtomicSub, "and" => llvm::AtomicAnd, "nand" => llvm::AtomicNand, "or" => llvm::AtomicOr, "xor" => llvm::AtomicXor, "max" => llvm::AtomicMax, "min" => llvm::AtomicMin, "umax" => llvm::AtomicUMax, "umin" => llvm::AtomicUMin, _ => ccx.sess().fatal("unknown atomic operation") }; let tp_ty = *substs.types.get(FnSpace, 0); let ptr = to_arg_ty_ptr(bcx, llargs[0], tp_ty); let val = from_arg_ty(bcx, llargs[1], tp_ty); AtomicRMW(bcx, atom_op, ptr, val, order) } } } (_, _) => { let intr = match Intrinsic::find(tcx, &name) { Some(intr) => intr, None => ccx.sess().span_bug(foreign_item.span, "unknown intrinsic"), }; fn ty_to_type(ccx: &CrateContext, t: &intrinsics::Type) -> Type { use intrinsics::Type::*; match *t { Integer(x) => Type::ix(ccx, x as u64), Float(x) => { match x { 32 => Type::f32(ccx), 64 => Type::f64(ccx), _ => unreachable!() } } Pointer(_) => unimplemented!(), Vector(ref t, length) => Type::vector(&ty_to_type(ccx, t), length as u64) } } let inputs = intr.inputs.iter().map(|t| ty_to_type(ccx, t)).collect::>(); let outputs = ty_to_type(ccx, &intr.output); match intr.definition { intrinsics::IntrinsicDef::Named(name) => { let f = declare::declare_cfn(ccx, name, Type::func(&inputs, &outputs), tcx.mk_nil()); Call(bcx, f, &llargs, None, call_debug_location) } } } }; if val_ty(llval) != Type::void(ccx) && machine::llsize_of_alloc(ccx, val_ty(llval)) != 0 { store_ty(bcx, llval, llresult, ret_ty); } // If we made a temporary stack slot, let's clean it up match dest { expr::Ignore => { bcx = glue::drop_ty(bcx, llresult, ret_ty, call_debug_location); } expr::SaveIn(_) => {} } fcx.pop_and_trans_custom_cleanup_scope(bcx, cleanup_scope); Result::new(bcx, llresult) } fn copy_intrinsic<'blk, 'tcx>(bcx: Block<'blk, 'tcx>, allow_overlap: bool, volatile: bool, tp_ty: Ty<'tcx>, dst: ValueRef, src: ValueRef, count: ValueRef, call_debug_location: DebugLoc) -> ValueRef { let ccx = bcx.ccx(); let lltp_ty = type_of::type_of(ccx, tp_ty); let align = C_i32(ccx, type_of::align_of(ccx, tp_ty) as i32); let size = machine::llsize_of(ccx, lltp_ty); let int_size = machine::llbitsize_of_real(ccx, ccx.int_type()); let operation = if allow_overlap { "memmove" } else { "memcpy" }; let name = format!("llvm.{}.p0i8.p0i8.i{}", operation, int_size); let dst_ptr = PointerCast(bcx, dst, Type::i8p(ccx)); let src_ptr = PointerCast(bcx, src, Type::i8p(ccx)); let llfn = ccx.get_intrinsic(&name); Call(bcx, llfn, &[dst_ptr, src_ptr, Mul(bcx, size, count, DebugLoc::None), align, C_bool(ccx, volatile)], None, call_debug_location) } fn memset_intrinsic<'blk, 'tcx>(bcx: Block<'blk, 'tcx>, volatile: bool, tp_ty: Ty<'tcx>, dst: ValueRef, val: ValueRef, count: ValueRef, call_debug_location: DebugLoc) -> ValueRef { let ccx = bcx.ccx(); let lltp_ty = type_of::type_of(ccx, tp_ty); let align = C_i32(ccx, type_of::align_of(ccx, tp_ty) as i32); let size = machine::llsize_of(ccx, lltp_ty); let int_size = machine::llbitsize_of_real(ccx, ccx.int_type()); let name = format!("llvm.memset.p0i8.i{}", int_size); let dst_ptr = PointerCast(bcx, dst, Type::i8p(ccx)); let llfn = ccx.get_intrinsic(&name); Call(bcx, llfn, &[dst_ptr, val, Mul(bcx, size, count, DebugLoc::None), align, C_bool(ccx, volatile)], None, call_debug_location) } fn count_zeros_intrinsic(bcx: Block, name: &'static str, val: ValueRef, call_debug_location: DebugLoc) -> ValueRef { let y = C_bool(bcx.ccx(), false); let llfn = bcx.ccx().get_intrinsic(&name); Call(bcx, llfn, &[val, y], None, call_debug_location) } fn with_overflow_intrinsic<'blk, 'tcx>(bcx: Block<'blk, 'tcx>, name: &'static str, t: Ty<'tcx>, a: ValueRef, b: ValueRef, call_debug_location: DebugLoc) -> ValueRef { let llfn = bcx.ccx().get_intrinsic(&name); // Convert `i1` to a `bool`, and write it to the out parameter let val = Call(bcx, llfn, &[a, b], None, call_debug_location); let result = ExtractValue(bcx, val, 0); let overflow = ZExt(bcx, ExtractValue(bcx, val, 1), Type::bool(bcx.ccx())); let ret = C_undef(type_of::type_of(bcx.ccx(), t)); let ret = InsertValue(bcx, ret, result, 0); let ret = InsertValue(bcx, ret, overflow, 1); if !arg_is_indirect(bcx.ccx(), t) { let tmp = alloc_ty(bcx, t, "tmp"); Store(bcx, ret, tmp); load_ty(bcx, tmp, t) } else { ret } } fn try_intrinsic<'blk, 'tcx>(bcx: Block<'blk, 'tcx>, func: ValueRef, data: ValueRef, dest: ValueRef, dloc: DebugLoc) -> Block<'blk, 'tcx> { if bcx.sess().no_landing_pads() { Call(bcx, func, &[data], None, dloc); Store(bcx, C_null(Type::i8p(bcx.ccx())), dest); bcx } else if wants_msvc_seh(bcx.sess()) { trans_msvc_try(bcx, func, data, dest, dloc) } else { trans_gnu_try(bcx, func, data, dest, dloc) } } // MSVC's definition of the `rust_try` function. The exact implementation here // is a little different than the GNU (standard) version below, not only because // of the personality function but also because of the other fiddly bits about // SEH. LLVM also currently requires us to structure this a very particular way // as explained below. // // Like with the GNU version we generate a shim wrapper fn trans_msvc_try<'blk, 'tcx>(bcx: Block<'blk, 'tcx>, func: ValueRef, data: ValueRef, dest: ValueRef, dloc: DebugLoc) -> Block<'blk, 'tcx> { let llfn = get_rust_try_fn(bcx.fcx, &mut |try_fn_ty, output| { let ccx = bcx.ccx(); let dloc = DebugLoc::None; let rust_try = declare::define_internal_rust_fn(ccx, "__rust_try", try_fn_ty); let (fcx, block_arena); block_arena = TypedArena::new(); fcx = new_fn_ctxt(ccx, rust_try, ast::DUMMY_NODE_ID, false, output, ccx.tcx().mk_substs(Substs::trans_empty()), None, &block_arena); let bcx = init_function(&fcx, true, output); let then = fcx.new_temp_block("then"); let catch = fcx.new_temp_block("catch"); let catch_return = fcx.new_temp_block("catch-return"); let catch_resume = fcx.new_temp_block("catch-resume"); let personality = fcx.eh_personality(); let eh_typeid_for = ccx.get_intrinsic(&"llvm.eh.typeid.for"); let rust_try_filter = match bcx.tcx().lang_items.msvc_try_filter() { Some(did) => callee::trans_fn_ref(ccx, did, ExprId(0), bcx.fcx.param_substs).val, None => bcx.sess().bug("msvc_try_filter not defined"), }; // Type indicator for the exception being thrown, not entirely sure // what's going on here but it's what all the examples in LLVM use. let lpad_ty = Type::struct_(ccx, &[Type::i8p(ccx), Type::i32(ccx)], false); llvm::SetFunctionAttribute(rust_try, llvm::Attribute::NoInline); llvm::SetFunctionAttribute(rust_try, llvm::Attribute::OptimizeNone); let func = llvm::get_param(rust_try, 0); let data = llvm::get_param(rust_try, 1); // Invoke the function, specifying our two temporary landing pads as the // ext point. After the invoke we've terminated our basic block. Invoke(bcx, func, &[data], then.llbb, catch.llbb, None, dloc); // All the magic happens in this landing pad, and this is basically the // only landing pad in rust tagged with "catch" to indicate that we're // catching an exception. The other catch handlers in the GNU version // below just catch *all* exceptions, but that's because most exceptions // are already filtered out by the gnu personality function. // // For MSVC we're just using a standard personality function that we // can't customize (e.g. _except_handler3 or __C_specific_handler), so // we need to do the exception filtering ourselves. This is currently // performed by the `__rust_try_filter` function. This function, // specified in the landingpad instruction, will be invoked by Windows // SEH routines and will return whether the exception in question can be // caught (aka the Rust runtime is the one that threw the exception). // // To get this to compile (currently LLVM segfaults if it's not in this // particular structure), when the landingpad is executing we test to // make sure that the ID of the exception being thrown is indeed the one // that we were expecting. If it's not, we resume the exception, and // otherwise we return the pointer that we got Full disclosure: It's not // clear to me what this `llvm.eh.typeid` stuff is doing *other* then // just allowing LLVM to compile this file without segfaulting. I would // expect the entire landing pad to just be: // // %vals = landingpad ... // %ehptr = extractvalue { i8*, i32 } %vals, 0 // ret i8* %ehptr // // but apparently LLVM chokes on this, so we do the more complicated // thing to placate it. let vals = LandingPad(catch, lpad_ty, personality, 1); let rust_try_filter = BitCast(catch, rust_try_filter, Type::i8p(ccx)); AddClause(catch, vals, rust_try_filter); let ehptr = ExtractValue(catch, vals, 0); let sel = ExtractValue(catch, vals, 1); let filter_sel = Call(catch, eh_typeid_for, &[rust_try_filter], None, dloc); let is_filter = ICmp(catch, llvm::IntEQ, sel, filter_sel, dloc); CondBr(catch, is_filter, catch_return.llbb, catch_resume.llbb, dloc); // Our "catch-return" basic block is where we've determined that we // actually need to catch this exception, in which case we just return // the exception pointer. Ret(catch_return, ehptr, dloc); // The "catch-resume" block is where we're running this landing pad but // we actually need to not catch the exception, so just resume the // exception to return. Resume(catch_resume, vals); // On the successful branch we just return null. Ret(then, C_null(Type::i8p(ccx)), dloc); return rust_try }); // Note that no invoke is used here because by definition this function // can't panic (that's what it's catching). let ret = Call(bcx, llfn, &[func, data], None, dloc); Store(bcx, ret, dest); return bcx; } // Definition of the standard "try" function for Rust using the GNU-like model // of exceptions (e.g. the normal semantics of LLVM's landingpad and invoke // instructions). // // This translation is a little surprising because // we always call a shim function instead of inlining the call to `invoke` // manually here. This is done because in LLVM we're only allowed to have one // personality per function definition. The call to the `try` intrinsic is // being inlined into the function calling it, and that function may already // have other personality functions in play. By calling a shim we're // guaranteed that our shim will have the right personality function. // fn trans_gnu_try<'blk, 'tcx>(bcx: Block<'blk, 'tcx>, func: ValueRef, data: ValueRef, dest: ValueRef, dloc: DebugLoc) -> Block<'blk, 'tcx> { let llfn = get_rust_try_fn(bcx.fcx, &mut |try_fn_ty, output| { let ccx = bcx.ccx(); let dloc = DebugLoc::None; // Translates the shims described above: // // bcx: // invoke %func(%args...) normal %normal unwind %catch // // normal: // ret null // // catch: // (ptr, _) = landingpad // ret ptr let rust_try = declare::define_internal_rust_fn(ccx, "__rust_try", try_fn_ty); attributes::emit_uwtable(rust_try, true); let catch_pers = match bcx.tcx().lang_items.eh_personality_catch() { Some(did) => callee::trans_fn_ref(ccx, did, ExprId(0), bcx.fcx.param_substs).val, None => bcx.tcx().sess.bug("eh_personality_catch not defined"), }; let (fcx, block_arena); block_arena = TypedArena::new(); fcx = new_fn_ctxt(ccx, rust_try, ast::DUMMY_NODE_ID, false, output, ccx.tcx().mk_substs(Substs::trans_empty()), None, &block_arena); let bcx = init_function(&fcx, true, output); let then = bcx.fcx.new_temp_block("then"); let catch = bcx.fcx.new_temp_block("catch"); let func = llvm::get_param(rust_try, 0); let data = llvm::get_param(rust_try, 1); Invoke(bcx, func, &[data], then.llbb, catch.llbb, None, dloc); Ret(then, C_null(Type::i8p(ccx)), dloc); // Type indicator for the exception being thrown. // The first value in this tuple is a pointer to the exception object being thrown. // The second value is a "selector" indicating which of the landing pad clauses // the exception's type had been matched to. rust_try ignores the selector. let lpad_ty = Type::struct_(ccx, &[Type::i8p(ccx), Type::i32(ccx)], false); let vals = LandingPad(catch, lpad_ty, catch_pers, 1); AddClause(catch, vals, C_null(Type::i8p(ccx))); let ptr = ExtractValue(catch, vals, 0); Ret(catch, ptr, dloc); fcx.cleanup(); return rust_try }); // Note that no invoke is used here because by definition this function // can't panic (that's what it's catching). let ret = Call(bcx, llfn, &[func, data], None, dloc); Store(bcx, ret, dest); return bcx; } // Helper to generate the `Ty` associated with `rust_try` fn get_rust_try_fn<'a, 'tcx>(fcx: &FunctionContext<'a, 'tcx>, f: &mut FnMut(Ty<'tcx>, ty::FnOutput<'tcx>) -> ValueRef) -> ValueRef { let ccx = fcx.ccx; if let Some(llfn) = *ccx.rust_try_fn().borrow() { return llfn } // Define the type up front for the signature of the rust_try function. let tcx = ccx.tcx(); let i8p = tcx.mk_mut_ptr(tcx.types.i8); let fn_ty = tcx.mk_bare_fn(ty::BareFnTy { unsafety: ast::Unsafety::Unsafe, abi: abi::Rust, sig: ty::Binder(ty::FnSig { inputs: vec![i8p], output: ty::FnOutput::FnConverging(tcx.mk_nil()), variadic: false, }), }); let fn_ty = tcx.mk_fn(None, fn_ty); let output = ty::FnOutput::FnConverging(i8p); let try_fn_ty = tcx.mk_bare_fn(ty::BareFnTy { unsafety: ast::Unsafety::Unsafe, abi: abi::Rust, sig: ty::Binder(ty::FnSig { inputs: vec![fn_ty, i8p], output: output, variadic: false, }), }); let rust_try = f(tcx.mk_fn(None, try_fn_ty), output); *ccx.rust_try_fn().borrow_mut() = Some(rust_try); return rust_try } fn generic_simd_intrinsic<'blk, 'tcx, 'a> (bcx: Block<'blk, 'tcx>, name: &str, substs: subst::Substs<'tcx>, callee_ty: Ty<'tcx>, args: Option<&[P]>, llargs: &[ValueRef], ret_ty: Ty<'tcx>, llret_ty: Type, call_debug_location: DebugLoc, call_info: NodeIdAndSpan) -> ValueRef { // macros for error handling: macro_rules! emit_error { ($msg: tt) => { emit_error!($msg, ) }; ($msg: tt, $($fmt: tt)*) => { bcx.sess().span_err(call_info.span, &format!(concat!("invalid monomorphization of `{}` intrinsic: ", $msg), name, $($fmt)*)); } } macro_rules! require { ($cond: expr, $($fmt: tt)*) => { if !$cond { emit_error!($($fmt)*); return C_null(llret_ty) } } } macro_rules! require_simd { ($ty: expr, $position: expr) => { require!($ty.is_simd(), "expected SIMD {} type, found non-SIMD `{}`", $position, $ty) } } let tcx = bcx.tcx(); let arg_tys = match callee_ty.sty { ty::TyBareFn(_, ref f) => { bcx.tcx().erase_late_bound_regions(&f.sig.inputs()) } _ => unreachable!() }; // every intrinsic takes a SIMD vector as its first argument require_simd!(arg_tys[0], "input"); let in_ty = arg_tys[0]; let in_elem = arg_tys[0].simd_type(tcx); let in_len = arg_tys[0].simd_size(tcx); let comparison = match name { "simd_eq" => Some(ast::BiEq), "simd_ne" => Some(ast::BiNe), "simd_lt" => Some(ast::BiLt), "simd_le" => Some(ast::BiLe), "simd_gt" => Some(ast::BiGt), "simd_ge" => Some(ast::BiGe), _ => None }; if let Some(cmp_op) = comparison { require_simd!(ret_ty, "return"); let out_len = ret_ty.simd_size(tcx); require!(in_len == out_len, "expected return type with length {} (same as input type `{}`), \ found `{}` with length {}", in_len, in_ty, ret_ty, out_len); require!(llret_ty.element_type().kind() == llvm::Integer, "expected return type with integer elements, found `{}` with non-integer `{}`", ret_ty, ret_ty.simd_type(tcx)); return compare_simd_types(bcx, llargs[0], llargs[1], in_elem, llret_ty, cmp_op, call_debug_location) } if name.starts_with("simd_shuffle") { let n: usize = match name["simd_shuffle".len()..].parse() { Ok(n) => n, Err(_) => tcx.sess.span_bug(call_info.span, "bad `simd_shuffle` instruction only caught in trans?") }; require_simd!(ret_ty, "return"); let out_len = ret_ty.simd_size(tcx); require!(out_len == n, "expected return type of length {}, found `{}` with length {}", n, ret_ty, out_len); require!(in_elem == ret_ty.simd_type(tcx), "expected return element type `{}` (element of input `{}`), \ found `{}` with element type `{}`", in_elem, in_ty, ret_ty, ret_ty.simd_type(tcx)); let total_len = in_len as u64 * 2; let vector = match args { Some(args) => &args[2], None => bcx.sess().span_bug(call_info.span, "intrinsic call with unexpected argument shape"), }; let vector = consts::const_expr(bcx.ccx(), vector, tcx.mk_substs(substs), None).0; let indices: Option> = (0..n) .map(|i| { let arg_idx = i; let val = const_get_elt(bcx.ccx(), vector, &[i as libc::c_uint]); let c = const_to_opt_uint(val); match c { None => { emit_error!("shuffle index #{} is not a constant", arg_idx); None } Some(idx) if idx >= total_len => { emit_error!("shuffle index #{} is out of bounds (limit {})", arg_idx, total_len); None } Some(idx) => Some(C_i32(bcx.ccx(), idx as i32)), } }) .collect(); let indices = match indices { Some(i) => i, None => return C_null(llret_ty) }; return ShuffleVector(bcx, llargs[0], llargs[1], C_vector(&indices)) } if name == "simd_insert" { require!(in_elem == arg_tys[2], "expected inserted type `{}` (element of input `{}`), found `{}`", in_elem, in_ty, arg_tys[2]); return InsertElement(bcx, llargs[0], llargs[2], llargs[1]) } if name == "simd_extract" { require!(ret_ty == in_elem, "expected return type `{}` (element of input `{}`), found `{}`", in_elem, in_ty, ret_ty); return ExtractElement(bcx, llargs[0], llargs[1]) } if name == "simd_cast" { require_simd!(ret_ty, "return"); let out_len = ret_ty.simd_size(tcx); require!(in_len == out_len, "expected return type with length {} (same as input type `{}`), \ found `{}` with length {}", in_len, in_ty, ret_ty, out_len); // casting cares about nominal type, not just structural type let out_elem = ret_ty.simd_type(tcx); if in_elem == out_elem { return llargs[0]; } enum Style { Float, Int(/* is signed? */ bool), Unsupported } let (in_style, in_width) = match in_elem.sty { // vectors of pointer-sized integers should've been // disallowed before here, so this unwrap is safe. ty::TyInt(i) => (Style::Int(true), i.bit_width().unwrap()), ty::TyUint(u) => (Style::Int(false), u.bit_width().unwrap()), ty::TyFloat(f) => (Style::Float, f.bit_width()), _ => (Style::Unsupported, 0) }; let (out_style, out_width) = match out_elem.sty { ty::TyInt(i) => (Style::Int(true), i.bit_width().unwrap()), ty::TyUint(u) => (Style::Int(false), u.bit_width().unwrap()), ty::TyFloat(f) => (Style::Float, f.bit_width()), _ => (Style::Unsupported, 0) }; match (in_style, out_style) { (Style::Int(in_is_signed), Style::Int(_)) => { return match in_width.cmp(&out_width) { Ordering::Greater => Trunc(bcx, llargs[0], llret_ty), Ordering::Equal => llargs[0], Ordering::Less => if in_is_signed { SExt(bcx, llargs[0], llret_ty) } else { ZExt(bcx, llargs[0], llret_ty) } } } (Style::Int(in_is_signed), Style::Float) => { return if in_is_signed { SIToFP(bcx, llargs[0], llret_ty) } else { UIToFP(bcx, llargs[0], llret_ty) } } (Style::Float, Style::Int(out_is_signed)) => { return if out_is_signed { FPToSI(bcx, llargs[0], llret_ty) } else { FPToUI(bcx, llargs[0], llret_ty) } } (Style::Float, Style::Float) => { return match in_width.cmp(&out_width) { Ordering::Greater => FPTrunc(bcx, llargs[0], llret_ty), Ordering::Equal => llargs[0], Ordering::Less => FPExt(bcx, llargs[0], llret_ty) } } _ => {/* Unsupported. Fallthrough. */} } require!(false, "unsupported cast from `{}` with element `{}` to `{}` with element `{}`", in_ty, in_elem, ret_ty, out_elem); } macro_rules! arith { ($($name: ident: $($($p: ident),* => $call: expr),*;)*) => { $( if name == stringify!($name) { match in_elem.sty { $( $(ty::$p(_))|* => { return $call(bcx, llargs[0], llargs[1], call_debug_location) } )* _ => {}, } require!(false, "unsupported operation on `{}` with element `{}`", in_ty, in_elem) })* } } arith! { simd_add: TyUint, TyInt => Add, TyFloat => FAdd; simd_sub: TyUint, TyInt => Sub, TyFloat => FSub; simd_mul: TyUint, TyInt => Mul, TyFloat => FMul; simd_div: TyFloat => FDiv; simd_shl: TyUint, TyInt => Shl; simd_shr: TyUint => LShr, TyInt => AShr; simd_and: TyUint, TyInt => And; simd_or: TyUint, TyInt => Or; simd_xor: TyUint, TyInt => Xor; } bcx.sess().span_bug(call_info.span, "unknown SIMD intrinsic"); }