Forbid old-style simd_shuffleN intrinsics
This commit is contained in:
Oli Scherer
parent
3cde1943af
commit
de69c67ffa
@ -1,11 +1,11 @@
|
||||
#[cfg(feature="master")]
|
||||
use gccjit::{ComparisonOp, UnaryOp};
|
||||
use gccjit::ToRValue;
|
||||
use gccjit::{BinaryOp, RValue, Type};
|
||||
#[cfg(feature = "master")]
|
||||
use gccjit::{ComparisonOp, UnaryOp};
|
||||
|
||||
use rustc_codegen_ssa::base::compare_simd_types;
|
||||
use rustc_codegen_ssa::common::{IntPredicate, TypeKind};
|
||||
#[cfg(feature="master")]
|
||||
#[cfg(feature = "master")]
|
||||
use rustc_codegen_ssa::errors::ExpectedPointerMutability;
|
||||
use rustc_codegen_ssa::errors::InvalidMonomorphization;
|
||||
use rustc_codegen_ssa::mir::operand::OperandRef;
|
||||
@ -19,7 +19,7 @@ use rustc_span::{sym, Span, Symbol};
|
||||
use rustc_target::abi::Align;
|
||||
|
||||
use crate::builder::Builder;
|
||||
#[cfg(feature="master")]
|
||||
#[cfg(feature = "master")]
|
||||
use crate::context::CodegenCx;
|
||||
|
||||
pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
|
||||
@ -57,7 +57,10 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
|
||||
let arg_tys = sig.inputs();
|
||||
|
||||
if name == sym::simd_select_bitmask {
|
||||
require_simd!(arg_tys[1], InvalidMonomorphization::SimdArgument { span, name, ty: arg_tys[1] });
|
||||
require_simd!(
|
||||
arg_tys[1],
|
||||
InvalidMonomorphization::SimdArgument { span, name, ty: arg_tys[1] }
|
||||
);
|
||||
let (len, _) = arg_tys[1].simd_size_and_type(bx.tcx());
|
||||
|
||||
let expected_int_bits = (len.max(8) - 1).next_power_of_two();
|
||||
@ -95,7 +98,8 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
|
||||
// NOTE: since the arguments can be vectors of floats, make sure the mask is a vector of
|
||||
// integer.
|
||||
let mask_element_type = bx.type_ix(arg1_element_type.get_size() as u64 * 8);
|
||||
let vector_mask_type = bx.context.new_vector_type(mask_element_type, arg1_vector_type.get_num_units() as u64);
|
||||
let vector_mask_type =
|
||||
bx.context.new_vector_type(mask_element_type, arg1_vector_type.get_num_units() as u64);
|
||||
|
||||
let mut elements = vec![];
|
||||
let one = bx.context.new_rvalue_one(mask.get_type());
|
||||
@ -149,38 +153,24 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
|
||||
// compare them as equal, so bitcast.
|
||||
// FIXME(antoyo): allow comparing vector types as equal in libgccjit.
|
||||
let arg2 = bx.context.new_bitcast(None, args[1].immediate(), arg1.get_type());
|
||||
return Ok(compare_simd_types(
|
||||
bx,
|
||||
arg1,
|
||||
arg2,
|
||||
in_elem,
|
||||
llret_ty,
|
||||
cmp_op,
|
||||
));
|
||||
return Ok(compare_simd_types(bx, arg1, arg2, in_elem, llret_ty, cmp_op));
|
||||
}
|
||||
|
||||
if let Some(stripped) = name.as_str().strip_prefix("simd_shuffle") {
|
||||
let n: u64 = if stripped.is_empty() {
|
||||
// Make sure this is actually an array, since typeck only checks the length-suffixed
|
||||
// version of this intrinsic.
|
||||
match args[2].layout.ty.kind() {
|
||||
ty::Array(ty, len) if matches!(ty.kind(), ty::Uint(ty::UintTy::U32)) => {
|
||||
len.try_eval_target_usize(bx.cx.tcx, ty::ParamEnv::reveal_all()).unwrap_or_else(
|
||||
|| span_bug!(span, "could not evaluate shuffle index array length"),
|
||||
)
|
||||
}
|
||||
_ => return_error!(InvalidMonomorphization::SimdShuffle {
|
||||
span,
|
||||
name,
|
||||
ty: args[2].layout.ty
|
||||
}),
|
||||
if name == sym::simd_shuffle {
|
||||
// Make sure this is actually an array, since typeck only checks the length-suffixed
|
||||
// version of this intrinsic.
|
||||
let n: u64 = match args[2].layout.ty.kind() {
|
||||
ty::Array(ty, len) if matches!(ty.kind(), ty::Uint(ty::UintTy::U32)) => {
|
||||
len.try_eval_target_usize(bx.cx.tcx, ty::ParamEnv::reveal_all()).unwrap_or_else(
|
||||
|| span_bug!(span, "could not evaluate shuffle index array length"),
|
||||
)
|
||||
}
|
||||
} else {
|
||||
stripped.parse().unwrap_or_else(|_| {
|
||||
span_bug!(span, "bad `simd_shuffle` instruction only caught in codegen?")
|
||||
})
|
||||
_ => return_error!(InvalidMonomorphization::SimdShuffle {
|
||||
span,
|
||||
name,
|
||||
ty: args[2].layout.ty
|
||||
}),
|
||||
};
|
||||
|
||||
require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty });
|
||||
|
||||
let (out_len, out_ty) = ret_ty.simd_size_and_type(bx.tcx());
|
||||
@ -202,7 +192,13 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
|
||||
if name == sym::simd_insert {
|
||||
require!(
|
||||
in_elem == arg_tys[2],
|
||||
InvalidMonomorphization::InsertedType { span, name, in_elem, in_ty, out_ty: arg_tys[2] }
|
||||
InvalidMonomorphization::InsertedType {
|
||||
span,
|
||||
name,
|
||||
in_elem,
|
||||
in_ty,
|
||||
out_ty: arg_tys[2]
|
||||
}
|
||||
);
|
||||
let vector = args[0].immediate();
|
||||
let index = args[1].immediate();
|
||||
@ -228,7 +224,10 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
|
||||
if name == sym::simd_select {
|
||||
let m_elem_ty = in_elem;
|
||||
let m_len = in_len;
|
||||
require_simd!(arg_tys[1], InvalidMonomorphization::SimdArgument { span, name, ty: arg_tys[1] });
|
||||
require_simd!(
|
||||
arg_tys[1],
|
||||
InvalidMonomorphization::SimdArgument { span, name, ty: arg_tys[1] }
|
||||
);
|
||||
let (v_len, _) = arg_tys[1].simd_size_and_type(bx.tcx());
|
||||
require!(
|
||||
m_len == v_len,
|
||||
@ -241,7 +240,7 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
|
||||
return Ok(bx.vector_select(args[0].immediate(), args[1].immediate(), args[2].immediate()));
|
||||
}
|
||||
|
||||
#[cfg(feature="master")]
|
||||
#[cfg(feature = "master")]
|
||||
if name == sym::simd_cast || name == sym::simd_as {
|
||||
require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty });
|
||||
let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx());
|
||||
@ -267,19 +266,17 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
|
||||
Unsupported,
|
||||
}
|
||||
|
||||
let in_style =
|
||||
match in_elem.kind() {
|
||||
ty::Int(_) | ty::Uint(_) => Style::Int,
|
||||
ty::Float(_) => Style::Float,
|
||||
_ => Style::Unsupported,
|
||||
};
|
||||
let in_style = match in_elem.kind() {
|
||||
ty::Int(_) | ty::Uint(_) => Style::Int,
|
||||
ty::Float(_) => Style::Float,
|
||||
_ => Style::Unsupported,
|
||||
};
|
||||
|
||||
let out_style =
|
||||
match out_elem.kind() {
|
||||
ty::Int(_) | ty::Uint(_) => Style::Int,
|
||||
ty::Float(_) => Style::Float,
|
||||
_ => Style::Unsupported,
|
||||
};
|
||||
let out_style = match out_elem.kind() {
|
||||
ty::Int(_) | ty::Uint(_) => Style::Int,
|
||||
ty::Float(_) => Style::Float,
|
||||
_ => Style::Unsupported,
|
||||
};
|
||||
|
||||
match (in_style, out_style) {
|
||||
(Style::Unsupported, Style::Unsupported) => {
|
||||
@ -294,7 +291,7 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
|
||||
out_elem
|
||||
}
|
||||
);
|
||||
},
|
||||
}
|
||||
_ => return Ok(bx.context.convert_vector(None, args[0].immediate(), llret_ty)),
|
||||
}
|
||||
}
|
||||
@ -342,10 +339,13 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
|
||||
|
||||
let mut shift = 0;
|
||||
for i in 0..in_len {
|
||||
let elem = bx.extract_element(vector, bx.context.new_rvalue_from_int(bx.int_type, i as i32));
|
||||
let elem =
|
||||
bx.extract_element(vector, bx.context.new_rvalue_from_int(bx.int_type, i as i32));
|
||||
let shifted = elem >> sign_shift;
|
||||
let masked = shifted & one;
|
||||
result = result | (bx.context.new_cast(None, masked, result_type) << bx.context.new_rvalue_from_int(result_type, shift));
|
||||
result = result
|
||||
| (bx.context.new_cast(None, masked, result_type)
|
||||
<< bx.context.new_rvalue_from_int(result_type, shift));
|
||||
shift += 1;
|
||||
}
|
||||
|
||||
@ -394,46 +394,50 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
|
||||
return Err(());
|
||||
}};
|
||||
}
|
||||
let (elem_ty_str, elem_ty) =
|
||||
if let ty::Float(f) = in_elem.kind() {
|
||||
let elem_ty = bx.cx.type_float_from_ty(*f);
|
||||
match f.bit_width() {
|
||||
32 => ("f", elem_ty),
|
||||
64 => ("", elem_ty),
|
||||
_ => {
|
||||
return_error!(InvalidMonomorphization::FloatingPointVector { span, name, f_ty: *f, in_ty });
|
||||
}
|
||||
let (elem_ty_str, elem_ty) = if let ty::Float(f) = in_elem.kind() {
|
||||
let elem_ty = bx.cx.type_float_from_ty(*f);
|
||||
match f.bit_width() {
|
||||
32 => ("f", elem_ty),
|
||||
64 => ("", elem_ty),
|
||||
_ => {
|
||||
return_error!(InvalidMonomorphization::FloatingPointVector {
|
||||
span,
|
||||
name,
|
||||
f_ty: *f,
|
||||
in_ty
|
||||
});
|
||||
}
|
||||
}
|
||||
else {
|
||||
return_error!(InvalidMonomorphization::FloatingPointType { span, name, in_ty });
|
||||
};
|
||||
} else {
|
||||
return_error!(InvalidMonomorphization::FloatingPointType { span, name, in_ty });
|
||||
};
|
||||
|
||||
let vec_ty = bx.cx.type_vector(elem_ty, in_len);
|
||||
|
||||
let intr_name =
|
||||
match name {
|
||||
sym::simd_ceil => "ceil",
|
||||
sym::simd_fabs => "fabs", // TODO(antoyo): pand with 170141183420855150465331762880109871103
|
||||
sym::simd_fcos => "cos",
|
||||
sym::simd_fexp2 => "exp2",
|
||||
sym::simd_fexp => "exp",
|
||||
sym::simd_flog10 => "log10",
|
||||
sym::simd_flog2 => "log2",
|
||||
sym::simd_flog => "log",
|
||||
sym::simd_floor => "floor",
|
||||
sym::simd_fma => "fma",
|
||||
sym::simd_fpowi => "__builtin_powi",
|
||||
sym::simd_fpow => "pow",
|
||||
sym::simd_fsin => "sin",
|
||||
sym::simd_fsqrt => "sqrt",
|
||||
sym::simd_round => "round",
|
||||
sym::simd_trunc => "trunc",
|
||||
_ => return_error!(InvalidMonomorphization::UnrecognizedIntrinsic { span, name })
|
||||
};
|
||||
let intr_name = match name {
|
||||
sym::simd_ceil => "ceil",
|
||||
sym::simd_fabs => "fabs", // TODO(antoyo): pand with 170141183420855150465331762880109871103
|
||||
sym::simd_fcos => "cos",
|
||||
sym::simd_fexp2 => "exp2",
|
||||
sym::simd_fexp => "exp",
|
||||
sym::simd_flog10 => "log10",
|
||||
sym::simd_flog2 => "log2",
|
||||
sym::simd_flog => "log",
|
||||
sym::simd_floor => "floor",
|
||||
sym::simd_fma => "fma",
|
||||
sym::simd_fpowi => "__builtin_powi",
|
||||
sym::simd_fpow => "pow",
|
||||
sym::simd_fsin => "sin",
|
||||
sym::simd_fsqrt => "sqrt",
|
||||
sym::simd_round => "round",
|
||||
sym::simd_trunc => "trunc",
|
||||
_ => return_error!(InvalidMonomorphization::UnrecognizedIntrinsic { span, name }),
|
||||
};
|
||||
let builtin_name = format!("{}{}", intr_name, elem_ty_str);
|
||||
let funcs = bx.cx.functions.borrow();
|
||||
let function = funcs.get(&builtin_name).unwrap_or_else(|| panic!("unable to find builtin function {}", builtin_name));
|
||||
let function = funcs
|
||||
.get(&builtin_name)
|
||||
.unwrap_or_else(|| panic!("unable to find builtin function {}", builtin_name));
|
||||
|
||||
// TODO(antoyo): add platform-specific behavior here for architectures that have these
|
||||
// intrinsics as instructions (for instance, gpus)
|
||||
@ -479,8 +483,12 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
|
||||
return simd_simple_float_intrinsic(name, in_elem, in_ty, in_len, bx, span, args);
|
||||
}
|
||||
|
||||
#[cfg(feature="master")]
|
||||
fn vector_ty<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, elem_ty: Ty<'tcx>, vec_len: u64) -> Type<'gcc> {
|
||||
#[cfg(feature = "master")]
|
||||
fn vector_ty<'gcc, 'tcx>(
|
||||
cx: &CodegenCx<'gcc, 'tcx>,
|
||||
elem_ty: Ty<'tcx>,
|
||||
vec_len: u64,
|
||||
) -> Type<'gcc> {
|
||||
// FIXME: use cx.layout_of(ty).llvm_type() ?
|
||||
let elem_ty = match *elem_ty.kind() {
|
||||
ty::Int(v) => cx.type_int_from_ty(v),
|
||||
@ -491,15 +499,22 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
|
||||
cx.type_vector(elem_ty, vec_len)
|
||||
}
|
||||
|
||||
#[cfg(feature="master")]
|
||||
fn gather<'a, 'gcc, 'tcx>(default: RValue<'gcc>, pointers: RValue<'gcc>, mask: RValue<'gcc>, pointer_count: usize, bx: &mut Builder<'a, 'gcc, 'tcx>, in_len: u64, underlying_ty: Ty<'tcx>, invert: bool) -> RValue<'gcc> {
|
||||
let vector_type =
|
||||
if pointer_count > 1 {
|
||||
bx.context.new_vector_type(bx.usize_type, in_len)
|
||||
}
|
||||
else {
|
||||
vector_ty(bx, underlying_ty, in_len)
|
||||
};
|
||||
#[cfg(feature = "master")]
|
||||
fn gather<'a, 'gcc, 'tcx>(
|
||||
default: RValue<'gcc>,
|
||||
pointers: RValue<'gcc>,
|
||||
mask: RValue<'gcc>,
|
||||
pointer_count: usize,
|
||||
bx: &mut Builder<'a, 'gcc, 'tcx>,
|
||||
in_len: u64,
|
||||
underlying_ty: Ty<'tcx>,
|
||||
invert: bool,
|
||||
) -> RValue<'gcc> {
|
||||
let vector_type = if pointer_count > 1 {
|
||||
bx.context.new_vector_type(bx.usize_type, in_len)
|
||||
} else {
|
||||
vector_ty(bx, underlying_ty, in_len)
|
||||
};
|
||||
let elem_type = vector_type.dyncast_vector().expect("vector type").get_element_type();
|
||||
|
||||
let mut values = vec![];
|
||||
@ -530,13 +545,12 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
|
||||
|
||||
if invert {
|
||||
bx.shuffle_vector(vector, default, mask)
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
bx.shuffle_vector(default, vector, mask)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature="master")]
|
||||
#[cfg(feature = "master")]
|
||||
if name == sym::simd_gather {
|
||||
// simd_gather(values: <N x T>, pointers: <N x *_ T>,
|
||||
// mask: <N x i{M}>) -> <N x T>
|
||||
@ -546,8 +560,14 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
|
||||
|
||||
// All types must be simd vector types
|
||||
require_simd!(in_ty, InvalidMonomorphization::SimdFirst { span, name, ty: in_ty });
|
||||
require_simd!(arg_tys[1], InvalidMonomorphization::SimdSecond { span, name, ty: arg_tys[1] });
|
||||
require_simd!(arg_tys[2], InvalidMonomorphization::SimdThird { span, name, ty: arg_tys[2] });
|
||||
require_simd!(
|
||||
arg_tys[1],
|
||||
InvalidMonomorphization::SimdSecond { span, name, ty: arg_tys[1] }
|
||||
);
|
||||
require_simd!(
|
||||
arg_tys[2],
|
||||
InvalidMonomorphization::SimdThird { span, name, ty: arg_tys[2] }
|
||||
);
|
||||
require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty });
|
||||
|
||||
// Of the same length:
|
||||
@ -641,10 +661,19 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
|
||||
}
|
||||
}
|
||||
|
||||
return Ok(gather(args[0].immediate(), args[1].immediate(), args[2].immediate(), pointer_count, bx, in_len, underlying_ty, false));
|
||||
return Ok(gather(
|
||||
args[0].immediate(),
|
||||
args[1].immediate(),
|
||||
args[2].immediate(),
|
||||
pointer_count,
|
||||
bx,
|
||||
in_len,
|
||||
underlying_ty,
|
||||
false,
|
||||
));
|
||||
}
|
||||
|
||||
#[cfg(feature="master")]
|
||||
#[cfg(feature = "master")]
|
||||
if name == sym::simd_scatter {
|
||||
// simd_scatter(values: <N x T>, pointers: <N x *mut T>,
|
||||
// mask: <N x i{M}>) -> ()
|
||||
@ -654,8 +683,14 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
|
||||
|
||||
// All types must be simd vector types
|
||||
require_simd!(in_ty, InvalidMonomorphization::SimdFirst { span, name, ty: in_ty });
|
||||
require_simd!(arg_tys[1], InvalidMonomorphization::SimdSecond { span, name, ty: arg_tys[1] });
|
||||
require_simd!(arg_tys[2], InvalidMonomorphization::SimdThird { span, name, ty: arg_tys[2] });
|
||||
require_simd!(
|
||||
arg_tys[1],
|
||||
InvalidMonomorphization::SimdSecond { span, name, ty: arg_tys[1] }
|
||||
);
|
||||
require_simd!(
|
||||
arg_tys[2],
|
||||
InvalidMonomorphization::SimdThird { span, name, ty: arg_tys[2] }
|
||||
);
|
||||
|
||||
// Of the same length:
|
||||
let (element_len1, _) = arg_tys[1].simd_size_and_type(bx.tcx());
|
||||
@ -744,17 +779,24 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
|
||||
}
|
||||
}
|
||||
|
||||
let result = gather(args[0].immediate(), args[1].immediate(), args[2].immediate(), pointer_count, bx, in_len, underlying_ty, true);
|
||||
let result = gather(
|
||||
args[0].immediate(),
|
||||
args[1].immediate(),
|
||||
args[2].immediate(),
|
||||
pointer_count,
|
||||
bx,
|
||||
in_len,
|
||||
underlying_ty,
|
||||
true,
|
||||
);
|
||||
|
||||
let pointers = args[1].immediate();
|
||||
|
||||
let vector_type =
|
||||
if pointer_count > 1 {
|
||||
bx.context.new_vector_type(bx.usize_type, in_len)
|
||||
}
|
||||
else {
|
||||
vector_ty(bx, underlying_ty, in_len)
|
||||
};
|
||||
let vector_type = if pointer_count > 1 {
|
||||
bx.context.new_vector_type(bx.usize_type, in_len)
|
||||
} else {
|
||||
vector_ty(bx, underlying_ty, in_len)
|
||||
};
|
||||
let elem_type = vector_type.dyncast_vector().expect("vector type").get_element_type();
|
||||
|
||||
for i in 0..in_len {
|
||||
@ -809,11 +851,12 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
|
||||
let rhs = args[1].immediate();
|
||||
let is_add = name == sym::simd_saturating_add;
|
||||
let ptr_bits = bx.tcx().data_layout.pointer_size.bits() as _;
|
||||
let (signed, elem_width, elem_ty) =
|
||||
match *in_elem.kind() {
|
||||
ty::Int(i) => (true, i.bit_width().unwrap_or(ptr_bits) / 8, bx.cx.type_int_from_ty(i)),
|
||||
ty::Uint(i) => (false, i.bit_width().unwrap_or(ptr_bits) / 8, bx.cx.type_uint_from_ty(i)),
|
||||
_ => {
|
||||
let (signed, elem_width, elem_ty) = match *in_elem.kind() {
|
||||
ty::Int(i) => (true, i.bit_width().unwrap_or(ptr_bits) / 8, bx.cx.type_int_from_ty(i)),
|
||||
ty::Uint(i) => {
|
||||
(false, i.bit_width().unwrap_or(ptr_bits) / 8, bx.cx.type_uint_from_ty(i))
|
||||
}
|
||||
_ => {
|
||||
return_error!(InvalidMonomorphization::ExpectedVectorElementType {
|
||||
span,
|
||||
name,
|
||||
@ -823,77 +866,82 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
|
||||
}
|
||||
};
|
||||
|
||||
let result =
|
||||
match (signed, is_add) {
|
||||
(false, true) => {
|
||||
let res = lhs + rhs;
|
||||
let cmp = bx.context.new_comparison(None, ComparisonOp::LessThan, res, lhs);
|
||||
res | cmp
|
||||
},
|
||||
(true, true) => {
|
||||
// Algorithm from: https://codereview.stackexchange.com/questions/115869/saturated-signed-addition
|
||||
// TODO(antoyo): improve using conditional operators if possible.
|
||||
// TODO(antoyo): dyncast_vector should not require a call to unqualified.
|
||||
let arg_type = lhs.get_type().unqualified();
|
||||
// TODO(antoyo): convert lhs and rhs to unsigned.
|
||||
let sum = lhs + rhs;
|
||||
let vector_type = arg_type.dyncast_vector().expect("vector type");
|
||||
let unit = vector_type.get_num_units();
|
||||
let a = bx.context.new_rvalue_from_int(elem_ty, ((elem_width as i32) << 3) - 1);
|
||||
let width = bx.context.new_rvalue_from_vector(None, lhs.get_type(), &vec![a; unit]);
|
||||
let result = match (signed, is_add) {
|
||||
(false, true) => {
|
||||
let res = lhs + rhs;
|
||||
let cmp = bx.context.new_comparison(None, ComparisonOp::LessThan, res, lhs);
|
||||
res | cmp
|
||||
}
|
||||
(true, true) => {
|
||||
// Algorithm from: https://codereview.stackexchange.com/questions/115869/saturated-signed-addition
|
||||
// TODO(antoyo): improve using conditional operators if possible.
|
||||
// TODO(antoyo): dyncast_vector should not require a call to unqualified.
|
||||
let arg_type = lhs.get_type().unqualified();
|
||||
// TODO(antoyo): convert lhs and rhs to unsigned.
|
||||
let sum = lhs + rhs;
|
||||
let vector_type = arg_type.dyncast_vector().expect("vector type");
|
||||
let unit = vector_type.get_num_units();
|
||||
let a = bx.context.new_rvalue_from_int(elem_ty, ((elem_width as i32) << 3) - 1);
|
||||
let width = bx.context.new_rvalue_from_vector(None, lhs.get_type(), &vec![a; unit]);
|
||||
|
||||
let xor1 = lhs ^ rhs;
|
||||
let xor2 = lhs ^ sum;
|
||||
let and = bx.context.new_unary_op(None, UnaryOp::BitwiseNegate, arg_type, xor1) & xor2;
|
||||
let mask = and >> width;
|
||||
let xor1 = lhs ^ rhs;
|
||||
let xor2 = lhs ^ sum;
|
||||
let and =
|
||||
bx.context.new_unary_op(None, UnaryOp::BitwiseNegate, arg_type, xor1) & xor2;
|
||||
let mask = and >> width;
|
||||
|
||||
let one = bx.context.new_rvalue_one(elem_ty);
|
||||
let ones = bx.context.new_rvalue_from_vector(None, lhs.get_type(), &vec![one; unit]);
|
||||
let shift1 = ones << width;
|
||||
let shift2 = sum >> width;
|
||||
let mask_min = shift1 ^ shift2;
|
||||
let one = bx.context.new_rvalue_one(elem_ty);
|
||||
let ones =
|
||||
bx.context.new_rvalue_from_vector(None, lhs.get_type(), &vec![one; unit]);
|
||||
let shift1 = ones << width;
|
||||
let shift2 = sum >> width;
|
||||
let mask_min = shift1 ^ shift2;
|
||||
|
||||
let and1 = bx.context.new_unary_op(None, UnaryOp::BitwiseNegate, arg_type, mask) & sum;
|
||||
let and2 = mask & mask_min;
|
||||
let and1 =
|
||||
bx.context.new_unary_op(None, UnaryOp::BitwiseNegate, arg_type, mask) & sum;
|
||||
let and2 = mask & mask_min;
|
||||
|
||||
and1 + and2
|
||||
},
|
||||
(false, false) => {
|
||||
let res = lhs - rhs;
|
||||
let cmp = bx.context.new_comparison(None, ComparisonOp::LessThanEquals, res, lhs);
|
||||
res & cmp
|
||||
},
|
||||
(true, false) => {
|
||||
// TODO(antoyo): dyncast_vector should not require a call to unqualified.
|
||||
let arg_type = lhs.get_type().unqualified();
|
||||
// TODO(antoyo): this uses the same algorithm from saturating add, but add the
|
||||
// negative of the right operand. Find a proper subtraction algorithm.
|
||||
let rhs = bx.context.new_unary_op(None, UnaryOp::Minus, arg_type, rhs);
|
||||
and1 + and2
|
||||
}
|
||||
(false, false) => {
|
||||
let res = lhs - rhs;
|
||||
let cmp = bx.context.new_comparison(None, ComparisonOp::LessThanEquals, res, lhs);
|
||||
res & cmp
|
||||
}
|
||||
(true, false) => {
|
||||
// TODO(antoyo): dyncast_vector should not require a call to unqualified.
|
||||
let arg_type = lhs.get_type().unqualified();
|
||||
// TODO(antoyo): this uses the same algorithm from saturating add, but add the
|
||||
// negative of the right operand. Find a proper subtraction algorithm.
|
||||
let rhs = bx.context.new_unary_op(None, UnaryOp::Minus, arg_type, rhs);
|
||||
|
||||
// TODO(antoyo): convert lhs and rhs to unsigned.
|
||||
let sum = lhs + rhs;
|
||||
let vector_type = arg_type.dyncast_vector().expect("vector type");
|
||||
let unit = vector_type.get_num_units();
|
||||
let a = bx.context.new_rvalue_from_int(elem_ty, ((elem_width as i32) << 3) - 1);
|
||||
let width = bx.context.new_rvalue_from_vector(None, lhs.get_type(), &vec![a; unit]);
|
||||
// TODO(antoyo): convert lhs and rhs to unsigned.
|
||||
let sum = lhs + rhs;
|
||||
let vector_type = arg_type.dyncast_vector().expect("vector type");
|
||||
let unit = vector_type.get_num_units();
|
||||
let a = bx.context.new_rvalue_from_int(elem_ty, ((elem_width as i32) << 3) - 1);
|
||||
let width = bx.context.new_rvalue_from_vector(None, lhs.get_type(), &vec![a; unit]);
|
||||
|
||||
let xor1 = lhs ^ rhs;
|
||||
let xor2 = lhs ^ sum;
|
||||
let and = bx.context.new_unary_op(None, UnaryOp::BitwiseNegate, arg_type, xor1) & xor2;
|
||||
let mask = and >> width;
|
||||
let xor1 = lhs ^ rhs;
|
||||
let xor2 = lhs ^ sum;
|
||||
let and =
|
||||
bx.context.new_unary_op(None, UnaryOp::BitwiseNegate, arg_type, xor1) & xor2;
|
||||
let mask = and >> width;
|
||||
|
||||
let one = bx.context.new_rvalue_one(elem_ty);
|
||||
let ones = bx.context.new_rvalue_from_vector(None, lhs.get_type(), &vec![one; unit]);
|
||||
let shift1 = ones << width;
|
||||
let shift2 = sum >> width;
|
||||
let mask_min = shift1 ^ shift2;
|
||||
let one = bx.context.new_rvalue_one(elem_ty);
|
||||
let ones =
|
||||
bx.context.new_rvalue_from_vector(None, lhs.get_type(), &vec![one; unit]);
|
||||
let shift1 = ones << width;
|
||||
let shift2 = sum >> width;
|
||||
let mask_min = shift1 ^ shift2;
|
||||
|
||||
let and1 = bx.context.new_unary_op(None, UnaryOp::BitwiseNegate, arg_type, mask) & sum;
|
||||
let and2 = mask & mask_min;
|
||||
let and1 =
|
||||
bx.context.new_unary_op(None, UnaryOp::BitwiseNegate, arg_type, mask) & sum;
|
||||
let and2 = mask & mask_min;
|
||||
|
||||
and1 + and2
|
||||
}
|
||||
};
|
||||
and1 + and2
|
||||
}
|
||||
};
|
||||
|
||||
return Ok(result);
|
||||
}
|
||||
@ -968,7 +1016,6 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
|
||||
1.0
|
||||
);
|
||||
|
||||
|
||||
macro_rules! minmax_red {
|
||||
($name:ident: $int_red:ident, $float_red:ident) => {
|
||||
if name == sym::$name {
|
||||
@ -979,13 +1026,13 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
|
||||
return match in_elem.kind() {
|
||||
ty::Int(_) | ty::Uint(_) => Ok(bx.$int_red(args[0].immediate())),
|
||||
ty::Float(_) => Ok(bx.$float_red(args[0].immediate())),
|
||||
_ => return_error!(InvalidMonomorphization::UnsupportedSymbol {
|
||||
span,
|
||||
_ => return_error!(InvalidMonomorphization::UnsupportedSymbol {
|
||||
span,
|
||||
name,
|
||||
symbol: sym::$name,
|
||||
in_ty,
|
||||
in_elem,
|
||||
ret_ty
|
||||
in_elem,
|
||||
ret_ty
|
||||
}),
|
||||
};
|
||||
}
|
||||
@ -1025,7 +1072,15 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
|
||||
return match in_elem.kind() {
|
||||
ty::Int(_) | ty::Uint(_) => {
|
||||
let r = bx.vector_reduce_op(input, $op);
|
||||
Ok(if !$boolean { r } else { bx.icmp(IntPredicate::IntNE, r, bx.context.new_rvalue_zero(r.get_type())) })
|
||||
Ok(if !$boolean {
|
||||
r
|
||||
} else {
|
||||
bx.icmp(
|
||||
IntPredicate::IntNE,
|
||||
r,
|
||||
bx.context.new_rvalue_zero(r.get_type()),
|
||||
)
|
||||
})
|
||||
}
|
||||
_ => return_error!(InvalidMonomorphization::UnsupportedSymbol {
|
||||
span,
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user