diff --git a/src/shims/intrinsics.rs b/src/shims/intrinsics.rs index 49c9c0fb0d9..b4416bfa985 100644 --- a/src/shims/intrinsics.rs +++ b/src/shims/intrinsics.rs @@ -3,7 +3,7 @@ use log::trace; use rustc_apfloat::{Float, Round}; -use rustc_middle::ty::layout::{IntegerExt, LayoutOf}; +use rustc_middle::ty::layout::{HasParamEnv, IntegerExt, LayoutOf}; use rustc_middle::{mir, mir::BinOp, ty, ty::FloatTy}; use rustc_target::abi::{Align, Integer}; @@ -570,8 +570,7 @@ enum Op { let no = this.read_immediate(&this.mplace_index(&no, i)?.into())?; let dest = this.mplace_index(&dest, i)?; - let mask = simd_element_to_bool(mask)?; - let val = if mask { yes } else { no }; + let val = if simd_element_to_bool(mask)? { yes } else { no }; this.write_immediate(*val, &dest.into())?; } } @@ -614,6 +613,91 @@ enum Op { this.write_immediate(val, &dest.into())?; } } + "simd_shuffle" => { + let &[ref left, ref right, ref index] = check_arg_count(args)?; + let (left, left_len) = this.operand_to_simd(left)?; + let (right, right_len) = this.operand_to_simd(right)?; + let (dest, dest_len) = this.place_to_simd(dest)?; + + // `index` is an array, not a SIMD type + let ty::Array(_, index_len) = index.layout.ty.kind() else { + bug!("simd_shuffle index argument has non-array type {}", index.layout.ty) + }; + let index_len = index_len.eval_usize(*this.tcx, this.param_env()); + + assert_eq!(left_len, right_len); + assert_eq!(index_len, dest_len); + + for i in 0..dest_len { + let src_index: u64 = this + .read_immediate(&this.operand_index(&index, i)?.into())? + .to_scalar()? + .to_u32()? + .into(); + let dest = this.mplace_index(&dest, i)?; + + let val = if src_index < left_len { + this.read_immediate(&this.mplace_index(&left, src_index)?.into())? + } else if src_index < left_len.checked_add(right_len).unwrap() { + this.read_immediate( + &this.mplace_index(&right, src_index - left_len)?.into(), + )? + } else { + bug!( + "simd_shuffle index {} is out of bounds for 2 vectors of size {}", + src_index, + left_len + ); + }; + this.write_immediate(*val, &dest.into())?; + } + } + "simd_gather" => { + let &[ref passthru, ref ptrs, ref mask] = check_arg_count(args)?; + let (passthru, passthru_len) = this.operand_to_simd(passthru)?; + let (ptrs, ptrs_len) = this.operand_to_simd(ptrs)?; + let (mask, mask_len) = this.operand_to_simd(mask)?; + let (dest, dest_len) = this.place_to_simd(dest)?; + + assert_eq!(dest_len, passthru_len); + assert_eq!(dest_len, ptrs_len); + assert_eq!(dest_len, mask_len); + + for i in 0..dest_len { + let passthru = this.read_immediate(&this.mplace_index(&passthru, i)?.into())?; + let ptr = this.read_immediate(&this.mplace_index(&ptrs, i)?.into())?; + let mask = this.read_immediate(&this.mplace_index(&mask, i)?.into())?; + let dest = this.mplace_index(&dest, i)?; + + let val = if simd_element_to_bool(mask)? { + let place = this.deref_operand(&ptr.into())?; + this.read_immediate(&place.into())? + } else { + passthru + }; + this.write_immediate(*val, &dest.into())?; + } + } + "simd_scatter" => { + let &[ref value, ref ptrs, ref mask] = check_arg_count(args)?; + let (value, value_len) = this.operand_to_simd(value)?; + let (ptrs, ptrs_len) = this.operand_to_simd(ptrs)?; + let (mask, mask_len) = this.operand_to_simd(mask)?; + + assert_eq!(ptrs_len, value_len); + assert_eq!(ptrs_len, mask_len); + + for i in 0..ptrs_len { + let value = this.read_immediate(&this.mplace_index(&value, i)?.into())?; + let ptr = this.read_immediate(&this.mplace_index(&ptrs, i)?.into())?; + let mask = this.read_immediate(&this.mplace_index(&mask, i)?.into())?; + + if simd_element_to_bool(mask)? { + let place = this.deref_operand(&ptr.into())?; + this.write_immediate(*value, &place.into())?; + } + } + } // Atomic operations "atomic_load" => this.atomic_load(args, dest, AtomicReadOp::SeqCst)?, diff --git a/tests/compile-fail/intrinsics/simd-gather.rs b/tests/compile-fail/intrinsics/simd-gather.rs new file mode 100644 index 00000000000..ae6f048226d --- /dev/null +++ b/tests/compile-fail/intrinsics/simd-gather.rs @@ -0,0 +1,9 @@ +// error-pattern: pointer to 1 byte starting at offset 9 is out-of-bounds +#![feature(portable_simd)] +use std::simd::*; + +fn main() { unsafe { + let vec: &[i8] = &[10, 11, 12, 13, 14, 15, 16, 17, 18]; + let idxs = Simd::from_array([9, 3, 0, 17]); + let _result = Simd::gather_select_unchecked(&vec, Mask::splat(true), idxs, Simd::splat(0)); +} } diff --git a/tests/compile-fail/intrinsics/simd-scatter.rs b/tests/compile-fail/intrinsics/simd-scatter.rs new file mode 100644 index 00000000000..f46e4f0d4f6 --- /dev/null +++ b/tests/compile-fail/intrinsics/simd-scatter.rs @@ -0,0 +1,9 @@ +// error-pattern: pointer to 1 byte starting at offset 9 is out-of-bounds +#![feature(portable_simd)] +use std::simd::*; + +fn main() { unsafe { + let mut vec: Vec = vec![10, 11, 12, 13, 14, 15, 16, 17, 18]; + let idxs = Simd::from_array([9, 3, 0, 17]); + Simd::from_array([-27, 82, -41, 124]).scatter_select_unchecked(&mut vec, Mask::splat(true), idxs); +} } diff --git a/tests/run-pass/portable-simd.rs b/tests/run-pass/portable-simd.rs index b87bd4fd6ad..c046af0bcf5 100644 --- a/tests/run-pass/portable-simd.rs +++ b/tests/run-pass/portable-simd.rs @@ -238,6 +238,28 @@ fn simd_cast() { } } +fn simd_swizzle() { + use Which::*; + + let a = f32x4::splat(10.0); + let b = f32x4::from_array([1.0, 2.0, 3.0, -4.0]); + + assert_eq!(simd_swizzle!(b, [3, 0, 0, 2]), f32x4::from_array([-4.0, 1.0, 1.0, 3.0])); + assert_eq!(simd_swizzle!(b, [1, 2]), f32x2::from_array([2.0, 3.0])); + assert_eq!(simd_swizzle!(b, a, [First(3), Second(0)]), f32x2::from_array([-4.0, 10.0])); +} + +fn simd_gather_scatter() { + let mut vec: Vec = vec![10, 11, 12, 13, 14, 15, 16, 17, 18]; + let idxs = Simd::from_array([9, 3, 0, 17]); + let result = Simd::gather_or_default(&vec, idxs); // Note the lane that is out-of-bounds. + assert_eq!(result, Simd::from_array([0, 13, 10, 0])); + + let idxs = Simd::from_array([9, 3, 0, 0]); + Simd::from_array([-27, 82, -41, 124]).scatter(&mut vec, idxs); + assert_eq!(vec, vec![124, 11, 12, 82, 14, 15, 16, 17, 18]); +} + fn simd_intrinsics() { extern "platform-intrinsic" { fn simd_eq(x: T, y: T) -> U; @@ -276,5 +298,7 @@ fn main() { simd_ops_f64(); simd_ops_i32(); simd_cast(); + simd_swizzle(); + simd_gather_scatter(); simd_intrinsics(); }