Auto merge of #2013 - RalfJung:simd, r=RalfJung

implement simd_{shuffle,gather,scatter}

This makes portable-simd doctests pass. :)

Cc https://github.com/rust-lang/miri/issues/1912

src/shims/intrinsics.rs

@@ -3,7 +3,7 @@
 use log::trace;
 
 use rustc_apfloat::{Float, Round};
-use rustc_middle::ty::layout::{IntegerExt, LayoutOf};
+use rustc_middle::ty::layout::{HasParamEnv, IntegerExt, LayoutOf};
 use rustc_middle::{mir, mir::BinOp, ty, ty::FloatTy};
 use rustc_target::abi::{Align, Integer};
 
@@ -570,8 +570,7 @@ enum Op {
                     let no = this.read_immediate(&this.mplace_index(&no, i)?.into())?;
                     let dest = this.mplace_index(&dest, i)?;
 
-                    let mask = simd_element_to_bool(mask)?;
-                    let val = if mask { yes } else { no };
+                    let val = if simd_element_to_bool(mask)? { yes } else { no };
                     this.write_immediate(*val, &dest.into())?;
                 }
             }
@@ -614,6 +613,91 @@ enum Op {
                     this.write_immediate(val, &dest.into())?;
                 }
             }
+            "simd_shuffle" => {
+                let &[ref left, ref right, ref index] = check_arg_count(args)?;
+                let (left, left_len) = this.operand_to_simd(left)?;
+                let (right, right_len) = this.operand_to_simd(right)?;
+                let (dest, dest_len) = this.place_to_simd(dest)?;
+
+                // `index` is an array, not a SIMD type
+                let ty::Array(_, index_len) = index.layout.ty.kind() else {
+                    bug!("simd_shuffle index argument has non-array type {}", index.layout.ty)
+                };
+                let index_len = index_len.eval_usize(*this.tcx, this.param_env());
+
+                assert_eq!(left_len, right_len);
+                assert_eq!(index_len, dest_len);
+
+                for i in 0..dest_len {
+                    let src_index: u64 = this
+                        .read_immediate(&this.operand_index(&index, i)?.into())?
+                        .to_scalar()?
+                        .to_u32()?
+                        .into();
+                    let dest = this.mplace_index(&dest, i)?;
+
+                    let val = if src_index < left_len {
+                        this.read_immediate(&this.mplace_index(&left, src_index)?.into())?
+                    } else if src_index < left_len.checked_add(right_len).unwrap() {
+                        this.read_immediate(
+                            &this.mplace_index(&right, src_index - left_len)?.into(),
+                        )?
+                    } else {
+                        bug!(
+                            "simd_shuffle index {} is out of bounds for 2 vectors of size {}",
+                            src_index,
+                            left_len
+                        );
+                    };
+                    this.write_immediate(*val, &dest.into())?;
+                }
+            }
+            "simd_gather" => {
+                let &[ref passthru, ref ptrs, ref mask] = check_arg_count(args)?;
+                let (passthru, passthru_len) = this.operand_to_simd(passthru)?;
+                let (ptrs, ptrs_len) = this.operand_to_simd(ptrs)?;
+                let (mask, mask_len) = this.operand_to_simd(mask)?;
+                let (dest, dest_len) = this.place_to_simd(dest)?;
+
+                assert_eq!(dest_len, passthru_len);
+                assert_eq!(dest_len, ptrs_len);
+                assert_eq!(dest_len, mask_len);
+
+                for i in 0..dest_len {
+                    let passthru = this.read_immediate(&this.mplace_index(&passthru, i)?.into())?;
+                    let ptr = this.read_immediate(&this.mplace_index(&ptrs, i)?.into())?;
+                    let mask = this.read_immediate(&this.mplace_index(&mask, i)?.into())?;
+                    let dest = this.mplace_index(&dest, i)?;
+
+                    let val = if simd_element_to_bool(mask)? {
+                        let place = this.deref_operand(&ptr.into())?;
+                        this.read_immediate(&place.into())?
+                    } else {
+                        passthru
+                    };
+                    this.write_immediate(*val, &dest.into())?;
+                }
+            }
+            "simd_scatter" => {
+                let &[ref value, ref ptrs, ref mask] = check_arg_count(args)?;
+                let (value, value_len) = this.operand_to_simd(value)?;
+                let (ptrs, ptrs_len) = this.operand_to_simd(ptrs)?;
+                let (mask, mask_len) = this.operand_to_simd(mask)?;
+
+                assert_eq!(ptrs_len, value_len);
+                assert_eq!(ptrs_len, mask_len);
+
+                for i in 0..ptrs_len {
+                    let value = this.read_immediate(&this.mplace_index(&value, i)?.into())?;
+                    let ptr = this.read_immediate(&this.mplace_index(&ptrs, i)?.into())?;
+                    let mask = this.read_immediate(&this.mplace_index(&mask, i)?.into())?;
+
+                    if simd_element_to_bool(mask)? {
+                        let place = this.deref_operand(&ptr.into())?;
+                        this.write_immediate(*value, &place.into())?;
+                    }
+                }
+            }
 
             // Atomic operations
             "atomic_load" => this.atomic_load(args, dest, AtomicReadOp::SeqCst)?,

tests/compile-fail/intrinsics/simd-gather.rs

@@ -0,0 +1,9 @@
+// error-pattern: pointer to 1 byte starting at offset 9 is out-of-bounds
+#![feature(portable_simd)]
+use std::simd::*;
+
+fn main() { unsafe {
+    let vec: &[i8] = &[10, 11, 12, 13, 14, 15, 16, 17, 18];
+    let idxs = Simd::from_array([9, 3, 0, 17]);
+    let _result = Simd::gather_select_unchecked(&vec, Mask::splat(true), idxs, Simd::splat(0));
+} }

tests/compile-fail/intrinsics/simd-scatter.rs

@@ -0,0 +1,9 @@
+// error-pattern: pointer to 1 byte starting at offset 9 is out-of-bounds
+#![feature(portable_simd)]
+use std::simd::*;
+
+fn main() { unsafe {
+    let mut vec: Vec<i8> = vec![10, 11, 12, 13, 14, 15, 16, 17, 18];
+    let idxs = Simd::from_array([9, 3, 0, 17]);
+    Simd::from_array([-27, 82, -41, 124]).scatter_select_unchecked(&mut vec, Mask::splat(true), idxs);
+} }

tests/run-pass/portable-simd.rs

@@ -238,6 +238,28 @@ fn simd_cast() {
     }
 }
 
+fn simd_swizzle() {
+    use Which::*;
+
+    let a = f32x4::splat(10.0);
+    let b = f32x4::from_array([1.0, 2.0, 3.0, -4.0]);
+
+    assert_eq!(simd_swizzle!(b, [3, 0, 0, 2]), f32x4::from_array([-4.0, 1.0, 1.0, 3.0]));
+    assert_eq!(simd_swizzle!(b, [1, 2]), f32x2::from_array([2.0, 3.0]));
+    assert_eq!(simd_swizzle!(b, a, [First(3), Second(0)]), f32x2::from_array([-4.0, 10.0]));
+}
+
+fn simd_gather_scatter() {
+    let mut vec: Vec<i16> = vec![10, 11, 12, 13, 14, 15, 16, 17, 18];
+    let idxs = Simd::from_array([9, 3, 0, 17]);
+    let result = Simd::gather_or_default(&vec, idxs); // Note the lane that is out-of-bounds.
+    assert_eq!(result, Simd::from_array([0, 13, 10, 0]));
+
+    let idxs = Simd::from_array([9, 3, 0, 0]);
+    Simd::from_array([-27, 82, -41, 124]).scatter(&mut vec, idxs);
+    assert_eq!(vec, vec![124, 11, 12, 82, 14, 15, 16, 17, 18]);
+}
+
 fn simd_intrinsics() {
     extern "platform-intrinsic" {
         fn simd_eq<T, U>(x: T, y: T) -> U;
@@ -276,5 +298,7 @@ fn main() {
     simd_ops_f64();
     simd_ops_i32();
     simd_cast();
+    simd_swizzle();
+    simd_gather_scatter();
     simd_intrinsics();
 }