From 3eba7fcf7336c7cebdea4b2c2140c947b6bac67c Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Wed, 9 Mar 2022 18:48:31 -0500 Subject: [PATCH 1/3] implement simd_shuffle --- src/shims/intrinsics.rs | 41 ++++++++++++++++++++++++++++++++- tests/run-pass/portable-simd.rs | 12 ++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/src/shims/intrinsics.rs b/src/shims/intrinsics.rs index 49c9c0fb0d9..43c61091b9e 100644 --- a/src/shims/intrinsics.rs +++ b/src/shims/intrinsics.rs @@ -3,7 +3,7 @@ use log::trace; use rustc_apfloat::{Float, Round}; -use rustc_middle::ty::layout::{IntegerExt, LayoutOf}; +use rustc_middle::ty::layout::{HasParamEnv, IntegerExt, LayoutOf}; use rustc_middle::{mir, mir::BinOp, ty, ty::FloatTy}; use rustc_target::abi::{Align, Integer}; @@ -614,6 +614,45 @@ enum Op { this.write_immediate(val, &dest.into())?; } } + "simd_shuffle" => { + let &[ref left, ref right, ref index] = check_arg_count(args)?; + let (left, left_len) = this.operand_to_simd(left)?; + let (right, right_len) = this.operand_to_simd(right)?; + let (dest, dest_len) = this.place_to_simd(dest)?; + + // `index` is an array, not a SIMD type + let ty::Array(_, index_len) = index.layout.ty.kind() else { + bug!("simd_shuffle index argument has non-array type {}", index.layout.ty) + }; + let index_len = index_len.eval_usize(*this.tcx, this.param_env()); + + assert_eq!(left_len, right_len); + assert_eq!(index_len, dest_len); + + for i in 0..dest_len { + let src_index: u64 = this + .read_immediate(&this.operand_index(&index, i)?.into())? + .to_scalar()? + .to_u32()? + .into(); + let dest = this.mplace_index(&dest, i)?; + + let val = if src_index < left_len { + this.read_immediate(&this.mplace_index(&left, src_index)?.into())? + } else if src_index < left_len.checked_add(right_len).unwrap() { + this.read_immediate( + &this.mplace_index(&right, src_index - left_len)?.into(), + )? + } else { + bug!( + "simd_shuffle index {} is out of bounds for 2 vectors of size {}", + src_index, + left_len + ); + }; + this.write_immediate(*val, &dest.into())?; + } + } // Atomic operations "atomic_load" => this.atomic_load(args, dest, AtomicReadOp::SeqCst)?, diff --git a/tests/run-pass/portable-simd.rs b/tests/run-pass/portable-simd.rs index b87bd4fd6ad..67e4e52a784 100644 --- a/tests/run-pass/portable-simd.rs +++ b/tests/run-pass/portable-simd.rs @@ -238,6 +238,17 @@ fn simd_cast() { } } +fn simd_swizzle() { + use Which::*; + + let a = f32x4::splat(10.0); + let b = f32x4::from_array([1.0, 2.0, 3.0, -4.0]); + + assert_eq!(simd_swizzle!(b, [3, 0, 0, 2]), f32x4::from_array([-4.0, 1.0, 1.0, 3.0])); + assert_eq!(simd_swizzle!(b, [1, 2]), f32x2::from_array([2.0, 3.0])); + assert_eq!(simd_swizzle!(b, a, [First(3), Second(0)]), f32x2::from_array([-4.0, 10.0])); +} + fn simd_intrinsics() { extern "platform-intrinsic" { fn simd_eq(x: T, y: T) -> U; @@ -276,5 +287,6 @@ fn main() { simd_ops_f64(); simd_ops_i32(); simd_cast(); + simd_swizzle(); simd_intrinsics(); } From 576e2bbed5cfa26fdab2af62e1372d1f8fa581d1 Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Wed, 9 Mar 2022 19:18:07 -0500 Subject: [PATCH 2/3] implement gather --- src/shims/intrinsics.rs | 27 ++++++++++++++++++++ tests/compile-fail/intrinsics/simd-gather.rs | 9 +++++++ tests/run-pass/portable-simd.rs | 8 ++++++ 3 files changed, 44 insertions(+) create mode 100644 tests/compile-fail/intrinsics/simd-gather.rs diff --git a/src/shims/intrinsics.rs b/src/shims/intrinsics.rs index 43c61091b9e..54e7118b259 100644 --- a/src/shims/intrinsics.rs +++ b/src/shims/intrinsics.rs @@ -653,6 +653,33 @@ enum Op { this.write_immediate(*val, &dest.into())?; } } + "simd_gather" => { + let &[ref passthru, ref ptrs, ref mask] = check_arg_count(args)?; + let (passthru, passthru_len) = this.operand_to_simd(passthru)?; + let (ptrs, ptrs_len) = this.operand_to_simd(ptrs)?; + let (mask, mask_len) = this.operand_to_simd(mask)?; + let (dest, dest_len) = this.place_to_simd(dest)?; + + assert_eq!(dest_len, passthru_len); + assert_eq!(dest_len, ptrs_len); + assert_eq!(dest_len, mask_len); + + for i in 0..dest_len { + let passthru = this.read_immediate(&this.mplace_index(&passthru, i)?.into())?; + let ptr = this.read_immediate(&this.mplace_index(&ptrs, i)?.into())?; + let mask = this.read_immediate(&this.mplace_index(&mask, i)?.into())?; + let dest = this.mplace_index(&dest, i)?; + + let mask = simd_element_to_bool(mask)?; + let val = if mask { + let place = this.deref_operand(&ptr.into())?; + this.read_immediate(&place.into())? + } else { + passthru + }; + this.write_immediate(*val, &dest.into())?; + } + } // Atomic operations "atomic_load" => this.atomic_load(args, dest, AtomicReadOp::SeqCst)?, diff --git a/tests/compile-fail/intrinsics/simd-gather.rs b/tests/compile-fail/intrinsics/simd-gather.rs new file mode 100644 index 00000000000..2fb5da01f10 --- /dev/null +++ b/tests/compile-fail/intrinsics/simd-gather.rs @@ -0,0 +1,9 @@ +// error-pattern: out-of-bounds +#![feature(portable_simd)] +use std::simd::*; + +fn main() { unsafe { + let vec: &[i16] = &[10, 11, 12, 13, 14, 15, 16, 17, 18]; + let idxs = Simd::from_array([9, 3, 0, 17]); + let _result = Simd::gather_select_unchecked(&vec, Mask::splat(true), idxs, Simd::splat(0)); +} } diff --git a/tests/run-pass/portable-simd.rs b/tests/run-pass/portable-simd.rs index 67e4e52a784..ad9dafea4ba 100644 --- a/tests/run-pass/portable-simd.rs +++ b/tests/run-pass/portable-simd.rs @@ -249,6 +249,13 @@ fn simd_swizzle() { assert_eq!(simd_swizzle!(b, a, [First(3), Second(0)]), f32x2::from_array([-4.0, 10.0])); } +fn simd_gather_scatter() { + let vec: &[i16] = &[10, 11, 12, 13, 14, 15, 16, 17, 18]; + let idxs = Simd::from_array([9, 3, 0, 17]); + let result = Simd::gather_or_default(&vec, idxs); // Note the lane that is out-of-bounds. + assert_eq!(result, Simd::from_array([0, 13, 10, 0])); +} + fn simd_intrinsics() { extern "platform-intrinsic" { fn simd_eq(x: T, y: T) -> U; @@ -288,5 +295,6 @@ fn main() { simd_ops_i32(); simd_cast(); simd_swizzle(); + simd_gather_scatter(); simd_intrinsics(); } From 41ffce1145efd87e23878ce0802a52557b24d852 Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Wed, 9 Mar 2022 19:26:40 -0500 Subject: [PATCH 3/3] implement simd_scatter --- src/shims/intrinsics.rs | 26 ++++++++++++++++--- tests/compile-fail/intrinsics/simd-gather.rs | 4 +-- tests/compile-fail/intrinsics/simd-scatter.rs | 9 +++++++ tests/run-pass/portable-simd.rs | 6 ++++- 4 files changed, 38 insertions(+), 7 deletions(-) create mode 100644 tests/compile-fail/intrinsics/simd-scatter.rs diff --git a/src/shims/intrinsics.rs b/src/shims/intrinsics.rs index 54e7118b259..b4416bfa985 100644 --- a/src/shims/intrinsics.rs +++ b/src/shims/intrinsics.rs @@ -570,8 +570,7 @@ enum Op { let no = this.read_immediate(&this.mplace_index(&no, i)?.into())?; let dest = this.mplace_index(&dest, i)?; - let mask = simd_element_to_bool(mask)?; - let val = if mask { yes } else { no }; + let val = if simd_element_to_bool(mask)? { yes } else { no }; this.write_immediate(*val, &dest.into())?; } } @@ -670,8 +669,7 @@ enum Op { let mask = this.read_immediate(&this.mplace_index(&mask, i)?.into())?; let dest = this.mplace_index(&dest, i)?; - let mask = simd_element_to_bool(mask)?; - let val = if mask { + let val = if simd_element_to_bool(mask)? { let place = this.deref_operand(&ptr.into())?; this.read_immediate(&place.into())? } else { @@ -680,6 +678,26 @@ enum Op { this.write_immediate(*val, &dest.into())?; } } + "simd_scatter" => { + let &[ref value, ref ptrs, ref mask] = check_arg_count(args)?; + let (value, value_len) = this.operand_to_simd(value)?; + let (ptrs, ptrs_len) = this.operand_to_simd(ptrs)?; + let (mask, mask_len) = this.operand_to_simd(mask)?; + + assert_eq!(ptrs_len, value_len); + assert_eq!(ptrs_len, mask_len); + + for i in 0..ptrs_len { + let value = this.read_immediate(&this.mplace_index(&value, i)?.into())?; + let ptr = this.read_immediate(&this.mplace_index(&ptrs, i)?.into())?; + let mask = this.read_immediate(&this.mplace_index(&mask, i)?.into())?; + + if simd_element_to_bool(mask)? { + let place = this.deref_operand(&ptr.into())?; + this.write_immediate(*value, &place.into())?; + } + } + } // Atomic operations "atomic_load" => this.atomic_load(args, dest, AtomicReadOp::SeqCst)?, diff --git a/tests/compile-fail/intrinsics/simd-gather.rs b/tests/compile-fail/intrinsics/simd-gather.rs index 2fb5da01f10..ae6f048226d 100644 --- a/tests/compile-fail/intrinsics/simd-gather.rs +++ b/tests/compile-fail/intrinsics/simd-gather.rs @@ -1,9 +1,9 @@ -// error-pattern: out-of-bounds +// error-pattern: pointer to 1 byte starting at offset 9 is out-of-bounds #![feature(portable_simd)] use std::simd::*; fn main() { unsafe { - let vec: &[i16] = &[10, 11, 12, 13, 14, 15, 16, 17, 18]; + let vec: &[i8] = &[10, 11, 12, 13, 14, 15, 16, 17, 18]; let idxs = Simd::from_array([9, 3, 0, 17]); let _result = Simd::gather_select_unchecked(&vec, Mask::splat(true), idxs, Simd::splat(0)); } } diff --git a/tests/compile-fail/intrinsics/simd-scatter.rs b/tests/compile-fail/intrinsics/simd-scatter.rs new file mode 100644 index 00000000000..f46e4f0d4f6 --- /dev/null +++ b/tests/compile-fail/intrinsics/simd-scatter.rs @@ -0,0 +1,9 @@ +// error-pattern: pointer to 1 byte starting at offset 9 is out-of-bounds +#![feature(portable_simd)] +use std::simd::*; + +fn main() { unsafe { + let mut vec: Vec = vec![10, 11, 12, 13, 14, 15, 16, 17, 18]; + let idxs = Simd::from_array([9, 3, 0, 17]); + Simd::from_array([-27, 82, -41, 124]).scatter_select_unchecked(&mut vec, Mask::splat(true), idxs); +} } diff --git a/tests/run-pass/portable-simd.rs b/tests/run-pass/portable-simd.rs index ad9dafea4ba..c046af0bcf5 100644 --- a/tests/run-pass/portable-simd.rs +++ b/tests/run-pass/portable-simd.rs @@ -250,10 +250,14 @@ fn simd_swizzle() { } fn simd_gather_scatter() { - let vec: &[i16] = &[10, 11, 12, 13, 14, 15, 16, 17, 18]; + let mut vec: Vec = vec![10, 11, 12, 13, 14, 15, 16, 17, 18]; let idxs = Simd::from_array([9, 3, 0, 17]); let result = Simd::gather_or_default(&vec, idxs); // Note the lane that is out-of-bounds. assert_eq!(result, Simd::from_array([0, 13, 10, 0])); + + let idxs = Simd::from_array([9, 3, 0, 0]); + Simd::from_array([-27, 82, -41, 124]).scatter(&mut vec, idxs); + assert_eq!(vec, vec![124, 11, 12, 82, 14, 15, 16, 17, 18]); } fn simd_intrinsics() {