diff --git a/library/portable-simd/beginners-guide.md b/library/portable-simd/beginners-guide.md index 75158e5aa85..17ade06ae80 100644 --- a/library/portable-simd/beginners-guide.md +++ b/library/portable-simd/beginners-guide.md @@ -82,5 +82,10 @@ Fortunately, most SIMD types have a fairly predictable size. `i32x4` is bit-equi However, this is not the same as alignment. Computer architectures generally prefer aligned accesses, especially when moving data between memory and vector registers, and while some support specialized operations that can bend the rules to help with this, unaligned access is still typically slow, or even undefined behavior. In addition, different architectures can require different alignments when interacting with their native SIMD types. For this reason, any `#[repr(simd)]` type has a non-portable alignment. If it is necessary to directly interact with the alignment of these types, it should be via [`mem::align_of`]. +When working with slices, data correctly aligned for SIMD can be acquired using the [`as_simd`] and [`as_simd_mut`] methods of the slice primitive. + [`mem::transmute`]: https://doc.rust-lang.org/core/mem/fn.transmute.html [`mem::align_of`]: https://doc.rust-lang.org/core/mem/fn.align_of.html +[`as_simd`]: https://doc.rust-lang.org/nightly/std/primitive.slice.html#method.as_simd +[`as_simd_mut`]: https://doc.rust-lang.org/nightly/std/primitive.slice.html#method.as_simd_mut + diff --git a/library/portable-simd/crates/core_simd/Cargo.toml b/library/portable-simd/crates/core_simd/Cargo.toml index 8877c6df66e..8a29cf15696 100644 --- a/library/portable-simd/crates/core_simd/Cargo.toml +++ b/library/portable-simd/crates/core_simd/Cargo.toml @@ -9,7 +9,8 @@ categories = ["hardware-support", "no-std"] license = "MIT OR Apache-2.0" [features] -default = [] +default = ["as_crate"] +as_crate = [] std = [] generic_const_exprs = [] diff --git a/library/portable-simd/crates/core_simd/src/comparisons.rs b/library/portable-simd/crates/core_simd/src/comparisons.rs deleted file mode 100644 index 7b0d0a6864b..00000000000 --- a/library/portable-simd/crates/core_simd/src/comparisons.rs +++ /dev/null @@ -1,120 +0,0 @@ -use crate::simd::intrinsics; -use crate::simd::{LaneCount, Mask, Simd, SimdElement, SupportedLaneCount}; - -impl Simd -where - T: SimdElement + PartialEq, - LaneCount: SupportedLaneCount, -{ - /// Test if each lane is equal to the corresponding lane in `other`. - #[inline] - #[must_use = "method returns a new mask and does not mutate the original value"] - pub fn lanes_eq(self, other: Self) -> Mask { - // Safety: `self` is a vector, and the result of the comparison - // is always a valid mask. - unsafe { Mask::from_int_unchecked(intrinsics::simd_eq(self, other)) } - } - - /// Test if each lane is not equal to the corresponding lane in `other`. - #[inline] - #[must_use = "method returns a new mask and does not mutate the original value"] - pub fn lanes_ne(self, other: Self) -> Mask { - // Safety: `self` is a vector, and the result of the comparison - // is always a valid mask. - unsafe { Mask::from_int_unchecked(intrinsics::simd_ne(self, other)) } - } -} - -impl Simd -where - T: SimdElement + PartialOrd, - LaneCount: SupportedLaneCount, -{ - /// Test if each lane is less than the corresponding lane in `other`. - #[inline] - #[must_use = "method returns a new mask and does not mutate the original value"] - pub fn lanes_lt(self, other: Self) -> Mask { - // Safety: `self` is a vector, and the result of the comparison - // is always a valid mask. - unsafe { Mask::from_int_unchecked(intrinsics::simd_lt(self, other)) } - } - - /// Test if each lane is greater than the corresponding lane in `other`. - #[inline] - #[must_use = "method returns a new mask and does not mutate the original value"] - pub fn lanes_gt(self, other: Self) -> Mask { - // Safety: `self` is a vector, and the result of the comparison - // is always a valid mask. - unsafe { Mask::from_int_unchecked(intrinsics::simd_gt(self, other)) } - } - - /// Test if each lane is less than or equal to the corresponding lane in `other`. - #[inline] - #[must_use = "method returns a new mask and does not mutate the original value"] - pub fn lanes_le(self, other: Self) -> Mask { - // Safety: `self` is a vector, and the result of the comparison - // is always a valid mask. - unsafe { Mask::from_int_unchecked(intrinsics::simd_le(self, other)) } - } - - /// Test if each lane is greater than or equal to the corresponding lane in `other`. - #[inline] - #[must_use = "method returns a new mask and does not mutate the original value"] - pub fn lanes_ge(self, other: Self) -> Mask { - // Safety: `self` is a vector, and the result of the comparison - // is always a valid mask. - unsafe { Mask::from_int_unchecked(intrinsics::simd_ge(self, other)) } - } -} - -macro_rules! impl_ord_methods_vector { - { $type:ty } => { - impl Simd<$type, LANES> - where - LaneCount: SupportedLaneCount, - { - /// Returns the lane-wise minimum with `other`. - #[must_use = "method returns a new vector and does not mutate the original value"] - #[inline] - pub fn min(self, other: Self) -> Self { - self.lanes_gt(other).select(other, self) - } - - /// Returns the lane-wise maximum with `other`. - #[must_use = "method returns a new vector and does not mutate the original value"] - #[inline] - pub fn max(self, other: Self) -> Self { - self.lanes_lt(other).select(other, self) - } - - /// Restrict each lane to a certain interval. - /// - /// For each lane, returns `max` if `self` is greater than `max`, and `min` if `self` is - /// less than `min`. Otherwise returns `self`. - /// - /// # Panics - /// - /// Panics if `min > max` on any lane. - #[must_use = "method returns a new vector and does not mutate the original value"] - #[inline] - pub fn clamp(self, min: Self, max: Self) -> Self { - assert!( - min.lanes_le(max).all(), - "each lane in `min` must be less than or equal to the corresponding lane in `max`", - ); - self.max(min).min(max) - } - } - } -} - -impl_ord_methods_vector!(i8); -impl_ord_methods_vector!(i16); -impl_ord_methods_vector!(i32); -impl_ord_methods_vector!(i64); -impl_ord_methods_vector!(isize); -impl_ord_methods_vector!(u8); -impl_ord_methods_vector!(u16); -impl_ord_methods_vector!(u32); -impl_ord_methods_vector!(u64); -impl_ord_methods_vector!(usize); diff --git a/library/portable-simd/crates/core_simd/src/elements.rs b/library/portable-simd/crates/core_simd/src/elements.rs new file mode 100644 index 00000000000..701eb66b248 --- /dev/null +++ b/library/portable-simd/crates/core_simd/src/elements.rs @@ -0,0 +1,11 @@ +mod float; +mod int; +mod uint; + +mod sealed { + pub trait Sealed {} +} + +pub use float::*; +pub use int::*; +pub use uint::*; diff --git a/library/portable-simd/crates/core_simd/src/elements/float.rs b/library/portable-simd/crates/core_simd/src/elements/float.rs new file mode 100644 index 00000000000..d6022327055 --- /dev/null +++ b/library/portable-simd/crates/core_simd/src/elements/float.rs @@ -0,0 +1,357 @@ +use super::sealed::Sealed; +use crate::simd::{ + intrinsics, LaneCount, Mask, Simd, SimdElement, SimdPartialEq, SimdPartialOrd, + SupportedLaneCount, +}; + +/// Operations on SIMD vectors of floats. +pub trait SimdFloat: Copy + Sealed { + /// Mask type used for manipulating this SIMD vector type. + type Mask; + + /// Scalar type contained by this SIMD vector type. + type Scalar; + + /// Bit representation of this SIMD vector type. + type Bits; + + /// Raw transmutation to an unsigned integer vector type with the + /// same size and number of lanes. + #[must_use = "method returns a new vector and does not mutate the original value"] + fn to_bits(self) -> Self::Bits; + + /// Raw transmutation from an unsigned integer vector type with the + /// same size and number of lanes. + #[must_use = "method returns a new vector and does not mutate the original value"] + fn from_bits(bits: Self::Bits) -> Self; + + /// Produces a vector where every lane has the absolute value of the + /// equivalently-indexed lane in `self`. + #[must_use = "method returns a new vector and does not mutate the original value"] + fn abs(self) -> Self; + + /// Takes the reciprocal (inverse) of each lane, `1/x`. + #[must_use = "method returns a new vector and does not mutate the original value"] + fn recip(self) -> Self; + + /// Converts each lane from radians to degrees. + #[must_use = "method returns a new vector and does not mutate the original value"] + fn to_degrees(self) -> Self; + + /// Converts each lane from degrees to radians. + #[must_use = "method returns a new vector and does not mutate the original value"] + fn to_radians(self) -> Self; + + /// Returns true for each lane if it has a positive sign, including + /// `+0.0`, `NaN`s with positive sign bit and positive infinity. + #[must_use = "method returns a new mask and does not mutate the original value"] + fn is_sign_positive(self) -> Self::Mask; + + /// Returns true for each lane if it has a negative sign, including + /// `-0.0`, `NaN`s with negative sign bit and negative infinity. + #[must_use = "method returns a new mask and does not mutate the original value"] + fn is_sign_negative(self) -> Self::Mask; + + /// Returns true for each lane if its value is `NaN`. + #[must_use = "method returns a new mask and does not mutate the original value"] + fn is_nan(self) -> Self::Mask; + + /// Returns true for each lane if its value is positive infinity or negative infinity. + #[must_use = "method returns a new mask and does not mutate the original value"] + fn is_infinite(self) -> Self::Mask; + + /// Returns true for each lane if its value is neither infinite nor `NaN`. + #[must_use = "method returns a new mask and does not mutate the original value"] + fn is_finite(self) -> Self::Mask; + + /// Returns true for each lane if its value is subnormal. + #[must_use = "method returns a new mask and does not mutate the original value"] + fn is_subnormal(self) -> Self::Mask; + + /// Returns true for each lane if its value is neither zero, infinite, + /// subnormal, nor `NaN`. + #[must_use = "method returns a new mask and does not mutate the original value"] + fn is_normal(self) -> Self::Mask; + + /// Replaces each lane with a number that represents its sign. + /// + /// * `1.0` if the number is positive, `+0.0`, or `INFINITY` + /// * `-1.0` if the number is negative, `-0.0`, or `NEG_INFINITY` + /// * `NAN` if the number is `NAN` + #[must_use = "method returns a new vector and does not mutate the original value"] + fn signum(self) -> Self; + + /// Returns each lane with the magnitude of `self` and the sign of `sign`. + /// + /// For any lane containing a `NAN`, a `NAN` with the sign of `sign` is returned. + #[must_use = "method returns a new vector and does not mutate the original value"] + fn copysign(self, sign: Self) -> Self; + + /// Returns the minimum of each lane. + /// + /// If one of the values is `NAN`, then the other value is returned. + #[must_use = "method returns a new vector and does not mutate the original value"] + fn simd_min(self, other: Self) -> Self; + + /// Returns the maximum of each lane. + /// + /// If one of the values is `NAN`, then the other value is returned. + #[must_use = "method returns a new vector and does not mutate the original value"] + fn simd_max(self, other: Self) -> Self; + + /// Restrict each lane to a certain interval unless it is NaN. + /// + /// For each lane in `self`, returns the corresponding lane in `max` if the lane is + /// greater than `max`, and the corresponding lane in `min` if the lane is less + /// than `min`. Otherwise returns the lane in `self`. + #[must_use = "method returns a new vector and does not mutate the original value"] + fn simd_clamp(self, min: Self, max: Self) -> Self; + + /// Returns the sum of the lanes of the vector. + /// + /// # Examples + /// + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::{f32x2, SimdFloat}; + /// let v = f32x2::from_array([1., 2.]); + /// assert_eq!(v.reduce_sum(), 3.); + /// ``` + fn reduce_sum(self) -> Self::Scalar; + + /// Reducing multiply. Returns the product of the lanes of the vector. + /// + /// # Examples + /// + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::{f32x2, SimdFloat}; + /// let v = f32x2::from_array([3., 4.]); + /// assert_eq!(v.reduce_product(), 12.); + /// ``` + fn reduce_product(self) -> Self::Scalar; + + /// Returns the maximum lane in the vector. + /// + /// Returns values based on equality, so a vector containing both `0.` and `-0.` may + /// return either. + /// + /// This function will not return `NaN` unless all lanes are `NaN`. + /// + /// # Examples + /// + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::{f32x2, SimdFloat}; + /// let v = f32x2::from_array([1., 2.]); + /// assert_eq!(v.reduce_max(), 2.); + /// + /// // NaN values are skipped... + /// let v = f32x2::from_array([1., f32::NAN]); + /// assert_eq!(v.reduce_max(), 1.); + /// + /// // ...unless all values are NaN + /// let v = f32x2::from_array([f32::NAN, f32::NAN]); + /// assert!(v.reduce_max().is_nan()); + /// ``` + fn reduce_max(self) -> Self::Scalar; + + /// Returns the minimum lane in the vector. + /// + /// Returns values based on equality, so a vector containing both `0.` and `-0.` may + /// return either. + /// + /// This function will not return `NaN` unless all lanes are `NaN`. + /// + /// # Examples + /// + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::{f32x2, SimdFloat}; + /// let v = f32x2::from_array([3., 7.]); + /// assert_eq!(v.reduce_min(), 3.); + /// + /// // NaN values are skipped... + /// let v = f32x2::from_array([1., f32::NAN]); + /// assert_eq!(v.reduce_min(), 1.); + /// + /// // ...unless all values are NaN + /// let v = f32x2::from_array([f32::NAN, f32::NAN]); + /// assert!(v.reduce_min().is_nan()); + /// ``` + fn reduce_min(self) -> Self::Scalar; +} + +macro_rules! impl_trait { + { $($ty:ty { bits: $bits_ty:ty, mask: $mask_ty:ty }),* } => { + $( + impl Sealed for Simd<$ty, LANES> + where + LaneCount: SupportedLaneCount, + { + } + + impl SimdFloat for Simd<$ty, LANES> + where + LaneCount: SupportedLaneCount, + { + type Mask = Mask<<$mask_ty as SimdElement>::Mask, LANES>; + type Scalar = $ty; + type Bits = Simd<$bits_ty, LANES>; + + #[inline] + fn to_bits(self) -> Simd<$bits_ty, LANES> { + assert_eq!(core::mem::size_of::(), core::mem::size_of::()); + // Safety: transmuting between vector types is safe + unsafe { core::mem::transmute_copy(&self) } + } + + #[inline] + fn from_bits(bits: Simd<$bits_ty, LANES>) -> Self { + assert_eq!(core::mem::size_of::(), core::mem::size_of::()); + // Safety: transmuting between vector types is safe + unsafe { core::mem::transmute_copy(&bits) } + } + + #[inline] + fn abs(self) -> Self { + // Safety: `self` is a float vector + unsafe { intrinsics::simd_fabs(self) } + } + + #[inline] + fn recip(self) -> Self { + Self::splat(1.0) / self + } + + #[inline] + fn to_degrees(self) -> Self { + // to_degrees uses a special constant for better precision, so extract that constant + self * Self::splat(Self::Scalar::to_degrees(1.)) + } + + #[inline] + fn to_radians(self) -> Self { + self * Self::splat(Self::Scalar::to_radians(1.)) + } + + #[inline] + fn is_sign_positive(self) -> Self::Mask { + !self.is_sign_negative() + } + + #[inline] + fn is_sign_negative(self) -> Self::Mask { + let sign_bits = self.to_bits() & Simd::splat((!0 >> 1) + 1); + sign_bits.simd_gt(Simd::splat(0)) + } + + #[inline] + fn is_nan(self) -> Self::Mask { + self.simd_ne(self) + } + + #[inline] + fn is_infinite(self) -> Self::Mask { + self.abs().simd_eq(Self::splat(Self::Scalar::INFINITY)) + } + + #[inline] + fn is_finite(self) -> Self::Mask { + self.abs().simd_lt(Self::splat(Self::Scalar::INFINITY)) + } + + #[inline] + fn is_subnormal(self) -> Self::Mask { + self.abs().simd_ne(Self::splat(0.0)) & (self.to_bits() & Self::splat(Self::Scalar::INFINITY).to_bits()).simd_eq(Simd::splat(0)) + } + + #[inline] + #[must_use = "method returns a new mask and does not mutate the original value"] + fn is_normal(self) -> Self::Mask { + !(self.abs().simd_eq(Self::splat(0.0)) | self.is_nan() | self.is_subnormal() | self.is_infinite()) + } + + #[inline] + fn signum(self) -> Self { + self.is_nan().select(Self::splat(Self::Scalar::NAN), Self::splat(1.0).copysign(self)) + } + + #[inline] + fn copysign(self, sign: Self) -> Self { + let sign_bit = sign.to_bits() & Self::splat(-0.).to_bits(); + let magnitude = self.to_bits() & !Self::splat(-0.).to_bits(); + Self::from_bits(sign_bit | magnitude) + } + + #[inline] + fn simd_min(self, other: Self) -> Self { + // Safety: `self` and `other` are float vectors + unsafe { intrinsics::simd_fmin(self, other) } + } + + #[inline] + fn simd_max(self, other: Self) -> Self { + // Safety: `self` and `other` are floating point vectors + unsafe { intrinsics::simd_fmax(self, other) } + } + + #[inline] + fn simd_clamp(self, min: Self, max: Self) -> Self { + assert!( + min.simd_le(max).all(), + "each lane in `min` must be less than or equal to the corresponding lane in `max`", + ); + let mut x = self; + x = x.simd_lt(min).select(min, x); + x = x.simd_gt(max).select(max, x); + x + } + + #[inline] + fn reduce_sum(self) -> Self::Scalar { + // LLVM sum is inaccurate on i586 + if cfg!(all(target_arch = "x86", not(target_feature = "sse2"))) { + self.as_array().iter().sum() + } else { + // Safety: `self` is a float vector + unsafe { intrinsics::simd_reduce_add_ordered(self, 0.) } + } + } + + #[inline] + fn reduce_product(self) -> Self::Scalar { + // LLVM product is inaccurate on i586 + if cfg!(all(target_arch = "x86", not(target_feature = "sse2"))) { + self.as_array().iter().product() + } else { + // Safety: `self` is a float vector + unsafe { intrinsics::simd_reduce_mul_ordered(self, 1.) } + } + } + + #[inline] + fn reduce_max(self) -> Self::Scalar { + // Safety: `self` is a float vector + unsafe { intrinsics::simd_reduce_max(self) } + } + + #[inline] + fn reduce_min(self) -> Self::Scalar { + // Safety: `self` is a float vector + unsafe { intrinsics::simd_reduce_min(self) } + } + } + )* + } +} + +impl_trait! { f32 { bits: u32, mask: i32 }, f64 { bits: u64, mask: i64 } } diff --git a/library/portable-simd/crates/core_simd/src/elements/int.rs b/library/portable-simd/crates/core_simd/src/elements/int.rs new file mode 100644 index 00000000000..9b8c37ed466 --- /dev/null +++ b/library/portable-simd/crates/core_simd/src/elements/int.rs @@ -0,0 +1,298 @@ +use super::sealed::Sealed; +use crate::simd::{ + intrinsics, LaneCount, Mask, Simd, SimdElement, SimdPartialOrd, SupportedLaneCount, +}; + +/// Operations on SIMD vectors of signed integers. +pub trait SimdInt: Copy + Sealed { + /// Mask type used for manipulating this SIMD vector type. + type Mask; + + /// Scalar type contained by this SIMD vector type. + type Scalar; + + /// Lanewise saturating add. + /// + /// # Examples + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::{Simd, SimdInt}; + /// use core::i32::{MIN, MAX}; + /// let x = Simd::from_array([MIN, 0, 1, MAX]); + /// let max = Simd::splat(MAX); + /// let unsat = x + max; + /// let sat = x.saturating_add(max); + /// assert_eq!(unsat, Simd::from_array([-1, MAX, MIN, -2])); + /// assert_eq!(sat, Simd::from_array([-1, MAX, MAX, MAX])); + /// ``` + fn saturating_add(self, second: Self) -> Self; + + /// Lanewise saturating subtract. + /// + /// # Examples + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::{Simd, SimdInt}; + /// use core::i32::{MIN, MAX}; + /// let x = Simd::from_array([MIN, -2, -1, MAX]); + /// let max = Simd::splat(MAX); + /// let unsat = x - max; + /// let sat = x.saturating_sub(max); + /// assert_eq!(unsat, Simd::from_array([1, MAX, MIN, 0])); + /// assert_eq!(sat, Simd::from_array([MIN, MIN, MIN, 0])); + fn saturating_sub(self, second: Self) -> Self; + + /// Lanewise absolute value, implemented in Rust. + /// Every lane becomes its absolute value. + /// + /// # Examples + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::{Simd, SimdInt}; + /// use core::i32::{MIN, MAX}; + /// let xs = Simd::from_array([MIN, MIN +1, -5, 0]); + /// assert_eq!(xs.abs(), Simd::from_array([MIN, MAX, 5, 0])); + /// ``` + fn abs(self) -> Self; + + /// Lanewise saturating absolute value, implemented in Rust. + /// As abs(), except the MIN value becomes MAX instead of itself. + /// + /// # Examples + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::{Simd, SimdInt}; + /// use core::i32::{MIN, MAX}; + /// let xs = Simd::from_array([MIN, -2, 0, 3]); + /// let unsat = xs.abs(); + /// let sat = xs.saturating_abs(); + /// assert_eq!(unsat, Simd::from_array([MIN, 2, 0, 3])); + /// assert_eq!(sat, Simd::from_array([MAX, 2, 0, 3])); + /// ``` + fn saturating_abs(self) -> Self; + + /// Lanewise saturating negation, implemented in Rust. + /// As neg(), except the MIN value becomes MAX instead of itself. + /// + /// # Examples + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::{Simd, SimdInt}; + /// use core::i32::{MIN, MAX}; + /// let x = Simd::from_array([MIN, -2, 3, MAX]); + /// let unsat = -x; + /// let sat = x.saturating_neg(); + /// assert_eq!(unsat, Simd::from_array([MIN, 2, -3, MIN + 1])); + /// assert_eq!(sat, Simd::from_array([MAX, 2, -3, MIN + 1])); + /// ``` + fn saturating_neg(self) -> Self; + + /// Returns true for each positive lane and false if it is zero or negative. + fn is_positive(self) -> Self::Mask; + + /// Returns true for each negative lane and false if it is zero or positive. + fn is_negative(self) -> Self::Mask; + + /// Returns numbers representing the sign of each lane. + /// * `0` if the number is zero + /// * `1` if the number is positive + /// * `-1` if the number is negative + fn signum(self) -> Self; + + /// Returns the sum of the lanes of the vector, with wrapping addition. + /// + /// # Examples + /// + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::{i32x4, SimdInt}; + /// let v = i32x4::from_array([1, 2, 3, 4]); + /// assert_eq!(v.reduce_sum(), 10); + /// + /// // SIMD integer addition is always wrapping + /// let v = i32x4::from_array([i32::MAX, 1, 0, 0]); + /// assert_eq!(v.reduce_sum(), i32::MIN); + /// ``` + fn reduce_sum(self) -> Self::Scalar; + + /// Returns the product of the lanes of the vector, with wrapping multiplication. + /// + /// # Examples + /// + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::{i32x4, SimdInt}; + /// let v = i32x4::from_array([1, 2, 3, 4]); + /// assert_eq!(v.reduce_product(), 24); + /// + /// // SIMD integer multiplication is always wrapping + /// let v = i32x4::from_array([i32::MAX, 2, 1, 1]); + /// assert!(v.reduce_product() < i32::MAX); + /// ``` + fn reduce_product(self) -> Self::Scalar; + + /// Returns the maximum lane in the vector. + /// + /// # Examples + /// + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::{i32x4, SimdInt}; + /// let v = i32x4::from_array([1, 2, 3, 4]); + /// assert_eq!(v.reduce_max(), 4); + /// ``` + fn reduce_max(self) -> Self::Scalar; + + /// Returns the minimum lane in the vector. + /// + /// # Examples + /// + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::{i32x4, SimdInt}; + /// let v = i32x4::from_array([1, 2, 3, 4]); + /// assert_eq!(v.reduce_min(), 1); + /// ``` + fn reduce_min(self) -> Self::Scalar; + + /// Returns the cumulative bitwise "and" across the lanes of the vector. + fn reduce_and(self) -> Self::Scalar; + + /// Returns the cumulative bitwise "or" across the lanes of the vector. + fn reduce_or(self) -> Self::Scalar; + + /// Returns the cumulative bitwise "xor" across the lanes of the vector. + fn reduce_xor(self) -> Self::Scalar; +} + +macro_rules! impl_trait { + { $($ty:ty),* } => { + $( + impl Sealed for Simd<$ty, LANES> + where + LaneCount: SupportedLaneCount, + { + } + + impl SimdInt for Simd<$ty, LANES> + where + LaneCount: SupportedLaneCount, + { + type Mask = Mask<<$ty as SimdElement>::Mask, LANES>; + type Scalar = $ty; + + #[inline] + fn saturating_add(self, second: Self) -> Self { + // Safety: `self` is a vector + unsafe { intrinsics::simd_saturating_add(self, second) } + } + + #[inline] + fn saturating_sub(self, second: Self) -> Self { + // Safety: `self` is a vector + unsafe { intrinsics::simd_saturating_sub(self, second) } + } + + #[inline] + fn abs(self) -> Self { + const SHR: $ty = <$ty>::BITS as $ty - 1; + let m = self >> Simd::splat(SHR); + (self^m) - m + } + + #[inline] + fn saturating_abs(self) -> Self { + // arith shift for -1 or 0 mask based on sign bit, giving 2s complement + const SHR: $ty = <$ty>::BITS as $ty - 1; + let m = self >> Simd::splat(SHR); + (self^m).saturating_sub(m) + } + + #[inline] + fn saturating_neg(self) -> Self { + Self::splat(0).saturating_sub(self) + } + + #[inline] + fn is_positive(self) -> Self::Mask { + self.simd_gt(Self::splat(0)) + } + + #[inline] + fn is_negative(self) -> Self::Mask { + self.simd_lt(Self::splat(0)) + } + + #[inline] + fn signum(self) -> Self { + self.is_positive().select( + Self::splat(1), + self.is_negative().select(Self::splat(-1), Self::splat(0)) + ) + } + + #[inline] + fn reduce_sum(self) -> Self::Scalar { + // Safety: `self` is an integer vector + unsafe { intrinsics::simd_reduce_add_ordered(self, 0) } + } + + #[inline] + fn reduce_product(self) -> Self::Scalar { + // Safety: `self` is an integer vector + unsafe { intrinsics::simd_reduce_mul_ordered(self, 1) } + } + + #[inline] + fn reduce_max(self) -> Self::Scalar { + // Safety: `self` is an integer vector + unsafe { intrinsics::simd_reduce_max(self) } + } + + #[inline] + fn reduce_min(self) -> Self::Scalar { + // Safety: `self` is an integer vector + unsafe { intrinsics::simd_reduce_min(self) } + } + + #[inline] + fn reduce_and(self) -> Self::Scalar { + // Safety: `self` is an integer vector + unsafe { intrinsics::simd_reduce_and(self) } + } + + #[inline] + fn reduce_or(self) -> Self::Scalar { + // Safety: `self` is an integer vector + unsafe { intrinsics::simd_reduce_or(self) } + } + + #[inline] + fn reduce_xor(self) -> Self::Scalar { + // Safety: `self` is an integer vector + unsafe { intrinsics::simd_reduce_xor(self) } + } + } + )* + } +} + +impl_trait! { i8, i16, i32, i64, isize } diff --git a/library/portable-simd/crates/core_simd/src/elements/uint.rs b/library/portable-simd/crates/core_simd/src/elements/uint.rs new file mode 100644 index 00000000000..21e7e76eb3d --- /dev/null +++ b/library/portable-simd/crates/core_simd/src/elements/uint.rs @@ -0,0 +1,139 @@ +use super::sealed::Sealed; +use crate::simd::{intrinsics, LaneCount, Simd, SupportedLaneCount}; + +/// Operations on SIMD vectors of unsigned integers. +pub trait SimdUint: Copy + Sealed { + /// Scalar type contained by this SIMD vector type. + type Scalar; + + /// Lanewise saturating add. + /// + /// # Examples + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::{Simd, SimdUint}; + /// use core::u32::MAX; + /// let x = Simd::from_array([2, 1, 0, MAX]); + /// let max = Simd::splat(MAX); + /// let unsat = x + max; + /// let sat = x.saturating_add(max); + /// assert_eq!(unsat, Simd::from_array([1, 0, MAX, MAX - 1])); + /// assert_eq!(sat, max); + /// ``` + fn saturating_add(self, second: Self) -> Self; + + /// Lanewise saturating subtract. + /// + /// # Examples + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::{Simd, SimdUint}; + /// use core::u32::MAX; + /// let x = Simd::from_array([2, 1, 0, MAX]); + /// let max = Simd::splat(MAX); + /// let unsat = x - max; + /// let sat = x.saturating_sub(max); + /// assert_eq!(unsat, Simd::from_array([3, 2, 1, 0])); + /// assert_eq!(sat, Simd::splat(0)); + fn saturating_sub(self, second: Self) -> Self; + + /// Returns the sum of the lanes of the vector, with wrapping addition. + fn reduce_sum(self) -> Self::Scalar; + + /// Returns the product of the lanes of the vector, with wrapping multiplication. + fn reduce_product(self) -> Self::Scalar; + + /// Returns the maximum lane in the vector. + fn reduce_max(self) -> Self::Scalar; + + /// Returns the minimum lane in the vector. + fn reduce_min(self) -> Self::Scalar; + + /// Returns the cumulative bitwise "and" across the lanes of the vector. + fn reduce_and(self) -> Self::Scalar; + + /// Returns the cumulative bitwise "or" across the lanes of the vector. + fn reduce_or(self) -> Self::Scalar; + + /// Returns the cumulative bitwise "xor" across the lanes of the vector. + fn reduce_xor(self) -> Self::Scalar; +} + +macro_rules! impl_trait { + { $($ty:ty),* } => { + $( + impl Sealed for Simd<$ty, LANES> + where + LaneCount: SupportedLaneCount, + { + } + + impl SimdUint for Simd<$ty, LANES> + where + LaneCount: SupportedLaneCount, + { + type Scalar = $ty; + + #[inline] + fn saturating_add(self, second: Self) -> Self { + // Safety: `self` is a vector + unsafe { intrinsics::simd_saturating_add(self, second) } + } + + #[inline] + fn saturating_sub(self, second: Self) -> Self { + // Safety: `self` is a vector + unsafe { intrinsics::simd_saturating_sub(self, second) } + } + + #[inline] + fn reduce_sum(self) -> Self::Scalar { + // Safety: `self` is an integer vector + unsafe { intrinsics::simd_reduce_add_ordered(self, 0) } + } + + #[inline] + fn reduce_product(self) -> Self::Scalar { + // Safety: `self` is an integer vector + unsafe { intrinsics::simd_reduce_mul_ordered(self, 1) } + } + + #[inline] + fn reduce_max(self) -> Self::Scalar { + // Safety: `self` is an integer vector + unsafe { intrinsics::simd_reduce_max(self) } + } + + #[inline] + fn reduce_min(self) -> Self::Scalar { + // Safety: `self` is an integer vector + unsafe { intrinsics::simd_reduce_min(self) } + } + + #[inline] + fn reduce_and(self) -> Self::Scalar { + // Safety: `self` is an integer vector + unsafe { intrinsics::simd_reduce_and(self) } + } + + #[inline] + fn reduce_or(self) -> Self::Scalar { + // Safety: `self` is an integer vector + unsafe { intrinsics::simd_reduce_or(self) } + } + + #[inline] + fn reduce_xor(self) -> Self::Scalar { + // Safety: `self` is an integer vector + unsafe { intrinsics::simd_reduce_xor(self) } + } + } + )* + } +} + +impl_trait! { u8, u16, u32, u64, usize } diff --git a/library/portable-simd/crates/core_simd/src/eq.rs b/library/portable-simd/crates/core_simd/src/eq.rs new file mode 100644 index 00000000000..c7111f720a8 --- /dev/null +++ b/library/portable-simd/crates/core_simd/src/eq.rs @@ -0,0 +1,73 @@ +use crate::simd::{intrinsics, LaneCount, Mask, Simd, SimdElement, SupportedLaneCount}; + +/// Parallel `PartialEq`. +pub trait SimdPartialEq { + /// The mask type returned by each comparison. + type Mask; + + /// Test if each lane is equal to the corresponding lane in `other`. + #[must_use = "method returns a new mask and does not mutate the original value"] + fn simd_eq(self, other: Self) -> Self::Mask; + + /// Test if each lane is equal to the corresponding lane in `other`. + #[must_use = "method returns a new mask and does not mutate the original value"] + fn simd_ne(self, other: Self) -> Self::Mask; +} + +macro_rules! impl_number { + { $($number:ty),* } => { + $( + impl SimdPartialEq for Simd<$number, LANES> + where + LaneCount: SupportedLaneCount, + { + type Mask = Mask<<$number as SimdElement>::Mask, LANES>; + + #[inline] + fn simd_eq(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Mask::from_int_unchecked(intrinsics::simd_eq(self, other)) } + } + + #[inline] + fn simd_ne(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Mask::from_int_unchecked(intrinsics::simd_ne(self, other)) } + } + } + )* + } +} + +impl_number! { f32, f64, u8, u16, u32, u64, usize, i8, i16, i32, i64, isize } + +macro_rules! impl_mask { + { $($integer:ty),* } => { + $( + impl SimdPartialEq for Mask<$integer, LANES> + where + LaneCount: SupportedLaneCount, + { + type Mask = Self; + + #[inline] + fn simd_eq(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Self::from_int_unchecked(intrinsics::simd_eq(self.to_int(), other.to_int())) } + } + + #[inline] + fn simd_ne(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Self::from_int_unchecked(intrinsics::simd_ne(self.to_int(), other.to_int())) } + } + } + )* + } +} + +impl_mask! { i8, i16, i32, i64, isize } diff --git a/library/portable-simd/crates/core_simd/src/lane_count.rs b/library/portable-simd/crates/core_simd/src/lane_count.rs index 3b316f12b3e..63723e2ec13 100644 --- a/library/portable-simd/crates/core_simd/src/lane_count.rs +++ b/library/portable-simd/crates/core_simd/src/lane_count.rs @@ -3,7 +3,7 @@ mod sealed { } use sealed::Sealed; -/// A type representing a vector lane count. +/// Specifies the number of lanes in a SIMD vector as a type. pub struct LaneCount; impl LaneCount { @@ -11,7 +11,11 @@ impl LaneCount { pub const BITMASK_LEN: usize = (LANES + 7) / 8; } -/// Helper trait for vector lane counts. +/// Statically guarantees that a lane count is marked as supported. +/// +/// This trait is *sealed*: the list of implementors below is total. +/// Users do not have the ability to mark additional `LaneCount` values as supported. +/// Only SIMD vectors with supported lane counts are constructable. pub trait SupportedLaneCount: Sealed { #[doc(hidden)] type BitMask: Copy + Default + AsRef<[u8]> + AsMut<[u8]>; diff --git a/library/portable-simd/crates/core_simd/src/lib.rs b/library/portable-simd/crates/core_simd/src/lib.rs index 2632073622e..715f258f617 100644 --- a/library/portable-simd/crates/core_simd/src/lib.rs +++ b/library/portable-simd/crates/core_simd/src/lib.rs @@ -12,7 +12,7 @@ #![cfg_attr(feature = "generic_const_exprs", feature(generic_const_exprs))] #![cfg_attr(feature = "generic_const_exprs", allow(incomplete_features))] #![warn(missing_docs)] -#![deny(unsafe_op_in_unsafe_fn)] +#![deny(unsafe_op_in_unsafe_fn, clippy::undocumented_unsafe_blocks)] #![unstable(feature = "portable_simd", issue = "86656")] //! Portable SIMD module. diff --git a/library/portable-simd/crates/core_simd/src/masks.rs b/library/portable-simd/crates/core_simd/src/masks.rs index e1cd7930450..c36c336d8a2 100644 --- a/library/portable-simd/crates/core_simd/src/masks.rs +++ b/library/portable-simd/crates/core_simd/src/masks.rs @@ -15,7 +15,10 @@ mod mask_impl; mod to_bitmask; pub use to_bitmask::ToBitMask; -use crate::simd::{intrinsics, LaneCount, Simd, SimdElement, SupportedLaneCount}; +#[cfg(feature = "generic_const_exprs")] +pub use to_bitmask::{bitmask_len, ToBitMaskArray}; + +use crate::simd::{intrinsics, LaneCount, Simd, SimdElement, SimdPartialEq, SupportedLaneCount}; use core::cmp::Ordering; use core::{fmt, mem}; @@ -56,7 +59,7 @@ macro_rules! impl_element { where LaneCount: SupportedLaneCount, { - (value.lanes_eq(Simd::splat(0)) | value.lanes_eq(Simd::splat(-1))).all() + (value.simd_eq(Simd::splat(0 as _)) | value.simd_eq(Simd::splat(-1 as _))).all() } fn eq(self, other: Self) -> bool { self == other } @@ -65,6 +68,7 @@ macro_rules! impl_element { const FALSE: Self = 0; } + // Safety: this is a valid mask element type unsafe impl MaskElement for $ty {} } } @@ -77,6 +81,8 @@ impl_element! { isize } /// A SIMD vector mask for `LANES` elements of width specified by `Element`. /// +/// Masks represent boolean inclusion/exclusion on a per-lane basis. +/// /// The layout of this type is unspecified. #[repr(transparent)] pub struct Mask(mask_impl::Mask) @@ -179,6 +185,13 @@ where self.0.to_int() } + /// Converts the mask to a mask of any other lane size. + #[inline] + #[must_use = "method returns a new mask and does not mutate the original value"] + pub fn cast(self) -> Mask { + Mask(self.0.convert()) + } + /// Tests the value of the specified lane. /// /// # Safety @@ -507,58 +520,58 @@ where } } -/// Vector of eight 8-bit masks +/// A mask for SIMD vectors with eight elements of 8 bits. pub type mask8x8 = Mask; -/// Vector of 16 8-bit masks +/// A mask for SIMD vectors with 16 elements of 8 bits. pub type mask8x16 = Mask; -/// Vector of 32 8-bit masks +/// A mask for SIMD vectors with 32 elements of 8 bits. pub type mask8x32 = Mask; -/// Vector of 16 8-bit masks +/// A mask for SIMD vectors with 64 elements of 8 bits. pub type mask8x64 = Mask; -/// Vector of four 16-bit masks +/// A mask for SIMD vectors with four elements of 16 bits. pub type mask16x4 = Mask; -/// Vector of eight 16-bit masks +/// A mask for SIMD vectors with eight elements of 16 bits. pub type mask16x8 = Mask; -/// Vector of 16 16-bit masks +/// A mask for SIMD vectors with 16 elements of 16 bits. pub type mask16x16 = Mask; -/// Vector of 32 16-bit masks +/// A mask for SIMD vectors with 32 elements of 16 bits. pub type mask16x32 = Mask; -/// Vector of two 32-bit masks +/// A mask for SIMD vectors with two elements of 32 bits. pub type mask32x2 = Mask; -/// Vector of four 32-bit masks +/// A mask for SIMD vectors with four elements of 32 bits. pub type mask32x4 = Mask; -/// Vector of eight 32-bit masks +/// A mask for SIMD vectors with eight elements of 32 bits. pub type mask32x8 = Mask; -/// Vector of 16 32-bit masks +/// A mask for SIMD vectors with 16 elements of 32 bits. pub type mask32x16 = Mask; -/// Vector of two 64-bit masks +/// A mask for SIMD vectors with two elements of 64 bits. pub type mask64x2 = Mask; -/// Vector of four 64-bit masks +/// A mask for SIMD vectors with four elements of 64 bits. pub type mask64x4 = Mask; -/// Vector of eight 64-bit masks +/// A mask for SIMD vectors with eight elements of 64 bits. pub type mask64x8 = Mask; -/// Vector of two pointer-width masks +/// A mask for SIMD vectors with two elements of pointer width. pub type masksizex2 = Mask; -/// Vector of four pointer-width masks +/// A mask for SIMD vectors with four elements of pointer width. pub type masksizex4 = Mask; -/// Vector of eight pointer-width masks +/// A mask for SIMD vectors with eight elements of pointer width. pub type masksizex8 = Mask; macro_rules! impl_from { @@ -569,7 +582,7 @@ macro_rules! impl_from { LaneCount: SupportedLaneCount, { fn from(value: Mask<$from, LANES>) -> Self { - Self(value.0.convert()) + value.cast() } } )* diff --git a/library/portable-simd/crates/core_simd/src/masks/bitmask.rs b/library/portable-simd/crates/core_simd/src/masks/bitmask.rs index ec4dd357ee9..365ecc0a325 100644 --- a/library/portable-simd/crates/core_simd/src/masks/bitmask.rs +++ b/library/portable-simd/crates/core_simd/src/masks/bitmask.rs @@ -115,6 +115,26 @@ where unsafe { Self(intrinsics::simd_bitmask(value), PhantomData) } } + #[cfg(feature = "generic_const_exprs")] + #[inline] + #[must_use = "method returns a new array and does not mutate the original value"] + pub fn to_bitmask_array(self) -> [u8; N] { + assert!(core::mem::size_of::() == N); + + // Safety: converting an integer to an array of bytes of the same size is safe + unsafe { core::mem::transmute_copy(&self.0) } + } + + #[cfg(feature = "generic_const_exprs")] + #[inline] + #[must_use = "method returns a new mask and does not mutate the original value"] + pub fn from_bitmask_array(bitmask: [u8; N]) -> Self { + assert!(core::mem::size_of::() == N); + + // Safety: converting an array of bytes to an integer of the same size is safe + Self(unsafe { core::mem::transmute_copy(&bitmask) }, PhantomData) + } + #[inline] pub fn to_bitmask_integer(self) -> U where diff --git a/library/portable-simd/crates/core_simd/src/masks/full_masks.rs b/library/portable-simd/crates/core_simd/src/masks/full_masks.rs index 8bbdf637de8..adf0fcbeae2 100644 --- a/library/portable-simd/crates/core_simd/src/masks/full_masks.rs +++ b/library/portable-simd/crates/core_simd/src/masks/full_masks.rs @@ -4,6 +4,9 @@ use super::MaskElement; use crate::simd::intrinsics; use crate::simd::{LaneCount, Simd, SupportedLaneCount, ToBitMask}; +#[cfg(feature = "generic_const_exprs")] +use crate::simd::ToBitMaskArray; + #[repr(transparent)] pub struct Mask(Simd) where @@ -68,14 +71,26 @@ where // Used for bitmask bit order workaround pub(crate) trait ReverseBits { - fn reverse_bits(self) -> Self; + // Reverse the least significant `n` bits of `self`. + // (Remaining bits must be 0.) + fn reverse_bits(self, n: usize) -> Self; } macro_rules! impl_reverse_bits { { $($int:ty),* } => { $( impl ReverseBits for $int { - fn reverse_bits(self) -> Self { <$int>::reverse_bits(self) } + #[inline(always)] + fn reverse_bits(self, n: usize) -> Self { + let rev = <$int>::reverse_bits(self); + let bitsize = core::mem::size_of::<$int>() * 8; + if n < bitsize { + // Shift things back to the right + rev >> (bitsize - n) + } else { + rev + } + } } )* } @@ -127,6 +142,68 @@ where unsafe { Mask(intrinsics::simd_cast(self.0)) } } + #[cfg(feature = "generic_const_exprs")] + #[inline] + #[must_use = "method returns a new array and does not mutate the original value"] + pub fn to_bitmask_array(self) -> [u8; N] + where + super::Mask: ToBitMaskArray, + [(); as ToBitMaskArray>::BYTES]: Sized, + { + assert_eq!( as ToBitMaskArray>::BYTES, N); + + // Safety: N is the correct bitmask size + unsafe { + // Compute the bitmask + let bitmask: [u8; as ToBitMaskArray>::BYTES] = + intrinsics::simd_bitmask(self.0); + + // Transmute to the return type, previously asserted to be the same size + let mut bitmask: [u8; N] = core::mem::transmute_copy(&bitmask); + + // LLVM assumes bit order should match endianness + if cfg!(target_endian = "big") { + for x in bitmask.as_mut() { + *x = x.reverse_bits(); + } + }; + + bitmask + } + } + + #[cfg(feature = "generic_const_exprs")] + #[inline] + #[must_use = "method returns a new mask and does not mutate the original value"] + pub fn from_bitmask_array(mut bitmask: [u8; N]) -> Self + where + super::Mask: ToBitMaskArray, + [(); as ToBitMaskArray>::BYTES]: Sized, + { + assert_eq!( as ToBitMaskArray>::BYTES, N); + + // Safety: N is the correct bitmask size + unsafe { + // LLVM assumes bit order should match endianness + if cfg!(target_endian = "big") { + for x in bitmask.as_mut() { + *x = x.reverse_bits(); + } + } + + // Transmute to the bitmask type, previously asserted to be the same size + let bitmask: [u8; as ToBitMaskArray>::BYTES] = + core::mem::transmute_copy(&bitmask); + + // Compute the regular mask + Self::from_int_unchecked(intrinsics::simd_select_bitmask( + bitmask, + Self::splat(true).to_int(), + Self::splat(false).to_int(), + )) + } + } + #[inline] pub(crate) fn to_bitmask_integer(self) -> U where @@ -137,7 +214,7 @@ where // LLVM assumes bit order should match endianness if cfg!(target_endian = "big") { - bitmask.reverse_bits() + bitmask.reverse_bits(LANES) } else { bitmask } @@ -150,7 +227,7 @@ where { // LLVM assumes bit order should match endianness let bitmask = if cfg!(target_endian = "big") { - bitmask.reverse_bits() + bitmask.reverse_bits(LANES) } else { bitmask }; diff --git a/library/portable-simd/crates/core_simd/src/masks/to_bitmask.rs b/library/portable-simd/crates/core_simd/src/masks/to_bitmask.rs index c263f6a4eec..65d3ce9be65 100644 --- a/library/portable-simd/crates/core_simd/src/masks/to_bitmask.rs +++ b/library/portable-simd/crates/core_simd/src/masks/to_bitmask.rs @@ -16,11 +16,7 @@ where /// Converts masks to and from integer bitmasks. /// /// Each bit of the bitmask corresponds to a mask lane, starting with the LSB. -/// -/// # Safety -/// This trait is `unsafe` and sealed, since the `BitMask` type must match the number of lanes in -/// the mask. -pub unsafe trait ToBitMask: Sealed { +pub trait ToBitMask: Sealed { /// The integer bitmask type. type BitMask; @@ -31,10 +27,25 @@ pub unsafe trait ToBitMask: Sealed { fn from_bitmask(bitmask: Self::BitMask) -> Self; } +/// Converts masks to and from byte array bitmasks. +/// +/// Each bit of the bitmask corresponds to a mask lane, starting with the LSB of the first byte. +#[cfg(feature = "generic_const_exprs")] +pub trait ToBitMaskArray: Sealed { + /// The length of the bitmask array. + const BYTES: usize; + + /// Converts a mask to a bitmask. + fn to_bitmask_array(self) -> [u8; Self::BYTES]; + + /// Converts a bitmask to a mask. + fn from_bitmask_array(bitmask: [u8; Self::BYTES]) -> Self; +} + macro_rules! impl_integer_intrinsic { - { $(unsafe impl ToBitMask for Mask<_, $lanes:literal>)* } => { + { $(impl ToBitMask for Mask<_, $lanes:literal>)* } => { $( - unsafe impl ToBitMask for Mask { + impl ToBitMask for Mask { type BitMask = $int; fn to_bitmask(self) -> $int { @@ -50,11 +61,33 @@ macro_rules! impl_integer_intrinsic { } impl_integer_intrinsic! { - unsafe impl ToBitMask for Mask<_, 1> - unsafe impl ToBitMask for Mask<_, 2> - unsafe impl ToBitMask for Mask<_, 4> - unsafe impl ToBitMask for Mask<_, 8> - unsafe impl ToBitMask for Mask<_, 16> - unsafe impl ToBitMask for Mask<_, 32> - unsafe impl ToBitMask for Mask<_, 64> + impl ToBitMask for Mask<_, 1> + impl ToBitMask for Mask<_, 2> + impl ToBitMask for Mask<_, 4> + impl ToBitMask for Mask<_, 8> + impl ToBitMask for Mask<_, 16> + impl ToBitMask for Mask<_, 32> + impl ToBitMask for Mask<_, 64> +} + +/// Returns the minimum numnber of bytes in a bitmask with `lanes` lanes. +#[cfg(feature = "generic_const_exprs")] +pub const fn bitmask_len(lanes: usize) -> usize { + (lanes + 7) / 8 +} + +#[cfg(feature = "generic_const_exprs")] +impl ToBitMaskArray for Mask +where + LaneCount: SupportedLaneCount, +{ + const BYTES: usize = bitmask_len(LANES); + + fn to_bitmask_array(self) -> [u8; Self::BYTES] { + self.0.to_bitmask_array() + } + + fn from_bitmask_array(bitmask: [u8; Self::BYTES]) -> Self { + Mask(mask_impl::Mask::from_bitmask_array(bitmask)) + } } diff --git a/library/portable-simd/crates/core_simd/src/math.rs b/library/portable-simd/crates/core_simd/src/math.rs deleted file mode 100644 index 606021e983e..00000000000 --- a/library/portable-simd/crates/core_simd/src/math.rs +++ /dev/null @@ -1,156 +0,0 @@ -use crate::simd::intrinsics::{simd_saturating_add, simd_saturating_sub}; -use crate::simd::{LaneCount, Simd, SupportedLaneCount}; - -macro_rules! impl_uint_arith { - ($($ty:ty),+) => { - $( impl Simd<$ty, LANES> where LaneCount: SupportedLaneCount { - - /// Lanewise saturating add. - /// - /// # Examples - /// ``` - /// # #![feature(portable_simd)] - /// # use core::simd::Simd; - #[doc = concat!("# use core::", stringify!($ty), "::MAX;")] - /// let x = Simd::from_array([2, 1, 0, MAX]); - /// let max = Simd::splat(MAX); - /// let unsat = x + max; - /// let sat = x.saturating_add(max); - /// assert_eq!(unsat, Simd::from_array([1, 0, MAX, MAX - 1])); - /// assert_eq!(sat, max); - /// ``` - #[inline] - pub fn saturating_add(self, second: Self) -> Self { - // Safety: `self` is a vector - unsafe { simd_saturating_add(self, second) } - } - - /// Lanewise saturating subtract. - /// - /// # Examples - /// ``` - /// # #![feature(portable_simd)] - /// # use core::simd::Simd; - #[doc = concat!("# use core::", stringify!($ty), "::MAX;")] - /// let x = Simd::from_array([2, 1, 0, MAX]); - /// let max = Simd::splat(MAX); - /// let unsat = x - max; - /// let sat = x.saturating_sub(max); - /// assert_eq!(unsat, Simd::from_array([3, 2, 1, 0])); - /// assert_eq!(sat, Simd::splat(0)); - #[inline] - pub fn saturating_sub(self, second: Self) -> Self { - // Safety: `self` is a vector - unsafe { simd_saturating_sub(self, second) } - } - })+ - } -} - -macro_rules! impl_int_arith { - ($($ty:ty),+) => { - $( impl Simd<$ty, LANES> where LaneCount: SupportedLaneCount { - - /// Lanewise saturating add. - /// - /// # Examples - /// ``` - /// # #![feature(portable_simd)] - /// # use core::simd::Simd; - #[doc = concat!("# use core::", stringify!($ty), "::{MIN, MAX};")] - /// let x = Simd::from_array([MIN, 0, 1, MAX]); - /// let max = Simd::splat(MAX); - /// let unsat = x + max; - /// let sat = x.saturating_add(max); - /// assert_eq!(unsat, Simd::from_array([-1, MAX, MIN, -2])); - /// assert_eq!(sat, Simd::from_array([-1, MAX, MAX, MAX])); - /// ``` - #[inline] - pub fn saturating_add(self, second: Self) -> Self { - // Safety: `self` is a vector - unsafe { simd_saturating_add(self, second) } - } - - /// Lanewise saturating subtract. - /// - /// # Examples - /// ``` - /// # #![feature(portable_simd)] - /// # use core::simd::Simd; - #[doc = concat!("# use core::", stringify!($ty), "::{MIN, MAX};")] - /// let x = Simd::from_array([MIN, -2, -1, MAX]); - /// let max = Simd::splat(MAX); - /// let unsat = x - max; - /// let sat = x.saturating_sub(max); - /// assert_eq!(unsat, Simd::from_array([1, MAX, MIN, 0])); - /// assert_eq!(sat, Simd::from_array([MIN, MIN, MIN, 0])); - #[inline] - pub fn saturating_sub(self, second: Self) -> Self { - // Safety: `self` is a vector - unsafe { simd_saturating_sub(self, second) } - } - - /// Lanewise absolute value, implemented in Rust. - /// Every lane becomes its absolute value. - /// - /// # Examples - /// ``` - /// # #![feature(portable_simd)] - /// # use core::simd::Simd; - #[doc = concat!("# use core::", stringify!($ty), "::{MIN, MAX};")] - /// let xs = Simd::from_array([MIN, MIN +1, -5, 0]); - /// assert_eq!(xs.abs(), Simd::from_array([MIN, MAX, 5, 0])); - /// ``` - #[inline] - pub fn abs(self) -> Self { - const SHR: $ty = <$ty>::BITS as $ty - 1; - let m = self >> Simd::splat(SHR); - (self^m) - m - } - - /// Lanewise saturating absolute value, implemented in Rust. - /// As abs(), except the MIN value becomes MAX instead of itself. - /// - /// # Examples - /// ``` - /// # #![feature(portable_simd)] - /// # use core::simd::Simd; - #[doc = concat!("# use core::", stringify!($ty), "::{MIN, MAX};")] - /// let xs = Simd::from_array([MIN, -2, 0, 3]); - /// let unsat = xs.abs(); - /// let sat = xs.saturating_abs(); - /// assert_eq!(unsat, Simd::from_array([MIN, 2, 0, 3])); - /// assert_eq!(sat, Simd::from_array([MAX, 2, 0, 3])); - /// ``` - #[inline] - pub fn saturating_abs(self) -> Self { - // arith shift for -1 or 0 mask based on sign bit, giving 2s complement - const SHR: $ty = <$ty>::BITS as $ty - 1; - let m = self >> Simd::splat(SHR); - (self^m).saturating_sub(m) - } - - /// Lanewise saturating negation, implemented in Rust. - /// As neg(), except the MIN value becomes MAX instead of itself. - /// - /// # Examples - /// ``` - /// # #![feature(portable_simd)] - /// # use core::simd::Simd; - #[doc = concat!("# use core::", stringify!($ty), "::{MIN, MAX};")] - /// let x = Simd::from_array([MIN, -2, 3, MAX]); - /// let unsat = -x; - /// let sat = x.saturating_neg(); - /// assert_eq!(unsat, Simd::from_array([MIN, 2, -3, MIN + 1])); - /// assert_eq!(sat, Simd::from_array([MAX, 2, -3, MIN + 1])); - /// ``` - #[inline] - pub fn saturating_neg(self) -> Self { - Self::splat(0).saturating_sub(self) - } - })+ - } -} - -impl_uint_arith! { u8, u16, u32, u64, usize } -impl_int_arith! { i8, i16, i32, i64, isize } diff --git a/library/portable-simd/crates/core_simd/src/mod.rs b/library/portable-simd/crates/core_simd/src/mod.rs index 85026265956..b472aa3abe2 100644 --- a/library/portable-simd/crates/core_simd/src/mod.rs +++ b/library/portable-simd/crates/core_simd/src/mod.rs @@ -1,6 +1,3 @@ -#[macro_use] -mod reduction; - #[macro_use] mod swizzle; @@ -9,14 +6,14 @@ pub(crate) mod intrinsics; #[cfg(feature = "generic_const_exprs")] mod to_bytes; -mod comparisons; +mod elements; +mod eq; mod fmt; mod iter; mod lane_count; mod masks; -mod math; mod ops; -mod round; +mod ord; mod select; mod vector; mod vendor; @@ -25,8 +22,11 @@ mod vendor; pub mod simd { pub(crate) use crate::core_simd::intrinsics; + pub use crate::core_simd::elements::*; + pub use crate::core_simd::eq::*; pub use crate::core_simd::lane_count::{LaneCount, SupportedLaneCount}; pub use crate::core_simd::masks::*; + pub use crate::core_simd::ord::*; pub use crate::core_simd::swizzle::*; pub use crate::core_simd::vector::*; } diff --git a/library/portable-simd/crates/core_simd/src/ops.rs b/library/portable-simd/crates/core_simd/src/ops.rs index 1b35b3e717a..5a077a469d8 100644 --- a/library/portable-simd/crates/core_simd/src/ops.rs +++ b/library/portable-simd/crates/core_simd/src/ops.rs @@ -1,4 +1,4 @@ -use crate::simd::{LaneCount, Simd, SimdElement, SupportedLaneCount}; +use crate::simd::{LaneCount, Simd, SimdElement, SimdPartialEq, SupportedLaneCount}; use core::ops::{Add, Mul}; use core::ops::{BitAnd, BitOr, BitXor}; use core::ops::{Div, Rem, Sub}; @@ -33,6 +33,7 @@ where macro_rules! unsafe_base { ($lhs:ident, $rhs:ident, {$simd_call:ident}, $($_:tt)*) => { + // Safety: $lhs and $rhs are vectors unsafe { $crate::simd::intrinsics::$simd_call($lhs, $rhs) } }; } @@ -48,6 +49,8 @@ macro_rules! unsafe_base { // cg_clif defaults to this, and scalar MIR shifts also default to wrapping macro_rules! wrap_bitshift { ($lhs:ident, $rhs:ident, {$simd_call:ident}, $int:ident) => { + #[allow(clippy::suspicious_arithmetic_impl)] + // Safety: $lhs and the bitand result are vectors unsafe { $crate::simd::intrinsics::$simd_call( $lhs, @@ -74,7 +77,7 @@ macro_rules! int_divrem_guard { $simd_call:ident }, $int:ident ) => { - if $rhs.lanes_eq(Simd::splat(0)).any() { + if $rhs.simd_eq(Simd::splat(0 as _)).any() { panic!($zero); } else { // Prevent otherwise-UB overflow on the MIN / -1 case. @@ -82,14 +85,15 @@ macro_rules! int_divrem_guard { // This should, at worst, optimize to a few branchless logical ops // Ideally, this entire conditional should evaporate // Fire LLVM and implement those manually if it doesn't get the hint - ($lhs.lanes_eq(Simd::splat(<$int>::MIN)) + ($lhs.simd_eq(Simd::splat(<$int>::MIN)) // type inference can break here, so cut an SInt to size - & $rhs.lanes_eq(Simd::splat(-1i64 as _))) - .select(Simd::splat(1), $rhs) + & $rhs.simd_eq(Simd::splat(-1i64 as _))) + .select(Simd::splat(1 as _), $rhs) } else { // Nice base case to make it easy to const-fold away the other branch. $rhs }; + // Safety: $lhs and rhs are vectors unsafe { $crate::simd::intrinsics::$simd_call($lhs, rhs) } } }; diff --git a/library/portable-simd/crates/core_simd/src/ops/unary.rs b/library/portable-simd/crates/core_simd/src/ops/unary.rs index 4ebea560fc6..4ad02215034 100644 --- a/library/portable-simd/crates/core_simd/src/ops/unary.rs +++ b/library/portable-simd/crates/core_simd/src/ops/unary.rs @@ -14,6 +14,7 @@ macro_rules! neg { #[inline] #[must_use = "operator returns a new vector without mutating the input"] fn neg(self) -> Self::Output { + // Safety: `self` is a signed vector unsafe { intrinsics::simd_neg(self) } } })* diff --git a/library/portable-simd/crates/core_simd/src/ord.rs b/library/portable-simd/crates/core_simd/src/ord.rs new file mode 100644 index 00000000000..9a87bc2e344 --- /dev/null +++ b/library/portable-simd/crates/core_simd/src/ord.rs @@ -0,0 +1,213 @@ +use crate::simd::{intrinsics, LaneCount, Mask, Simd, SimdPartialEq, SupportedLaneCount}; + +/// Parallel `PartialOrd`. +pub trait SimdPartialOrd: SimdPartialEq { + /// Test if each lane is less than the corresponding lane in `other`. + #[must_use = "method returns a new mask and does not mutate the original value"] + fn simd_lt(self, other: Self) -> Self::Mask; + + /// Test if each lane is less than or equal to the corresponding lane in `other`. + #[must_use = "method returns a new mask and does not mutate the original value"] + fn simd_le(self, other: Self) -> Self::Mask; + + /// Test if each lane is greater than the corresponding lane in `other`. + #[must_use = "method returns a new mask and does not mutate the original value"] + fn simd_gt(self, other: Self) -> Self::Mask; + + /// Test if each lane is greater than or equal to the corresponding lane in `other`. + #[must_use = "method returns a new mask and does not mutate the original value"] + fn simd_ge(self, other: Self) -> Self::Mask; +} + +/// Parallel `Ord`. +pub trait SimdOrd: SimdPartialOrd { + /// Returns the lane-wise maximum with `other`. + #[must_use = "method returns a new vector and does not mutate the original value"] + fn simd_max(self, other: Self) -> Self; + + /// Returns the lane-wise minimum with `other`. + #[must_use = "method returns a new vector and does not mutate the original value"] + fn simd_min(self, other: Self) -> Self; + + /// Restrict each lane to a certain interval. + /// + /// For each lane, returns `max` if `self` is greater than `max`, and `min` if `self` is + /// less than `min`. Otherwise returns `self`. + /// + /// # Panics + /// + /// Panics if `min > max` on any lane. + #[must_use = "method returns a new vector and does not mutate the original value"] + fn simd_clamp(self, min: Self, max: Self) -> Self; +} + +macro_rules! impl_integer { + { $($integer:ty),* } => { + $( + impl SimdPartialOrd for Simd<$integer, LANES> + where + LaneCount: SupportedLaneCount, + { + #[inline] + fn simd_lt(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Mask::from_int_unchecked(intrinsics::simd_lt(self, other)) } + } + + #[inline] + fn simd_le(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Mask::from_int_unchecked(intrinsics::simd_le(self, other)) } + } + + #[inline] + fn simd_gt(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Mask::from_int_unchecked(intrinsics::simd_gt(self, other)) } + } + + #[inline] + fn simd_ge(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Mask::from_int_unchecked(intrinsics::simd_ge(self, other)) } + } + } + + impl SimdOrd for Simd<$integer, LANES> + where + LaneCount: SupportedLaneCount, + { + #[inline] + fn simd_max(self, other: Self) -> Self { + self.simd_lt(other).select(other, self) + } + + #[inline] + fn simd_min(self, other: Self) -> Self { + self.simd_gt(other).select(other, self) + } + + #[inline] + fn simd_clamp(self, min: Self, max: Self) -> Self { + assert!( + min.simd_le(max).all(), + "each lane in `min` must be less than or equal to the corresponding lane in `max`", + ); + self.simd_max(min).simd_min(max) + } + } + )* + } +} + +impl_integer! { u8, u16, u32, u64, usize, i8, i16, i32, i64, isize } + +macro_rules! impl_float { + { $($float:ty),* } => { + $( + impl SimdPartialOrd for Simd<$float, LANES> + where + LaneCount: SupportedLaneCount, + { + #[inline] + fn simd_lt(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Mask::from_int_unchecked(intrinsics::simd_lt(self, other)) } + } + + #[inline] + fn simd_le(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Mask::from_int_unchecked(intrinsics::simd_le(self, other)) } + } + + #[inline] + fn simd_gt(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Mask::from_int_unchecked(intrinsics::simd_gt(self, other)) } + } + + #[inline] + fn simd_ge(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Mask::from_int_unchecked(intrinsics::simd_ge(self, other)) } + } + } + )* + } +} + +impl_float! { f32, f64 } + +macro_rules! impl_mask { + { $($integer:ty),* } => { + $( + impl SimdPartialOrd for Mask<$integer, LANES> + where + LaneCount: SupportedLaneCount, + { + #[inline] + fn simd_lt(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Self::from_int_unchecked(intrinsics::simd_lt(self.to_int(), other.to_int())) } + } + + #[inline] + fn simd_le(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Self::from_int_unchecked(intrinsics::simd_le(self.to_int(), other.to_int())) } + } + + #[inline] + fn simd_gt(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Self::from_int_unchecked(intrinsics::simd_gt(self.to_int(), other.to_int())) } + } + + #[inline] + fn simd_ge(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Self::from_int_unchecked(intrinsics::simd_ge(self.to_int(), other.to_int())) } + } + } + + impl SimdOrd for Mask<$integer, LANES> + where + LaneCount: SupportedLaneCount, + { + #[inline] + fn simd_max(self, other: Self) -> Self { + self.simd_gt(other).select_mask(other, self) + } + + #[inline] + fn simd_min(self, other: Self) -> Self { + self.simd_lt(other).select_mask(other, self) + } + + #[inline] + fn simd_clamp(self, min: Self, max: Self) -> Self { + assert!( + min.simd_le(max).all(), + "each lane in `min` must be less than or equal to the corresponding lane in `max`", + ); + self.simd_max(min).simd_min(max) + } + } + )* + } +} + +impl_mask! { i8, i16, i32, i64, isize } diff --git a/library/portable-simd/crates/core_simd/src/reduction.rs b/library/portable-simd/crates/core_simd/src/reduction.rs deleted file mode 100644 index 3177fd167fc..00000000000 --- a/library/portable-simd/crates/core_simd/src/reduction.rs +++ /dev/null @@ -1,153 +0,0 @@ -use crate::simd::intrinsics::{ - simd_reduce_add_ordered, simd_reduce_and, simd_reduce_max, simd_reduce_min, - simd_reduce_mul_ordered, simd_reduce_or, simd_reduce_xor, -}; -use crate::simd::{LaneCount, Simd, SimdElement, SupportedLaneCount}; -use core::ops::{BitAnd, BitOr, BitXor}; - -macro_rules! impl_integer_reductions { - { $scalar:ty } => { - impl Simd<$scalar, LANES> - where - LaneCount: SupportedLaneCount, - { - /// Reducing wrapping add. Returns the sum of the lanes of the vector, with wrapping addition. - #[inline] - pub fn reduce_sum(self) -> $scalar { - // Safety: `self` is an integer vector - unsafe { simd_reduce_add_ordered(self, 0) } - } - - /// Reducing wrapping multiply. Returns the product of the lanes of the vector, with wrapping multiplication. - #[inline] - pub fn reduce_product(self) -> $scalar { - // Safety: `self` is an integer vector - unsafe { simd_reduce_mul_ordered(self, 1) } - } - - /// Reducing maximum. Returns the maximum lane in the vector. - #[inline] - pub fn reduce_max(self) -> $scalar { - // Safety: `self` is an integer vector - unsafe { simd_reduce_max(self) } - } - - /// Reducing minimum. Returns the minimum lane in the vector. - #[inline] - pub fn reduce_min(self) -> $scalar { - // Safety: `self` is an integer vector - unsafe { simd_reduce_min(self) } - } - } - } -} - -impl_integer_reductions! { i8 } -impl_integer_reductions! { i16 } -impl_integer_reductions! { i32 } -impl_integer_reductions! { i64 } -impl_integer_reductions! { isize } -impl_integer_reductions! { u8 } -impl_integer_reductions! { u16 } -impl_integer_reductions! { u32 } -impl_integer_reductions! { u64 } -impl_integer_reductions! { usize } - -macro_rules! impl_float_reductions { - { $scalar:ty } => { - impl Simd<$scalar, LANES> - where - LaneCount: SupportedLaneCount, - { - - /// Reducing add. Returns the sum of the lanes of the vector. - #[inline] - pub fn reduce_sum(self) -> $scalar { - // LLVM sum is inaccurate on i586 - if cfg!(all(target_arch = "x86", not(target_feature = "sse2"))) { - self.as_array().iter().sum() - } else { - // Safety: `self` is a float vector - unsafe { simd_reduce_add_ordered(self, 0.) } - } - } - - /// Reducing multiply. Returns the product of the lanes of the vector. - #[inline] - pub fn reduce_product(self) -> $scalar { - // LLVM product is inaccurate on i586 - if cfg!(all(target_arch = "x86", not(target_feature = "sse2"))) { - self.as_array().iter().product() - } else { - // Safety: `self` is a float vector - unsafe { simd_reduce_mul_ordered(self, 1.) } - } - } - - /// Reducing maximum. Returns the maximum lane in the vector. - /// - /// Returns values based on equality, so a vector containing both `0.` and `-0.` may - /// return either. This function will not return `NaN` unless all lanes are `NaN`. - #[inline] - pub fn reduce_max(self) -> $scalar { - // Safety: `self` is a float vector - unsafe { simd_reduce_max(self) } - } - - /// Reducing minimum. Returns the minimum lane in the vector. - /// - /// Returns values based on equality, so a vector containing both `0.` and `-0.` may - /// return either. This function will not return `NaN` unless all lanes are `NaN`. - #[inline] - pub fn reduce_min(self) -> $scalar { - // Safety: `self` is a float vector - unsafe { simd_reduce_min(self) } - } - } - } -} - -impl_float_reductions! { f32 } -impl_float_reductions! { f64 } - -impl Simd -where - Self: BitAnd, - T: SimdElement + BitAnd, - LaneCount: SupportedLaneCount, -{ - /// Reducing bitwise "and". Returns the cumulative bitwise "and" across the lanes of - /// the vector. - #[inline] - pub fn reduce_and(self) -> T { - unsafe { simd_reduce_and(self) } - } -} - -impl Simd -where - Self: BitOr, - T: SimdElement + BitOr, - LaneCount: SupportedLaneCount, -{ - /// Reducing bitwise "or". Returns the cumulative bitwise "or" across the lanes of - /// the vector. - #[inline] - pub fn reduce_or(self) -> T { - unsafe { simd_reduce_or(self) } - } -} - -impl Simd -where - Self: BitXor, - T: SimdElement + BitXor, - LaneCount: SupportedLaneCount, -{ - /// Reducing bitwise "xor". Returns the cumulative bitwise "xor" across the lanes of - /// the vector. - #[inline] - pub fn reduce_xor(self) -> T { - unsafe { simd_reduce_xor(self) } - } -} diff --git a/library/portable-simd/crates/core_simd/src/round.rs b/library/portable-simd/crates/core_simd/src/round.rs deleted file mode 100644 index 556bc2cc1fe..00000000000 --- a/library/portable-simd/crates/core_simd/src/round.rs +++ /dev/null @@ -1,40 +0,0 @@ -use crate::simd::intrinsics; -use crate::simd::{LaneCount, Simd, SimdElement, SupportedLaneCount}; -use core::convert::FloatToInt; - -macro_rules! implement { - { - $type:ty - } => { - impl Simd<$type, LANES> - where - LaneCount: SupportedLaneCount, - { - /// Rounds toward zero and converts to the same-width integer type, assuming that - /// the value is finite and fits in that type. - /// - /// # Safety - /// The value must: - /// - /// * Not be NaN - /// * Not be infinite - /// * Be representable in the return type, after truncating off its fractional part - /// - /// If these requirements are infeasible or costly, consider using the safe function [cast], - /// which saturates on conversion. - /// - /// [cast]: Simd::cast - #[inline] - pub unsafe fn to_int_unchecked(self) -> Simd - where - $type: FloatToInt, - I: SimdElement, - { - unsafe { intrinsics::simd_cast(self) } - } - } - } -} - -implement! { f32 } -implement! { f64 } diff --git a/library/portable-simd/crates/core_simd/src/swizzle.rs b/library/portable-simd/crates/core_simd/src/swizzle.rs index ef47c4f3a4c..22999d24950 100644 --- a/library/portable-simd/crates/core_simd/src/swizzle.rs +++ b/library/portable-simd/crates/core_simd/src/swizzle.rs @@ -1,44 +1,46 @@ use crate::simd::intrinsics; use crate::simd::{LaneCount, Simd, SimdElement, SupportedLaneCount}; -/// Constructs a new vector by selecting values from the lanes of the source vector or vectors to use. +/// Constructs a new SIMD vector by copying elements from selected lanes in other vectors. /// -/// When swizzling one vector, the indices of the result vector are indicated by a `const` array -/// of `usize`, like [`Swizzle`]. -/// When swizzling two vectors, the indices are indicated by a `const` array of [`Which`], like -/// [`Swizzle2`]. +/// When swizzling one vector, lanes are selected by a `const` array of `usize`, +/// like [`Swizzle`]. +/// +/// When swizzling two vectors, lanes are selected by a `const` array of [`Which`], +/// like [`Swizzle2`]. /// /// # Examples -/// ## One source vector +/// +/// With a single SIMD vector, the const array specifies lane indices in that vector: /// ``` /// # #![feature(portable_simd)] -/// # use core::simd::{Simd, simd_swizzle}; -/// let v = Simd::::from_array([0., 1., 2., 3.]); +/// # use core::simd::{u32x2, u32x4, simd_swizzle}; +/// let v = u32x4::from_array([10, 11, 12, 13]); /// /// // Keeping the same size -/// let r = simd_swizzle!(v, [3, 0, 1, 2]); -/// assert_eq!(r.to_array(), [3., 0., 1., 2.]); +/// let r: u32x4 = simd_swizzle!(v, [3, 0, 1, 2]); +/// assert_eq!(r.to_array(), [13, 10, 11, 12]); /// /// // Changing the number of lanes -/// let r = simd_swizzle!(v, [3, 1]); -/// assert_eq!(r.to_array(), [3., 1.]); +/// let r: u32x2 = simd_swizzle!(v, [3, 1]); +/// assert_eq!(r.to_array(), [13, 11]); /// ``` /// -/// ## Two source vectors +/// With two input SIMD vectors, the const array uses `Which` to specify the source of each index: /// ``` /// # #![feature(portable_simd)] -/// # use core::simd::{Simd, simd_swizzle, Which}; -/// use Which::*; -/// let a = Simd::::from_array([0., 1., 2., 3.]); -/// let b = Simd::::from_array([4., 5., 6., 7.]); +/// # use core::simd::{u32x2, u32x4, simd_swizzle, Which}; +/// use Which::{First, Second}; +/// let a = u32x4::from_array([0, 1, 2, 3]); +/// let b = u32x4::from_array([4, 5, 6, 7]); /// /// // Keeping the same size -/// let r = simd_swizzle!(a, b, [First(0), First(1), Second(2), Second(3)]); -/// assert_eq!(r.to_array(), [0., 1., 6., 7.]); +/// let r: u32x4 = simd_swizzle!(a, b, [First(0), First(1), Second(2), Second(3)]); +/// assert_eq!(r.to_array(), [0, 1, 6, 7]); /// /// // Changing the number of lanes -/// let r = simd_swizzle!(a, b, [First(0), Second(0)]); -/// assert_eq!(r.to_array(), [0., 4.]); +/// let r: u32x2 = simd_swizzle!(a, b, [First(0), Second(0)]); +/// assert_eq!(r.to_array(), [0, 4]); /// ``` #[allow(unused_macros)] pub macro simd_swizzle { @@ -68,12 +70,14 @@ pub macro simd_swizzle { } } -/// An index into one of two vectors. +/// Specifies a lane index into one of two SIMD vectors. +/// +/// This is an input type for [Swizzle2] and helper macros like [simd_swizzle]. #[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum Which { - /// Indexes the first vector. + /// Index of a lane in the first input SIMD vector. First(usize), - /// Indexes the second vector. + /// Index of a lane in the second input SIMD vector. Second(usize), } diff --git a/library/portable-simd/crates/core_simd/src/vector.rs b/library/portable-simd/crates/core_simd/src/vector.rs index b9cd2e2021e..78f56402eba 100644 --- a/library/portable-simd/crates/core_simd/src/vector.rs +++ b/library/portable-simd/crates/core_simd/src/vector.rs @@ -9,8 +9,9 @@ pub use uint::*; // Vectors of pointers are not for public use at the current time. pub(crate) mod ptr; -use crate::simd::intrinsics; -use crate::simd::{LaneCount, Mask, MaskElement, SupportedLaneCount}; +use crate::simd::{ + intrinsics, LaneCount, Mask, MaskElement, SimdPartialOrd, SupportedLaneCount, Swizzle, +}; /// A SIMD vector of `LANES` elements of type `T`. `Simd` has the same shape as [`[T; N]`](array), but operates like `T`. /// @@ -99,17 +100,50 @@ where /// Number of lanes in this vector. pub const LANES: usize = LANES; - /// Get the number of lanes in this vector. + /// Returns the number of lanes in this SIMD vector. + /// + /// # Examples + /// + /// ``` + /// # #![feature(portable_simd)] + /// # use core::simd::u32x4; + /// let v = u32x4::splat(0); + /// assert_eq!(v.lanes(), 4); + /// ``` pub const fn lanes(&self) -> usize { LANES } - /// Construct a SIMD vector by setting all lanes to the given value. - pub const fn splat(value: T) -> Self { - Self([value; LANES]) + /// Constructs a new SIMD vector with all lanes set to the given value. + /// + /// # Examples + /// + /// ``` + /// # #![feature(portable_simd)] + /// # use core::simd::u32x4; + /// let v = u32x4::splat(8); + /// assert_eq!(v.as_array(), &[8, 8, 8, 8]); + /// ``` + pub fn splat(value: T) -> Self { + // This is preferred over `[value; LANES]`, since it's explicitly a splat: + // https://github.com/rust-lang/rust/issues/97804 + struct Splat; + impl Swizzle<1, LANES> for Splat { + const INDEX: [usize; LANES] = [0; LANES]; + } + Splat::swizzle(Simd::::from([value])) } /// Returns an array reference containing the entire SIMD vector. + /// + /// # Examples + /// + /// ``` + /// # #![feature(portable_simd)] + /// # use core::simd::{Simd, u64x4}; + /// let v: u64x4 = Simd::from_array([0, 1, 2, 3]); + /// assert_eq!(v.as_array(), &[0, 1, 2, 3]); + /// ``` pub const fn as_array(&self) -> &[T; LANES] { &self.0 } @@ -129,9 +163,21 @@ where self.0 } - /// Converts a slice to a SIMD vector containing `slice[..LANES]` + /// Converts a slice to a SIMD vector containing `slice[..LANES]`. + /// /// # Panics - /// `from_slice` will panic if the slice's `len` is less than the vector's `Simd::LANES`. + /// + /// Panics if the slice's length is less than the vector's `Simd::LANES`. + /// + /// # Examples + /// + /// ``` + /// # #![feature(portable_simd)] + /// # use core::simd::u32x4; + /// let source = vec![1, 2, 3, 4, 5, 6]; + /// let v = u32x4::from_slice(&source); + /// assert_eq!(v.as_array(), &[1, 2, 3, 4]); + /// ``` #[must_use] pub const fn from_slice(slice: &[T]) -> Self { assert!(slice.len() >= LANES, "slice length must be at least the number of lanes"); @@ -145,6 +191,7 @@ where } /// Performs lanewise conversion of a SIMD vector's elements to another SIMD-valid type. + /// /// This follows the semantics of Rust's `as` conversion for casting /// integers to unsigned integers (interpreting as the other type, so `-1` to `MAX`), /// and from floats to integers (truncating, or saturating at the limits) for each lane, @@ -169,10 +216,35 @@ where #[must_use] #[inline] pub fn cast(self) -> Simd { - // Safety: The input argument is a vector of a known SIMD type. + // Safety: The input argument is a vector of a valid SIMD element type. unsafe { intrinsics::simd_as(self) } } + /// Rounds toward zero and converts to the same-width integer type, assuming that + /// the value is finite and fits in that type. + /// + /// # Safety + /// The value must: + /// + /// * Not be NaN + /// * Not be infinite + /// * Be representable in the return type, after truncating off its fractional part + /// + /// If these requirements are infeasible or costly, consider using the safe function [cast], + /// which saturates on conversion. + /// + /// [cast]: Simd::cast + #[inline] + pub unsafe fn to_int_unchecked(self) -> Simd + where + T: core::convert::FloatToInt, + I: SimdElement, + { + // Safety: `self` is a vector, and `FloatToInt` ensures the type can be casted to + // an integer. + unsafe { intrinsics::simd_cast(self) } + } + /// Reads from potentially discontiguous indices in `slice` to construct a SIMD vector. /// If an index is out-of-bounds, the lane is instead selected from the `or` vector. /// @@ -239,7 +311,7 @@ where idxs: Simd, or: Self, ) -> Self { - let enable: Mask = enable & idxs.lanes_lt(Simd::splat(slice.len())); + let enable: Mask = enable & idxs.simd_lt(Simd::splat(slice.len())); // Safety: We have masked-off out-of-bounds lanes. unsafe { Self::gather_select_unchecked(slice, enable, idxs, or) } } @@ -256,13 +328,15 @@ where /// # Examples /// ``` /// # #![feature(portable_simd)] - /// # use core::simd::{Simd, Mask}; + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::{Simd, SimdPartialOrd, Mask}; /// let vec: Vec = vec![10, 11, 12, 13, 14, 15, 16, 17, 18]; /// let idxs = Simd::from_array([9, 3, 0, 5]); /// let alt = Simd::from_array([-5, -4, -3, -2]); /// let enable = Mask::from_array([true, true, true, false]); // Note the final mask lane. /// // If this mask was used to gather, it would be unsound. Let's fix that. - /// let enable = enable & idxs.lanes_lt(Simd::splat(vec.len())); + /// let enable = enable & idxs.simd_lt(Simd::splat(vec.len())); /// /// // We have masked the OOB lane, so it's safe to gather now. /// let result = unsafe { Simd::gather_select_unchecked(&vec, enable, idxs, alt) }; @@ -313,7 +387,9 @@ where /// # Examples /// ``` /// # #![feature(portable_simd)] - /// # use core::simd::{Simd, Mask}; + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::{Simd, Mask}; /// let mut vec: Vec = vec![10, 11, 12, 13, 14, 15, 16, 17, 18]; /// let idxs = Simd::from_array([9, 3, 0, 0]); /// let vals = Simd::from_array([-27, 82, -41, 124]); @@ -329,7 +405,7 @@ where enable: Mask, idxs: Simd, ) { - let enable: Mask = enable & idxs.lanes_lt(Simd::splat(slice.len())); + let enable: Mask = enable & idxs.simd_lt(Simd::splat(slice.len())); // Safety: We have masked-off out-of-bounds lanes. unsafe { self.scatter_select_unchecked(slice, enable, idxs) } } @@ -347,13 +423,15 @@ where /// # Examples /// ``` /// # #![feature(portable_simd)] - /// # use core::simd::{Simd, Mask}; + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::{Simd, SimdPartialOrd, Mask}; /// let mut vec: Vec = vec![10, 11, 12, 13, 14, 15, 16, 17, 18]; /// let idxs = Simd::from_array([9, 3, 0, 0]); /// let vals = Simd::from_array([-27, 82, -41, 124]); /// let enable = Mask::from_array([true, true, true, false]); // Note the mask of the last lane. /// // If this mask was used to scatter, it would be unsound. Let's fix that. - /// let enable = enable & idxs.lanes_lt(Simd::splat(vec.len())); + /// let enable = enable & idxs.simd_lt(Simd::splat(vec.len())); /// /// // We have masked the OOB lane, so it's safe to scatter now. /// unsafe { vals.scatter_select_unchecked(&mut vec, enable, idxs); } @@ -425,8 +503,27 @@ where { #[inline] fn eq(&self, other: &Self) -> bool { - // TODO use SIMD equality - self.to_array() == other.to_array() + // Safety: All SIMD vectors are SimdPartialEq, and the comparison produces a valid mask. + let mask = unsafe { + let tfvec: Simd<::Mask, LANES> = intrinsics::simd_eq(*self, *other); + Mask::from_int_unchecked(tfvec) + }; + + // Two vectors are equal if all lanes tested true for vertical equality. + mask.all() + } + + #[allow(clippy::partialeq_ne_impl)] + #[inline] + fn ne(&self, other: &Self) -> bool { + // Safety: All SIMD vectors are SimdPartialEq, and the comparison produces a valid mask. + let mask = unsafe { + let tfvec: Simd<::Mask, LANES> = intrinsics::simd_ne(*self, *other); + Mask::from_int_unchecked(tfvec) + }; + + // Two vectors are non-equal if any lane tested true for vertical non-equality. + mask.any() } } @@ -561,61 +658,85 @@ pub unsafe trait SimdElement: Sealed + Copy { } impl Sealed for u8 {} + +// Safety: u8 is a valid SIMD element type, and is supported by this API unsafe impl SimdElement for u8 { type Mask = i8; } impl Sealed for u16 {} + +// Safety: u16 is a valid SIMD element type, and is supported by this API unsafe impl SimdElement for u16 { type Mask = i16; } impl Sealed for u32 {} + +// Safety: u32 is a valid SIMD element type, and is supported by this API unsafe impl SimdElement for u32 { type Mask = i32; } impl Sealed for u64 {} + +// Safety: u64 is a valid SIMD element type, and is supported by this API unsafe impl SimdElement for u64 { type Mask = i64; } impl Sealed for usize {} + +// Safety: usize is a valid SIMD element type, and is supported by this API unsafe impl SimdElement for usize { type Mask = isize; } impl Sealed for i8 {} + +// Safety: i8 is a valid SIMD element type, and is supported by this API unsafe impl SimdElement for i8 { type Mask = i8; } impl Sealed for i16 {} + +// Safety: i16 is a valid SIMD element type, and is supported by this API unsafe impl SimdElement for i16 { type Mask = i16; } impl Sealed for i32 {} + +// Safety: i32 is a valid SIMD element type, and is supported by this API unsafe impl SimdElement for i32 { type Mask = i32; } impl Sealed for i64 {} + +// Safety: i64 is a valid SIMD element type, and is supported by this API unsafe impl SimdElement for i64 { type Mask = i64; } impl Sealed for isize {} + +// Safety: isize is a valid SIMD element type, and is supported by this API unsafe impl SimdElement for isize { type Mask = isize; } impl Sealed for f32 {} + +// Safety: f32 is a valid SIMD element type, and is supported by this API unsafe impl SimdElement for f32 { type Mask = i32; } impl Sealed for f64 {} + +// Safety: f64 is a valid SIMD element type, and is supported by this API unsafe impl SimdElement for f64 { type Mask = i64; } diff --git a/library/portable-simd/crates/core_simd/src/vector/float.rs b/library/portable-simd/crates/core_simd/src/vector/float.rs index fcc7f6d8d1c..f836c99b1e2 100644 --- a/library/portable-simd/crates/core_simd/src/vector/float.rs +++ b/library/portable-simd/crates/core_simd/src/vector/float.rs @@ -1,199 +1,24 @@ #![allow(non_camel_case_types)] -use crate::simd::intrinsics; -use crate::simd::{LaneCount, Mask, Simd, SupportedLaneCount}; +use crate::simd::Simd; -/// Implements inherent methods for a float vector containing multiple -/// `$lanes` of float `$type`, which uses `$bits_ty` as its binary -/// representation. -macro_rules! impl_float_vector { - { $type:ty, $bits_ty:ty, $mask_ty:ty } => { - impl Simd<$type, LANES> - where - LaneCount: SupportedLaneCount, - { - /// Raw transmutation to an unsigned integer vector type with the - /// same size and number of lanes. - #[inline] - #[must_use = "method returns a new vector and does not mutate the original value"] - pub fn to_bits(self) -> Simd<$bits_ty, LANES> { - assert_eq!(core::mem::size_of::(), core::mem::size_of::>()); - unsafe { core::mem::transmute_copy(&self) } - } - - /// Raw transmutation from an unsigned integer vector type with the - /// same size and number of lanes. - #[inline] - #[must_use = "method returns a new vector and does not mutate the original value"] - pub fn from_bits(bits: Simd<$bits_ty, LANES>) -> Self { - assert_eq!(core::mem::size_of::(), core::mem::size_of::>()); - unsafe { core::mem::transmute_copy(&bits) } - } - - /// Produces a vector where every lane has the absolute value of the - /// equivalently-indexed lane in `self`. - #[inline] - #[must_use = "method returns a new vector and does not mutate the original value"] - pub fn abs(self) -> Self { - unsafe { intrinsics::simd_fabs(self) } - } - - /// Takes the reciprocal (inverse) of each lane, `1/x`. - #[inline] - #[must_use = "method returns a new vector and does not mutate the original value"] - pub fn recip(self) -> Self { - Self::splat(1.0) / self - } - - /// Converts each lane from radians to degrees. - #[inline] - #[must_use = "method returns a new vector and does not mutate the original value"] - pub fn to_degrees(self) -> Self { - // to_degrees uses a special constant for better precision, so extract that constant - self * Self::splat(<$type>::to_degrees(1.)) - } - - /// Converts each lane from degrees to radians. - #[inline] - #[must_use = "method returns a new vector and does not mutate the original value"] - pub fn to_radians(self) -> Self { - self * Self::splat(<$type>::to_radians(1.)) - } - - /// Returns true for each lane if it has a positive sign, including - /// `+0.0`, `NaN`s with positive sign bit and positive infinity. - #[inline] - #[must_use = "method returns a new mask and does not mutate the original value"] - pub fn is_sign_positive(self) -> Mask<$mask_ty, LANES> { - !self.is_sign_negative() - } - - /// Returns true for each lane if it has a negative sign, including - /// `-0.0`, `NaN`s with negative sign bit and negative infinity. - #[inline] - #[must_use = "method returns a new mask and does not mutate the original value"] - pub fn is_sign_negative(self) -> Mask<$mask_ty, LANES> { - let sign_bits = self.to_bits() & Simd::splat((!0 >> 1) + 1); - sign_bits.lanes_gt(Simd::splat(0)) - } - - /// Returns true for each lane if its value is `NaN`. - #[inline] - #[must_use = "method returns a new mask and does not mutate the original value"] - pub fn is_nan(self) -> Mask<$mask_ty, LANES> { - self.lanes_ne(self) - } - - /// Returns true for each lane if its value is positive infinity or negative infinity. - #[inline] - #[must_use = "method returns a new mask and does not mutate the original value"] - pub fn is_infinite(self) -> Mask<$mask_ty, LANES> { - self.abs().lanes_eq(Self::splat(<$type>::INFINITY)) - } - - /// Returns true for each lane if its value is neither infinite nor `NaN`. - #[inline] - #[must_use = "method returns a new mask and does not mutate the original value"] - pub fn is_finite(self) -> Mask<$mask_ty, LANES> { - self.abs().lanes_lt(Self::splat(<$type>::INFINITY)) - } - - /// Returns true for each lane if its value is subnormal. - #[inline] - #[must_use = "method returns a new mask and does not mutate the original value"] - pub fn is_subnormal(self) -> Mask<$mask_ty, LANES> { - self.abs().lanes_ne(Self::splat(0.0)) & (self.to_bits() & Self::splat(<$type>::INFINITY).to_bits()).lanes_eq(Simd::splat(0)) - } - - /// Returns true for each lane if its value is neither zero, infinite, - /// subnormal, nor `NaN`. - #[inline] - #[must_use = "method returns a new mask and does not mutate the original value"] - pub fn is_normal(self) -> Mask<$mask_ty, LANES> { - !(self.abs().lanes_eq(Self::splat(0.0)) | self.is_nan() | self.is_subnormal() | self.is_infinite()) - } - - /// Replaces each lane with a number that represents its sign. - /// - /// * `1.0` if the number is positive, `+0.0`, or `INFINITY` - /// * `-1.0` if the number is negative, `-0.0`, or `NEG_INFINITY` - /// * `NAN` if the number is `NAN` - #[inline] - #[must_use = "method returns a new vector and does not mutate the original value"] - pub fn signum(self) -> Self { - self.is_nan().select(Self::splat(<$type>::NAN), Self::splat(1.0).copysign(self)) - } - - /// Returns each lane with the magnitude of `self` and the sign of `sign`. - /// - /// If any lane is a `NAN`, then a `NAN` with the sign of `sign` is returned. - #[inline] - #[must_use = "method returns a new vector and does not mutate the original value"] - pub fn copysign(self, sign: Self) -> Self { - let sign_bit = sign.to_bits() & Self::splat(-0.).to_bits(); - let magnitude = self.to_bits() & !Self::splat(-0.).to_bits(); - Self::from_bits(sign_bit | magnitude) - } - - /// Returns the minimum of each lane. - /// - /// If one of the values is `NAN`, then the other value is returned. - #[inline] - #[must_use = "method returns a new vector and does not mutate the original value"] - pub fn min(self, other: Self) -> Self { - unsafe { intrinsics::simd_fmin(self, other) } - } - - /// Returns the maximum of each lane. - /// - /// If one of the values is `NAN`, then the other value is returned. - #[inline] - #[must_use = "method returns a new vector and does not mutate the original value"] - pub fn max(self, other: Self) -> Self { - unsafe { intrinsics::simd_fmax(self, other) } - } - - /// Restrict each lane to a certain interval unless it is NaN. - /// - /// For each lane in `self`, returns the corresponding lane in `max` if the lane is - /// greater than `max`, and the corresponding lane in `min` if the lane is less - /// than `min`. Otherwise returns the lane in `self`. - #[inline] - #[must_use = "method returns a new vector and does not mutate the original value"] - pub fn clamp(self, min: Self, max: Self) -> Self { - assert!( - min.lanes_le(max).all(), - "each lane in `min` must be less than or equal to the corresponding lane in `max`", - ); - let mut x = self; - x = x.lanes_lt(min).select(min, x); - x = x.lanes_gt(max).select(max, x); - x - } - } - }; -} - -impl_float_vector! { f32, u32, i32 } -impl_float_vector! { f64, u64, i64 } - -/// Vector of two `f32` values +/// A 64-bit SIMD vector with two elements of type `f32`. pub type f32x2 = Simd; -/// Vector of four `f32` values +/// A 128-bit SIMD vector with four elements of type `f32`. pub type f32x4 = Simd; -/// Vector of eight `f32` values +/// A 256-bit SIMD vector with eight elements of type `f32`. pub type f32x8 = Simd; -/// Vector of 16 `f32` values +/// A 512-bit SIMD vector with 16 elements of type `f32`. pub type f32x16 = Simd; -/// Vector of two `f64` values +/// A 128-bit SIMD vector with two elements of type `f64`. pub type f64x2 = Simd; -/// Vector of four `f64` values +/// A 256-bit SIMD vector with four elements of type `f64`. pub type f64x4 = Simd; -/// Vector of eight `f64` values +/// A 512-bit SIMD vector with eight elements of type `f64`. pub type f64x8 = Simd; diff --git a/library/portable-simd/crates/core_simd/src/vector/int.rs b/library/portable-simd/crates/core_simd/src/vector/int.rs index 3eac02a2761..20e56c7dc64 100644 --- a/library/portable-simd/crates/core_simd/src/vector/int.rs +++ b/library/portable-simd/crates/core_simd/src/vector/int.rs @@ -1,103 +1,63 @@ #![allow(non_camel_case_types)] -use crate::simd::{LaneCount, Mask, Simd, SupportedLaneCount}; +use crate::simd::Simd; -/// Implements additional integer traits (Eq, Ord, Hash) on the specified vector `$name`, holding multiple `$lanes` of `$type`. -macro_rules! impl_integer_vector { - { $type:ty } => { - impl Simd<$type, LANES> - where - LaneCount: SupportedLaneCount, - { - /// Returns true for each positive lane and false if it is zero or negative. - #[inline] - pub fn is_positive(self) -> Mask<$type, LANES> { - self.lanes_gt(Self::splat(0)) - } - - /// Returns true for each negative lane and false if it is zero or positive. - #[inline] - pub fn is_negative(self) -> Mask<$type, LANES> { - self.lanes_lt(Self::splat(0)) - } - - /// Returns numbers representing the sign of each lane. - /// * `0` if the number is zero - /// * `1` if the number is positive - /// * `-1` if the number is negative - #[inline] - pub fn signum(self) -> Self { - self.is_positive().select( - Self::splat(1), - self.is_negative().select(Self::splat(-1), Self::splat(0)) - ) - } - } - } -} - -impl_integer_vector! { isize } -impl_integer_vector! { i16 } -impl_integer_vector! { i32 } -impl_integer_vector! { i64 } -impl_integer_vector! { i8 } - -/// Vector of two `isize` values +/// A SIMD vector with two elements of type `isize`. pub type isizex2 = Simd; -/// Vector of four `isize` values +/// A SIMD vector with four elements of type `isize`. pub type isizex4 = Simd; -/// Vector of eight `isize` values +/// A SIMD vector with eight elements of type `isize`. pub type isizex8 = Simd; -/// Vector of two `i16` values +/// A 32-bit SIMD vector with two elements of type `i16`. pub type i16x2 = Simd; -/// Vector of four `i16` values +/// A 64-bit SIMD vector with four elements of type `i16`. pub type i16x4 = Simd; -/// Vector of eight `i16` values +/// A 128-bit SIMD vector with eight elements of type `i16`. pub type i16x8 = Simd; -/// Vector of 16 `i16` values +/// A 256-bit SIMD vector with 16 elements of type `i16`. pub type i16x16 = Simd; -/// Vector of 32 `i16` values +/// A 512-bit SIMD vector with 32 elements of type `i16`. pub type i16x32 = Simd; -/// Vector of two `i32` values +/// A 64-bit SIMD vector with two elements of type `i32`. pub type i32x2 = Simd; -/// Vector of four `i32` values +/// A 128-bit SIMD vector with four elements of type `i32`. pub type i32x4 = Simd; -/// Vector of eight `i32` values +/// A 256-bit SIMD vector with eight elements of type `i32`. pub type i32x8 = Simd; -/// Vector of 16 `i32` values +/// A 512-bit SIMD vector with 16 elements of type `i32`. pub type i32x16 = Simd; -/// Vector of two `i64` values +/// A 128-bit SIMD vector with two elements of type `i64`. pub type i64x2 = Simd; -/// Vector of four `i64` values +/// A 256-bit SIMD vector with four elements of type `i64`. pub type i64x4 = Simd; -/// Vector of eight `i64` values +/// A 512-bit SIMD vector with eight elements of type `i64`. pub type i64x8 = Simd; -/// Vector of four `i8` values +/// A 32-bit SIMD vector with four elements of type `i8`. pub type i8x4 = Simd; -/// Vector of eight `i8` values +/// A 64-bit SIMD vector with eight elements of type `i8`. pub type i8x8 = Simd; -/// Vector of 16 `i8` values +/// A 128-bit SIMD vector with 16 elements of type `i8`. pub type i8x16 = Simd; -/// Vector of 32 `i8` values +/// A 256-bit SIMD vector with 32 elements of type `i8`. pub type i8x32 = Simd; -/// Vector of 64 `i8` values +/// A 512-bit SIMD vector with 64 elements of type `i8`. pub type i8x64 = Simd; diff --git a/library/portable-simd/crates/core_simd/src/vector/uint.rs b/library/portable-simd/crates/core_simd/src/vector/uint.rs index ed91fc3640e..b4a69c44363 100644 --- a/library/portable-simd/crates/core_simd/src/vector/uint.rs +++ b/library/portable-simd/crates/core_simd/src/vector/uint.rs @@ -2,62 +2,62 @@ use crate::simd::Simd; -/// Vector of two `usize` values +/// A SIMD vector with two elements of type `usize`. pub type usizex2 = Simd; -/// Vector of four `usize` values +/// A SIMD vector with four elements of type `usize`. pub type usizex4 = Simd; -/// Vector of eight `usize` values +/// A SIMD vector with eight elements of type `usize`. pub type usizex8 = Simd; -/// Vector of two `u16` values +/// A 32-bit SIMD vector with two elements of type `u16`. pub type u16x2 = Simd; -/// Vector of four `u16` values +/// A 64-bit SIMD vector with four elements of type `u16`. pub type u16x4 = Simd; -/// Vector of eight `u16` values +/// A 128-bit SIMD vector with eight elements of type `u16`. pub type u16x8 = Simd; -/// Vector of 16 `u16` values +/// A 256-bit SIMD vector with 16 elements of type `u16`. pub type u16x16 = Simd; -/// Vector of 32 `u16` values +/// A 512-bit SIMD vector with 32 elements of type `u16`. pub type u16x32 = Simd; -/// Vector of two `u32` values +/// A 64-bit SIMD vector with two elements of type `u32`. pub type u32x2 = Simd; -/// Vector of four `u32` values +/// A 128-bit SIMD vector with four elements of type `u32`. pub type u32x4 = Simd; -/// Vector of eight `u32` values +/// A 256-bit SIMD vector with eight elements of type `u32`. pub type u32x8 = Simd; -/// Vector of 16 `u32` values +/// A 512-bit SIMD vector with 16 elements of type `u32`. pub type u32x16 = Simd; -/// Vector of two `u64` values +/// A 128-bit SIMD vector with two elements of type `u64`. pub type u64x2 = Simd; -/// Vector of four `u64` values +/// A 256-bit SIMD vector with four elements of type `u64`. pub type u64x4 = Simd; -/// Vector of eight `u64` values +/// A 512-bit SIMD vector with eight elements of type `u64`. pub type u64x8 = Simd; -/// Vector of four `u8` values +/// A 32-bit SIMD vector with four elements of type `u8`. pub type u8x4 = Simd; -/// Vector of eight `u8` values +/// A 64-bit SIMD vector with eight elements of type `u8`. pub type u8x8 = Simd; -/// Vector of 16 `u8` values +/// A 128-bit SIMD vector with 16 elements of type `u8`. pub type u8x16 = Simd; -/// Vector of 32 `u8` values +/// A 256-bit SIMD vector with 32 elements of type `u8`. pub type u8x32 = Simd; -/// Vector of 64 `u8` values +/// A 512-bit SIMD vector with 64 elements of type `u8`. pub type u8x64 = Simd; diff --git a/library/portable-simd/crates/core_simd/tests/i16_ops.rs b/library/portable-simd/crates/core_simd/tests/i16_ops.rs index 171e5b472fa..f6c5d74fbbc 100644 --- a/library/portable-simd/crates/core_simd/tests/i16_ops.rs +++ b/library/portable-simd/crates/core_simd/tests/i16_ops.rs @@ -1,32 +1,5 @@ #![feature(portable_simd)] -use core_simd::i16x2; #[macro_use] mod ops_macros; impl_signed_tests! { i16 } - -#[test] -fn max_is_not_lexicographic() { - let a = i16x2::splat(10); - let b = i16x2::from_array([-4, 12]); - assert_eq!(a.max(b), i16x2::from_array([10, 12])); -} - -#[test] -fn min_is_not_lexicographic() { - let a = i16x2::splat(10); - let b = i16x2::from_array([12, -4]); - assert_eq!(a.min(b), i16x2::from_array([10, -4])); -} - -#[test] -fn clamp_is_not_lexicographic() { - let a = i16x2::splat(10); - let lo = i16x2::from_array([-12, -4]); - let up = i16x2::from_array([-4, 12]); - assert_eq!(a.clamp(lo, up), i16x2::from_array([-4, 10])); - - let x = i16x2::from_array([1, 10]); - let y = x.clamp(i16x2::splat(0), i16x2::splat(9)); - assert_eq!(y, i16x2::from_array([1, 9])); -} diff --git a/library/portable-simd/crates/core_simd/tests/masks.rs b/library/portable-simd/crates/core_simd/tests/masks.rs index 3aec36ca7b7..673d0db93fe 100644 --- a/library/portable-simd/crates/core_simd/tests/masks.rs +++ b/library/portable-simd/crates/core_simd/tests/masks.rs @@ -80,6 +80,62 @@ macro_rules! test_mask_api { assert_eq!(bitmask, 0b1000001101001001); assert_eq!(core_simd::Mask::<$type, 16>::from_bitmask(bitmask), mask); } + + #[test] + fn roundtrip_bitmask_conversion_short() { + use core_simd::ToBitMask; + + let values = [ + false, false, false, true, + ]; + let mask = core_simd::Mask::<$type, 4>::from_array(values); + let bitmask = mask.to_bitmask(); + assert_eq!(bitmask, 0b1000); + assert_eq!(core_simd::Mask::<$type, 4>::from_bitmask(bitmask), mask); + + let values = [true, false]; + let mask = core_simd::Mask::<$type, 2>::from_array(values); + let bitmask = mask.to_bitmask(); + assert_eq!(bitmask, 0b01); + assert_eq!(core_simd::Mask::<$type, 2>::from_bitmask(bitmask), mask); + } + + #[test] + fn cast() { + fn cast_impl() + where + core_simd::Mask<$type, 8>: Into>, + { + let values = [true, false, false, true, false, false, true, false]; + let mask = core_simd::Mask::<$type, 8>::from_array(values); + + let cast_mask = mask.cast::(); + assert_eq!(values, cast_mask.to_array()); + + let into_mask: core_simd::Mask = mask.into(); + assert_eq!(values, into_mask.to_array()); + } + + cast_impl::(); + cast_impl::(); + cast_impl::(); + cast_impl::(); + cast_impl::(); + } + + #[cfg(feature = "generic_const_exprs")] + #[test] + fn roundtrip_bitmask_array_conversion() { + use core_simd::ToBitMaskArray; + let values = [ + true, false, false, true, false, false, true, false, + true, true, false, false, false, false, false, true, + ]; + let mask = core_simd::Mask::<$type, 16>::from_array(values); + let bitmask = mask.to_bitmask_array(); + assert_eq!(bitmask, [0b01001001, 0b10000011]); + assert_eq!(core_simd::Mask::<$type, 16>::from_bitmask_array(bitmask), mask); + } } } } diff --git a/library/portable-simd/crates/core_simd/tests/ops_macros.rs b/library/portable-simd/crates/core_simd/tests/ops_macros.rs index 7c9b17673ef..f759394d075 100644 --- a/library/portable-simd/crates/core_simd/tests/ops_macros.rs +++ b/library/portable-simd/crates/core_simd/tests/ops_macros.rs @@ -172,6 +172,7 @@ macro_rules! impl_common_integer_tests { macro_rules! impl_signed_tests { { $scalar:tt } => { mod $scalar { + use core_simd::simd::SimdInt; type Vector = core_simd::Simd; type Scalar = $scalar; @@ -222,34 +223,37 @@ macro_rules! impl_signed_tests { assert_eq!(a % b, Vector::::splat(0)); } - fn min() { + fn simd_min() { + use core_simd::simd::SimdOrd; let a = Vector::::splat(Scalar::MIN); let b = Vector::::splat(0); - assert_eq!(a.min(b), a); + assert_eq!(a.simd_min(b), a); let a = Vector::::splat(Scalar::MAX); let b = Vector::::splat(0); - assert_eq!(a.min(b), b); + assert_eq!(a.simd_min(b), b); } - fn max() { + fn simd_max() { + use core_simd::simd::SimdOrd; let a = Vector::::splat(Scalar::MIN); let b = Vector::::splat(0); - assert_eq!(a.max(b), b); + assert_eq!(a.simd_max(b), b); let a = Vector::::splat(Scalar::MAX); let b = Vector::::splat(0); - assert_eq!(a.max(b), a); + assert_eq!(a.simd_max(b), a); } - fn clamp() { + fn simd_clamp() { + use core_simd::simd::SimdOrd; let min = Vector::::splat(Scalar::MIN); let max = Vector::::splat(Scalar::MAX); let zero = Vector::::splat(0); let one = Vector::::splat(1); let negone = Vector::::splat(-1); - assert_eq!(zero.clamp(min, max), zero); - assert_eq!(zero.clamp(min, one), zero); - assert_eq!(zero.clamp(one, max), one); - assert_eq!(zero.clamp(min, negone), negone); + assert_eq!(zero.simd_clamp(min, max), zero); + assert_eq!(zero.simd_clamp(min, one), zero); + assert_eq!(zero.simd_clamp(one, max), one); + assert_eq!(zero.simd_clamp(min, negone), negone); } } @@ -309,6 +313,7 @@ macro_rules! impl_signed_tests { macro_rules! impl_unsigned_tests { { $scalar:tt } => { mod $scalar { + use core_simd::simd::SimdUint; type Vector = core_simd::Simd; type Scalar = $scalar; @@ -343,6 +348,7 @@ macro_rules! impl_unsigned_tests { macro_rules! impl_float_tests { { $scalar:tt, $int_scalar:tt } => { mod $scalar { + use core_simd::SimdFloat; type Vector = core_simd::Simd; type Scalar = $scalar; @@ -458,10 +464,10 @@ macro_rules! impl_float_tests { ) } - fn min() { + fn simd_min() { // Regular conditions (both values aren't zero) test_helpers::test_binary_elementwise( - &Vector::::min, + &Vector::::simd_min, &Scalar::min, // Reject the case where both values are zero with different signs &|a, b| { @@ -477,14 +483,14 @@ macro_rules! impl_float_tests { // Special case where both values are zero let p_zero = Vector::::splat(0.); let n_zero = Vector::::splat(-0.); - assert!(p_zero.min(n_zero).to_array().iter().all(|x| *x == 0.)); - assert!(n_zero.min(p_zero).to_array().iter().all(|x| *x == 0.)); + assert!(p_zero.simd_min(n_zero).to_array().iter().all(|x| *x == 0.)); + assert!(n_zero.simd_min(p_zero).to_array().iter().all(|x| *x == 0.)); } - fn max() { + fn simd_max() { // Regular conditions (both values aren't zero) test_helpers::test_binary_elementwise( - &Vector::::max, + &Vector::::simd_max, &Scalar::max, // Reject the case where both values are zero with different signs &|a, b| { @@ -500,11 +506,11 @@ macro_rules! impl_float_tests { // Special case where both values are zero let p_zero = Vector::::splat(0.); let n_zero = Vector::::splat(-0.); - assert!(p_zero.max(n_zero).to_array().iter().all(|x| *x == 0.)); - assert!(n_zero.max(p_zero).to_array().iter().all(|x| *x == 0.)); + assert!(p_zero.simd_max(n_zero).to_array().iter().all(|x| *x == 0.)); + assert!(n_zero.simd_max(p_zero).to_array().iter().all(|x| *x == 0.)); } - fn clamp() { + fn simd_clamp() { test_helpers::test_3(&|value: [Scalar; LANES], mut min: [Scalar; LANES], mut max: [Scalar; LANES]| { for (min, max) in min.iter_mut().zip(max.iter_mut()) { if max < min { @@ -522,7 +528,7 @@ macro_rules! impl_float_tests { for i in 0..LANES { result_scalar[i] = value[i].clamp(min[i], max[i]); } - let result_vector = Vector::from_array(value).clamp(min.into(), max.into()).to_array(); + let result_vector = Vector::from_array(value).simd_clamp(min.into(), max.into()).to_array(); test_helpers::prop_assert_biteq!(result_scalar, result_vector); Ok(()) }) diff --git a/library/portable-simd/crates/core_simd/tests/round.rs b/library/portable-simd/crates/core_simd/tests/round.rs index 7feb0320a16..484fd5bf47d 100644 --- a/library/portable-simd/crates/core_simd/tests/round.rs +++ b/library/portable-simd/crates/core_simd/tests/round.rs @@ -59,7 +59,7 @@ macro_rules! float_rounding_test { const MAX_REPRESENTABLE_VALUE: Scalar = (ALL_MANTISSA_BITS << (core::mem::size_of::() * 8 - ::MANTISSA_DIGITS as usize - 1)) as Scalar; - let mut runner = proptest::test_runner::TestRunner::default(); + let mut runner = test_helpers::make_runner(); runner.run( &test_helpers::array::UniformArrayStrategy::new(-MAX_REPRESENTABLE_VALUE..MAX_REPRESENTABLE_VALUE), |x| { diff --git a/library/portable-simd/crates/test_helpers/src/lib.rs b/library/portable-simd/crates/test_helpers/src/lib.rs index 8bf7f5ed3d2..141bee18a9a 100644 --- a/library/portable-simd/crates/test_helpers/src/lib.rs +++ b/library/portable-simd/crates/test_helpers/src/lib.rs @@ -78,11 +78,11 @@ impl DefaultStrategy } #[cfg(not(miri))] -fn make_runner() -> proptest::test_runner::TestRunner { +pub fn make_runner() -> proptest::test_runner::TestRunner { Default::default() } #[cfg(miri)] -fn make_runner() -> proptest::test_runner::TestRunner { +pub fn make_runner() -> proptest::test_runner::TestRunner { // Only run a few tests on Miri proptest::test_runner::TestRunner::new(proptest::test_runner::Config::with_cases(4)) }