diff --git a/crates/core_simd/src/cast.rs b/crates/core_simd/src/cast.rs new file mode 100644 index 00000000000..e04a9042b1b --- /dev/null +++ b/crates/core_simd/src/cast.rs @@ -0,0 +1,45 @@ +use crate::simd::SimdElement; + +/// Supporting trait for `Simd::cast`. Typically doesn't need to be used directly. +pub trait SimdCast: SimdElement {} + +macro_rules! into_number { + { $($type:ty),* } => { + $( + impl SimdCast for $type {} + impl SimdCast for $type {} + impl SimdCast for $type {} + impl SimdCast for $type {} + impl SimdCast for $type {} + + impl SimdCast for $type {} + impl SimdCast for $type {} + impl SimdCast for $type {} + impl SimdCast for $type {} + impl SimdCast for $type {} + + impl SimdCast for $type {} + impl SimdCast for $type {} + )* + } +} + +into_number! { i8, i16, i32, i64, isize, u8, u16, u32, u64, usize, f32, f64 } + +macro_rules! into_pointer { + { $($type:ty),* } => { + $( + impl SimdCast<$type> for *const T {} + impl SimdCast<$type> for *mut T {} + impl SimdCast<*const T> for $type {} + impl SimdCast<*mut T> for $type {} + )* + } +} + +into_pointer! { i8, i16, i32, i64, isize, u8, u16, u32, u64, usize } + +impl SimdCast<*const T> for *const U {} +impl SimdCast<*const T> for *mut U {} +impl SimdCast<*mut T> for *const U {} +impl SimdCast<*mut T> for *mut U {} diff --git a/crates/core_simd/src/elements.rs b/crates/core_simd/src/elements.rs index 701eb66b248..dc7f52a4d57 100644 --- a/crates/core_simd/src/elements.rs +++ b/crates/core_simd/src/elements.rs @@ -1,11 +1,15 @@ +mod const_ptr; mod float; mod int; +mod mut_ptr; mod uint; mod sealed { pub trait Sealed {} } +pub use const_ptr::*; pub use float::*; pub use int::*; +pub use mut_ptr::*; pub use uint::*; diff --git a/crates/core_simd/src/elements/const_ptr.rs b/crates/core_simd/src/elements/const_ptr.rs new file mode 100644 index 00000000000..ab6b5b8b5f4 --- /dev/null +++ b/crates/core_simd/src/elements/const_ptr.rs @@ -0,0 +1,59 @@ +use super::sealed::Sealed; +use crate::simd::{intrinsics, LaneCount, Mask, Simd, SimdPartialEq, SupportedLaneCount}; + +/// Operations on SIMD vectors of constant pointers. +pub trait SimdConstPtr: Copy + Sealed { + /// Vector type representing the pointers as bits. + type Bits; + + /// Vector of mutable pointers to the same type. + type MutPtr; + + /// Mask type used for manipulating this SIMD vector type. + type Mask; + + /// Returns `true` for each lane that is null. + fn is_null(self) -> Self::Mask; + + /// Changes constness without changing the type. + fn as_mut(self) -> Self::MutPtr; + + /// Cast pointers to raw bits. + fn to_bits(self) -> Self::Bits; + + /// Cast raw bits to pointers. + fn from_bits(bits: Self::Bits) -> Self; +} + +impl Sealed for Simd<*const T, LANES> where + LaneCount: SupportedLaneCount +{ +} + +impl SimdConstPtr for Simd<*const T, LANES> +where + LaneCount: SupportedLaneCount, +{ + type Bits = Simd; + type MutPtr = Simd<*mut T, LANES>; + type Mask = Mask; + + fn is_null(self) -> Self::Mask { + Simd::splat(core::ptr::null()).simd_eq(self) + } + + fn as_mut(self) -> Self::MutPtr { + // Converting between pointers is safe + unsafe { intrinsics::simd_as(self) } + } + + fn to_bits(self) -> Self::Bits { + // Casting pointers to usize is safe + unsafe { intrinsics::simd_as(self) } + } + + fn from_bits(bits: Self::Bits) -> Self { + // Casting usize to pointers is safe + unsafe { intrinsics::simd_as(bits) } + } +} diff --git a/crates/core_simd/src/elements/mut_ptr.rs b/crates/core_simd/src/elements/mut_ptr.rs new file mode 100644 index 00000000000..b49f9fda7e4 --- /dev/null +++ b/crates/core_simd/src/elements/mut_ptr.rs @@ -0,0 +1,57 @@ +use super::sealed::Sealed; +use crate::simd::{intrinsics, LaneCount, Mask, Simd, SimdPartialEq, SupportedLaneCount}; + +/// Operations on SIMD vectors of mutable pointers. +pub trait SimdMutPtr: Copy + Sealed { + /// Vector type representing the pointers as bits. + type Bits; + + /// Vector of constant pointers to the same type. + type ConstPtr; + + /// Mask type used for manipulating this SIMD vector type. + type Mask; + + /// Returns `true` for each lane that is null. + fn is_null(self) -> Self::Mask; + + /// Changes constness without changing the type. + fn as_const(self) -> Self::ConstPtr; + + /// Cast pointers to raw bits. + fn to_bits(self) -> Self::Bits; + + /// Cast raw bits to pointers. + fn from_bits(bits: Self::Bits) -> Self; +} + +impl Sealed for Simd<*mut T, LANES> where LaneCount: SupportedLaneCount +{} + +impl SimdMutPtr for Simd<*mut T, LANES> +where + LaneCount: SupportedLaneCount, +{ + type Bits = Simd; + type ConstPtr = Simd<*const T, LANES>; + type Mask = Mask; + + fn is_null(self) -> Self::Mask { + Simd::splat(core::ptr::null_mut()).simd_eq(self) + } + + fn as_const(self) -> Self::ConstPtr { + // Converting between pointers is safe + unsafe { intrinsics::simd_as(self) } + } + + fn to_bits(self) -> Self::Bits { + // Casting pointers to usize is safe + unsafe { intrinsics::simd_as(self) } + } + + fn from_bits(bits: Self::Bits) -> Self { + // Casting usize to pointers is safe + unsafe { intrinsics::simd_as(bits) } + } +} diff --git a/crates/core_simd/src/eq.rs b/crates/core_simd/src/eq.rs index c7111f720a8..149380746e7 100644 --- a/crates/core_simd/src/eq.rs +++ b/crates/core_simd/src/eq.rs @@ -71,3 +71,45 @@ fn simd_ne(self, other: Self) -> Self::Mask { } impl_mask! { i8, i16, i32, i64, isize } + +impl SimdPartialEq for Simd<*const T, LANES> +where + LaneCount: SupportedLaneCount, +{ + type Mask = Mask; + + #[inline] + fn simd_eq(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Mask::from_int_unchecked(intrinsics::simd_eq(self, other)) } + } + + #[inline] + fn simd_ne(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Mask::from_int_unchecked(intrinsics::simd_ne(self, other)) } + } +} + +impl SimdPartialEq for Simd<*mut T, LANES> +where + LaneCount: SupportedLaneCount, +{ + type Mask = Mask; + + #[inline] + fn simd_eq(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Mask::from_int_unchecked(intrinsics::simd_eq(self, other)) } + } + + #[inline] + fn simd_ne(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Mask::from_int_unchecked(intrinsics::simd_ne(self, other)) } + } +} diff --git a/crates/core_simd/src/mod.rs b/crates/core_simd/src/mod.rs index 9909d639874..ece026a448b 100644 --- a/crates/core_simd/src/mod.rs +++ b/crates/core_simd/src/mod.rs @@ -7,6 +7,7 @@ mod to_bytes; mod alias; +mod cast; mod elements; mod eq; mod fmt; @@ -24,6 +25,7 @@ pub mod simd { pub(crate) use crate::core_simd::intrinsics; pub use crate::core_simd::alias::*; + pub use crate::core_simd::cast::*; pub use crate::core_simd::elements::*; pub use crate::core_simd::eq::*; pub use crate::core_simd::lane_count::{LaneCount, SupportedLaneCount}; diff --git a/crates/core_simd/src/ord.rs b/crates/core_simd/src/ord.rs index 9a87bc2e344..95a1ecaeeda 100644 --- a/crates/core_simd/src/ord.rs +++ b/crates/core_simd/src/ord.rs @@ -211,3 +211,117 @@ fn simd_clamp(self, min: Self, max: Self) -> Self { } impl_mask! { i8, i16, i32, i64, isize } + +impl SimdPartialOrd for Simd<*const T, LANES> +where + LaneCount: SupportedLaneCount, +{ + #[inline] + fn simd_lt(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Mask::from_int_unchecked(intrinsics::simd_lt(self, other)) } + } + + #[inline] + fn simd_le(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Mask::from_int_unchecked(intrinsics::simd_le(self, other)) } + } + + #[inline] + fn simd_gt(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Mask::from_int_unchecked(intrinsics::simd_gt(self, other)) } + } + + #[inline] + fn simd_ge(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Mask::from_int_unchecked(intrinsics::simd_ge(self, other)) } + } +} + +impl SimdOrd for Simd<*const T, LANES> +where + LaneCount: SupportedLaneCount, +{ + #[inline] + fn simd_max(self, other: Self) -> Self { + self.simd_lt(other).select(other, self) + } + + #[inline] + fn simd_min(self, other: Self) -> Self { + self.simd_gt(other).select(other, self) + } + + #[inline] + fn simd_clamp(self, min: Self, max: Self) -> Self { + assert!( + min.simd_le(max).all(), + "each lane in `min` must be less than or equal to the corresponding lane in `max`", + ); + self.simd_max(min).simd_min(max) + } +} + +impl SimdPartialOrd for Simd<*mut T, LANES> +where + LaneCount: SupportedLaneCount, +{ + #[inline] + fn simd_lt(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Mask::from_int_unchecked(intrinsics::simd_lt(self, other)) } + } + + #[inline] + fn simd_le(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Mask::from_int_unchecked(intrinsics::simd_le(self, other)) } + } + + #[inline] + fn simd_gt(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Mask::from_int_unchecked(intrinsics::simd_gt(self, other)) } + } + + #[inline] + fn simd_ge(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Mask::from_int_unchecked(intrinsics::simd_ge(self, other)) } + } +} + +impl SimdOrd for Simd<*mut T, LANES> +where + LaneCount: SupportedLaneCount, +{ + #[inline] + fn simd_max(self, other: Self) -> Self { + self.simd_lt(other).select(other, self) + } + + #[inline] + fn simd_min(self, other: Self) -> Self { + self.simd_gt(other).select(other, self) + } + + #[inline] + fn simd_clamp(self, min: Self, max: Self) -> Self { + assert!( + min.simd_le(max).all(), + "each lane in `min` must be less than or equal to the corresponding lane in `max`", + ); + self.simd_max(min).simd_min(max) + } +} diff --git a/crates/core_simd/src/vector.rs b/crates/core_simd/src/vector.rs index 7f0e8350cf8..cbc8ced5a84 100644 --- a/crates/core_simd/src/vector.rs +++ b/crates/core_simd/src/vector.rs @@ -2,7 +2,7 @@ pub(crate) mod ptr; use crate::simd::{ - intrinsics, LaneCount, Mask, MaskElement, SimdPartialOrd, SupportedLaneCount, Swizzle, + intrinsics, LaneCount, Mask, MaskElement, SimdCast, SimdPartialOrd, SupportedLaneCount, Swizzle, }; /// A SIMD vector of `LANES` elements of type `T`. `Simd` has the same shape as [`[T; N]`](array), but operates like `T`. @@ -211,7 +211,10 @@ pub const fn from_slice(slice: &[T]) -> Self { #[must_use] #[inline] #[cfg(not(bootstrap))] - pub fn cast(self) -> Simd { + pub fn cast(self) -> Simd + where + T: SimdCast, + { // Safety: The input argument is a vector of a valid SIMD element type. unsafe { intrinsics::simd_as(self) } } @@ -234,7 +237,7 @@ pub fn cast(self) -> Simd { #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub unsafe fn to_int_unchecked(self) -> Simd where - T: core::convert::FloatToInt, + T: core::convert::FloatToInt + SimdCast, I: SimdElement, { // Safety: `self` is a vector, and `FloatToInt` ensures the type can be casted to @@ -739,3 +742,13 @@ impl Sealed for f64 {} unsafe impl SimdElement for f64 { type Mask = i64; } + +impl Sealed for *const T {} +unsafe impl SimdElement for *const T { + type Mask = isize; +} + +impl Sealed for *mut T {} +unsafe impl SimdElement for *mut T { + type Mask = isize; +}