mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #83 from rust-lang/feature/reductions

Browse Source 
Add reductions
This commit is contained in:
Jubilee 2021-04-22 16:44:41 -07:00 committed by GitHub
commit 24ebae870e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 465 additions and 120 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
crates/core_simd/src/intrinsics.rs
View File

@ -73,4 +73,15 @@ extern "platform-intrinsic" {
// {s,u}sub.sat
pub(crate) fn simd_saturating_sub<T>(x: T, y: T) -> T;
// reductions
pub(crate) fn simd_reduce_add_ordered<T, U>(x: T, y: U) -> U;
pub(crate) fn simd_reduce_mul_ordered<T, U>(x: T, y: U) -> U;
pub(crate) fn simd_reduce_all<T>(x: T) -> bool;
pub(crate) fn simd_reduce_any<T>(x: T) -> bool;
pub(crate) fn simd_reduce_max<T, U>(x: T) -> U;
pub(crate) fn simd_reduce_min<T, U>(x: T) -> U;
pub(crate) fn simd_reduce_and<T, U>(x: T) -> U;
pub(crate) fn simd_reduce_or<T, U>(x: T) -> U;
pub(crate) fn simd_reduce_xor<T, U>(x: T) -> U;
}

2
crates/core_simd/src/lib.rs
View File

@ -11,6 +11,8 @@ mod first;
mod permute;
#[macro_use]
mod transmute;
#[macro_use]
mod reduction;
mod comparisons;
mod fmt;

4
crates/core_simd/src/masks/bitmask.rs
View File

@ -1,7 +1,7 @@
use crate::LanesAtMost32;
/// A mask where each lane is represented by a single bit.
#[derive(Copy, Clone, Debug)]
#[derive(Copy, Clone, Debug, PartialOrd, PartialEq, Ord, Eq, Hash)]
#[repr(transparent)]
pub struct BitMask<const LANES: usize>(u64)
where
@ -14,7 +14,7 @@ where
/// Construct a mask by setting all lanes to the given value.
pub fn splat(value: bool) -> Self {
if value {
Self(u64::MAX)
Self(u64::MAX >> (64 - LANES))
} else {
Self(u64::MIN)
}

130
crates/core_simd/src/masks/full_masks.rs
View File

@ -14,22 +14,27 @@ impl core::fmt::Display for TryFromMaskError {
}
macro_rules! define_mask {
{ $(#[$attr:meta])* struct $name:ident<const $lanes:ident: usize>($type:ty); } => {
{
$(#[$attr:meta])*
struct $name:ident<const $lanes:ident: usize>(
crate::$type:ident<$lanes2:ident>
);
} => {
$(#[$attr])*
#[derive(Default, PartialEq, PartialOrd, Eq, Ord, Hash)]
#[repr(transparent)]
pub struct $name<const $lanes: usize>($type)
pub struct $name<const $lanes: usize>(crate::$type<$lanes2>)
where
$type: crate::LanesAtMost32;
crate::$type<LANES>: crate::LanesAtMost32;
impl<const LANES: usize> Copy for $name<LANES>
where
$type: crate::LanesAtMost32,
crate::$type<LANES>: crate::LanesAtMost32,
{}
impl<const LANES: usize> Clone for $name<LANES>
where
$type: crate::LanesAtMost32,
crate::$type<LANES>: crate::LanesAtMost32,
{
#[inline]
fn clone(&self) -> Self {
@ -37,13 +42,13 @@ macro_rules! define_mask {
}
}
impl<const $lanes: usize> $name<$lanes>
impl<const LANES: usize> $name<LANES>
where
$type: crate::LanesAtMost32,
crate::$type<LANES>: crate::LanesAtMost32,
{
/// Construct a mask by setting all lanes to the given value.
pub fn splat(value: bool) -> Self {
Self(<$type>::splat(
Self(<crate::$type<LANES>>::splat(
if value {
-1
} else {
@ -76,42 +81,51 @@ macro_rules! define_mask {
}
}
/// Creates a mask from an integer vector.
///
/// # Safety
/// All lanes must be either 0 or -1.
/// Converts the mask to the equivalent integer representation, where -1 represents
/// "set" and 0 represents "unset".
#[inline]
pub unsafe fn from_int_unchecked(value: $type) -> Self {
pub fn to_int(self) -> crate::$type<LANES> {
self.0
}
/// Creates a mask from the equivalent integer representation, where -1 represents
/// "set" and 0 represents "unset".
///
/// Each provided lane must be either 0 or -1.
#[inline]
pub unsafe fn from_int_unchecked(value: crate::$type<LANES>) -> Self {
Self(value)
}
/// Creates a mask from an integer vector.
/// Creates a mask from the equivalent integer representation, where -1 represents
/// "set" and 0 represents "unset".
///
/// # Panics
/// Panics if any lane is not 0 or -1.
#[inline]
pub fn from_int(value: $type) -> Self {
pub fn from_int(value: crate::$type<LANES>) -> Self {
use core::convert::TryInto;
value.try_into().unwrap()
}
}
impl<const $lanes: usize> core::convert::From<bool> for $name<$lanes>
impl<const LANES: usize> core::convert::From<bool> for $name<LANES>
where
$type: crate::LanesAtMost32,
crate::$type<LANES>: crate::LanesAtMost32,
{
fn from(value: bool) -> Self {
Self::splat(value)
}
}
impl<const $lanes: usize> core::convert::TryFrom<$type> for $name<$lanes>
impl<const LANES: usize> core::convert::TryFrom<crate::$type<LANES>> for $name<LANES>
where
$type: crate::LanesAtMost32,
crate::$type<LANES>: crate::LanesAtMost32,
{
type Error = TryFromMaskError;
fn try_from(value: $type) -> Result<Self, Self::Error> {
if value.as_slice().iter().all(|x| *x == 0 || *x == -1) {
fn try_from(value: crate::$type<LANES>) -> Result<Self, Self::Error> {
let valid = (value.lanes_eq(crate::$type::<LANES>::splat(0)) | value.lanes_eq(crate::$type::<LANES>::splat(-1))).all();
if valid {
Ok(Self(value))
} else {
Err(TryFromMaskError(()))
@ -119,21 +133,21 @@ macro_rules! define_mask {
}
}
impl<const $lanes: usize> core::convert::From<$name<$lanes>> for $type
impl<const LANES: usize> core::convert::From<$name<LANES>> for crate::$type<LANES>
where
$type: crate::LanesAtMost32,
crate::$type<LANES>: crate::LanesAtMost32,
{
fn from(value: $name<$lanes>) -> Self {
fn from(value: $name<LANES>) -> Self {
value.0
}
}
impl<const $lanes: usize> core::convert::From<crate::BitMask<$lanes>> for $name<$lanes>
impl<const LANES: usize> core::convert::From<crate::BitMask<LANES>> for $name<LANES>
where
$type: crate::LanesAtMost32,
crate::BitMask<$lanes>: crate::LanesAtMost32,
crate::$type<LANES>: crate::LanesAtMost32,
crate::BitMask<LANES>: crate::LanesAtMost32,
{
fn from(value: crate::BitMask<$lanes>) -> Self {
fn from(value: crate::BitMask<LANES>) -> Self {
// TODO use an intrinsic to do this efficiently (with LLVM's sext instruction)
let mut mask = Self::splat(false);
for lane in 0..LANES {
@ -143,10 +157,10 @@ macro_rules! define_mask {
}
}
impl<const $lanes: usize> core::convert::From<$name<$lanes>> for crate::BitMask<$lanes>
impl<const LANES: usize> core::convert::From<$name<LANES>> for crate::BitMask<LANES>
where
$type: crate::LanesAtMost32,
crate::BitMask<$lanes>: crate::LanesAtMost32,
crate::$type<LANES>: crate::LanesAtMost32,
crate::BitMask<LANES>: crate::LanesAtMost32,
{
fn from(value: $name<$lanes>) -> Self {
// TODO use an intrinsic to do this efficiently (with LLVM's trunc instruction)
@ -158,9 +172,9 @@ macro_rules! define_mask {
}
}
impl<const $lanes: usize> core::fmt::Debug for $name<$lanes>
impl<const LANES: usize> core::fmt::Debug for $name<LANES>
where
$type: crate::LanesAtMost32,
crate::$type<LANES>: crate::LanesAtMost32,
{
fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result {
f.debug_list()
@ -169,36 +183,36 @@ macro_rules! define_mask {
}
}
impl<const $lanes: usize> core::fmt::Binary for $name<$lanes>
impl<const LANES: usize> core::fmt::Binary for $name<LANES>
where
$type: crate::LanesAtMost32,
crate::$type<LANES>: crate::LanesAtMost32,
{
fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result {
core::fmt::Binary::fmt(&self.0, f)
}
}
impl<const $lanes: usize> core::fmt::Octal for $name<$lanes>
impl<const LANES: usize> core::fmt::Octal for $name<LANES>
where
$type: crate::LanesAtMost32,
crate::$type<LANES>: crate::LanesAtMost32,
{
fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result {
core::fmt::Octal::fmt(&self.0, f)
}
}
impl<const $lanes: usize> core::fmt::LowerHex for $name<$lanes>
impl<const LANES: usize> core::fmt::LowerHex for $name<LANES>
where
$type: crate::LanesAtMost32,
crate::$type<LANES>: crate::LanesAtMost32,
{
fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result {
core::fmt::LowerHex::fmt(&self.0, f)
}
}
impl<const $lanes: usize> core::fmt::UpperHex for $name<$lanes>
impl<const LANES: usize> core::fmt::UpperHex for $name<LANES>
where
$type: crate::LanesAtMost32,
crate::$type<LANES>: crate::LanesAtMost32,
{
fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result {
core::fmt::UpperHex::fmt(&self.0, f)
@ -207,7 +221,7 @@ macro_rules! define_mask {
impl<const LANES: usize> core::ops::BitAnd for $name<LANES>
where
$type: crate::LanesAtMost32,
crate::$type<LANES>: crate::LanesAtMost32,
{
type Output = Self;
#[inline]
@ -218,7 +232,7 @@ macro_rules! define_mask {
impl<const LANES: usize> core::ops::BitAnd<bool> for $name<LANES>
where
$type: crate::LanesAtMost32,
crate::$type<LANES>: crate::LanesAtMost32,
{
type Output = Self;
#[inline]
@ -229,7 +243,7 @@ macro_rules! define_mask {
impl<const LANES: usize> core::ops::BitAnd<$name<LANES>> for bool
where
$type: crate::LanesAtMost32,
crate::$type<LANES>: crate::LanesAtMost32,
{
type Output = $name<LANES>;
#[inline]
@ -240,7 +254,7 @@ macro_rules! define_mask {
impl<const LANES: usize> core::ops::BitOr for $name<LANES>
where
$type: crate::LanesAtMost32,
crate::$type<LANES>: crate::LanesAtMost32,
{
type Output = Self;
#[inline]
@ -251,7 +265,7 @@ macro_rules! define_mask {
impl<const LANES: usize> core::ops::BitOr<bool> for $name<LANES>
where
$type: crate::LanesAtMost32,
crate::$type<LANES>: crate::LanesAtMost32,
{
type Output = Self;
#[inline]
@ -262,7 +276,7 @@ macro_rules! define_mask {
impl<const LANES: usize> core::ops::BitOr<$name<LANES>> for bool
where
$type: crate::LanesAtMost32,
crate::$type<LANES>: crate::LanesAtMost32,
{
type Output = $name<LANES>;
#[inline]
@ -273,7 +287,7 @@ macro_rules! define_mask {
impl<const LANES: usize> core::ops::BitXor for $name<LANES>
where
$type: crate::LanesAtMost32,
crate::$type<LANES>: crate::LanesAtMost32,
{
type Output = Self;
#[inline]
@ -284,7 +298,7 @@ macro_rules! define_mask {
impl<const LANES: usize> core::ops::BitXor<bool> for $name<LANES>
where
$type: crate::LanesAtMost32,
crate::$type<LANES>: crate::LanesAtMost32,
{
type Output = Self;
#[inline]
@ -295,7 +309,7 @@ macro_rules! define_mask {
impl<const LANES: usize> core::ops::BitXor<$name<LANES>> for bool
where
$type: crate::LanesAtMost32,
crate::$type<LANES>: crate::LanesAtMost32,
{
type Output = $name<LANES>;
#[inline]
@ -306,7 +320,7 @@ macro_rules! define_mask {
impl<const LANES: usize> core::ops::Not for $name<LANES>
where
$type: crate::LanesAtMost32,
crate::$type<LANES>: crate::LanesAtMost32,
{
type Output = $name<LANES>;
#[inline]
@ -317,7 +331,7 @@ macro_rules! define_mask {
impl<const LANES: usize> core::ops::BitAndAssign for $name<LANES>
where
$type: crate::LanesAtMost32,
crate::$type<LANES>: crate::LanesAtMost32,
{
#[inline]
fn bitand_assign(&mut self, rhs: Self) {
@ -327,7 +341,7 @@ macro_rules! define_mask {
impl<const LANES: usize> core::ops::BitAndAssign<bool> for $name<LANES>
where
$type: crate::LanesAtMost32,
crate::$type<LANES>: crate::LanesAtMost32,
{
#[inline]
fn bitand_assign(&mut self, rhs: bool) {
@ -337,7 +351,7 @@ macro_rules! define_mask {
impl<const LANES: usize> core::ops::BitOrAssign for $name<LANES>
where
$type: crate::LanesAtMost32,
crate::$type<LANES>: crate::LanesAtMost32,
{
#[inline]
fn bitor_assign(&mut self, rhs: Self) {
@ -347,7 +361,7 @@ macro_rules! define_mask {
impl<const LANES: usize> core::ops::BitOrAssign<bool> for $name<LANES>
where
$type: crate::LanesAtMost32,
crate::$type<LANES>: crate::LanesAtMost32,
{
#[inline]
fn bitor_assign(&mut self, rhs: bool) {
@ -357,7 +371,7 @@ macro_rules! define_mask {
impl<const LANES: usize> core::ops::BitXorAssign for $name<LANES>
where
$type: crate::LanesAtMost32,
crate::$type<LANES>: crate::LanesAtMost32,
{
#[inline]
fn bitxor_assign(&mut self, rhs: Self) {
@ -367,13 +381,15 @@ macro_rules! define_mask {
impl<const LANES: usize> core::ops::BitXorAssign<bool> for $name<LANES>
where
$type: crate::LanesAtMost32,
crate::$type<LANES>: crate::LanesAtMost32,
{
#[inline]
fn bitxor_assign(&mut self, rhs: bool) {
*self ^= Self::splat(rhs);
}
}
impl_full_mask_reductions! { $name, $type }
}
}

128
crates/core_simd/src/masks/mod.rs
View File

@ -7,25 +7,27 @@ pub use full_masks::*;
mod bitmask;
pub use bitmask::*;
use crate::LanesAtMost32;
use crate::{LanesAtMost32, SimdI128, SimdI16, SimdI32, SimdI64, SimdI8, SimdIsize};
macro_rules! define_opaque_mask {
{
$(#[$attr:meta])*
struct $name:ident<const $lanes:ident: usize>($inner_ty:ty);
@bits $bits_ty:ty
struct $name:ident<const $lanes:ident: usize>($inner_ty:ident<$lanes2:ident>);
@bits $bits_ty:ident
} => {
$(#[$attr])*
#[allow(non_camel_case_types)]
pub struct $name<const $lanes: usize>($inner_ty) where $bits_ty: LanesAtMost32;
pub struct $name<const LANES: usize>($inner_ty<LANES>) where $bits_ty<LANES>: LanesAtMost32;
impl<const $lanes: usize> $name<$lanes>
impl_opaque_mask_reductions! { $name, $inner_ty, $bits_ty }
impl<const LANES: usize> $name<LANES>
where
$bits_ty: LanesAtMost32
$bits_ty<LANES>: LanesAtMost32
{
/// Construct a mask by setting all lanes to the given value.
pub fn splat(value: bool) -> Self {
Self(<$inner_ty>::splat(value))
Self(<$inner_ty<LANES>>::splat(value))
}
/// Converts an array to a SIMD vector.
@ -69,66 +71,72 @@ macro_rules! define_opaque_mask {
}
}
impl<const $lanes: usize> From<BitMask<$lanes>> for $name<$lanes>
impl<const LANES: usize> From<BitMask<LANES>> for $name<LANES>
where
$bits_ty: LanesAtMost32,
BitMask<$lanes>: LanesAtMost32,
$bits_ty<LANES>: LanesAtMost32,
BitMask<LANES>: LanesAtMost32,
{
fn from(value: BitMask<$lanes>) -> Self {
fn from(value: BitMask<LANES>) -> Self {
Self(value.into())
}
}
impl<const $lanes: usize> From<$name<$lanes>> for crate::BitMask<$lanes>
impl<const LANES: usize> From<$name<LANES>> for crate::BitMask<LANES>
where
$bits_ty: LanesAtMost32,
BitMask<$lanes>: LanesAtMost32,
$bits_ty<LANES>: LanesAtMost32,
BitMask<LANES>: LanesAtMost32,
{
fn from(value: $name<$lanes>) -> Self {
fn from(value: $name<LANES>) -> Self {
value.0.into()
}
}
impl<const $lanes: usize> From<$inner_ty> for $name<$lanes>
impl<const LANES: usize> From<$inner_ty<LANES>> for $name<LANES>
where
$bits_ty: LanesAtMost32,
$bits_ty<LANES>: LanesAtMost32,
{
fn from(value: $inner_ty) -> Self {
fn from(value: $inner_ty<LANES>) -> Self {
Self(value)
}
}
impl<const $lanes: usize> From<$name<$lanes>> for $inner_ty
impl<const LANES: usize> From<$name<LANES>> for $inner_ty<LANES>
where
$bits_ty: LanesAtMost32,
$bits_ty<LANES>: LanesAtMost32,
{
fn from(value: $name<$lanes>) -> Self {
fn from(value: $name<LANES>) -> Self {
value.0
}
}
// vector/array conversion
impl<const $lanes: usize> From<[bool; $lanes]> for $name<$lanes> where $bits_ty: crate::LanesAtMost32 {
fn from(array: [bool; $lanes]) -> Self {
impl<const LANES: usize> From<[bool; LANES]> for $name<LANES>
where
$bits_ty<LANES>: crate::LanesAtMost32
{
fn from(array: [bool; LANES]) -> Self {
Self::from_array(array)
}
}
impl <const $lanes: usize> From<$name<$lanes>> for [bool; $lanes] where $bits_ty: crate::LanesAtMost32 {
fn from(vector: $name<$lanes>) -> Self {
impl <const LANES: usize> From<$name<LANES>> for [bool; LANES]
where
$bits_ty<LANES>: crate::LanesAtMost32
{
fn from(vector: $name<LANES>) -> Self {
vector.to_array()
}
}
impl<const $lanes: usize> Copy for $name<$lanes>
impl<const LANES: usize> Copy for $name<LANES>
where
$inner_ty: Copy,
$bits_ty: LanesAtMost32,
$inner_ty<LANES>: Copy,
$bits_ty<LANES>: LanesAtMost32,
{}
impl<const $lanes: usize> Clone for $name<$lanes>
impl<const LANES: usize> Clone for $name<LANES>
where
$bits_ty: LanesAtMost32,
$bits_ty<LANES>: LanesAtMost32,
{
#[inline]
fn clone(&self) -> Self {
@ -136,9 +144,9 @@ macro_rules! define_opaque_mask {
}
}
impl<const $lanes: usize> Default for $name<$lanes>
impl<const LANES: usize> Default for $name<LANES>
where
$bits_ty: LanesAtMost32,
$bits_ty<LANES>: LanesAtMost32,
{
#[inline]
fn default() -> Self {
@ -146,9 +154,9 @@ macro_rules! define_opaque_mask {
}
}
impl<const $lanes: usize> PartialEq for $name<$lanes>
impl<const LANES: usize> PartialEq for $name<LANES>
where
$bits_ty: LanesAtMost32,
$bits_ty<LANES>: LanesAtMost32,
{
#[inline]
fn eq(&self, other: &Self) -> bool {
@ -156,9 +164,9 @@ macro_rules! define_opaque_mask {
}
}
impl<const $lanes: usize> PartialOrd for $name<$lanes>
impl<const LANES: usize> PartialOrd for $name<LANES>
where
$bits_ty: LanesAtMost32,
$bits_ty<LANES>: LanesAtMost32,
{
#[inline]
fn partial_cmp(&self, other: &Self) -> Option<core::cmp::Ordering> {
@ -166,9 +174,9 @@ macro_rules! define_opaque_mask {
}
}
impl<const $lanes: usize> core::fmt::Debug for $name<$lanes>
impl<const LANES: usize> core::fmt::Debug for $name<LANES>
where
$bits_ty: LanesAtMost32,
$bits_ty<LANES>: LanesAtMost32,
{
fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result {
core::fmt::Debug::fmt(&self.0, f)
@ -177,7 +185,7 @@ macro_rules! define_opaque_mask {
impl<const LANES: usize> core::ops::BitAnd for $name<LANES>
where
$bits_ty: LanesAtMost32,
$bits_ty<LANES>: LanesAtMost32,
{
type Output = Self;
#[inline]
@ -188,7 +196,7 @@ macro_rules! define_opaque_mask {
impl<const LANES: usize> core::ops::BitAnd<bool> for $name<LANES>
where
$bits_ty: LanesAtMost32,
$bits_ty<LANES>: LanesAtMost32,
{
type Output = Self;
#[inline]
@ -199,7 +207,7 @@ macro_rules! define_opaque_mask {
impl<const LANES: usize> core::ops::BitAnd<$name<LANES>> for bool
where
$bits_ty: LanesAtMost32,
$bits_ty<LANES>: LanesAtMost32,
{
type Output = $name<LANES>;
#[inline]
@ -210,7 +218,7 @@ macro_rules! define_opaque_mask {
impl<const LANES: usize> core::ops::BitOr for $name<LANES>
where
$bits_ty: LanesAtMost32,
$bits_ty<LANES>: LanesAtMost32,
{
type Output = Self;
#[inline]
@ -221,7 +229,7 @@ macro_rules! define_opaque_mask {
impl<const LANES: usize> core::ops::BitOr<bool> for $name<LANES>
where
$bits_ty: LanesAtMost32,
$bits_ty<LANES>: LanesAtMost32,
{
type Output = Self;
#[inline]
@ -232,7 +240,7 @@ macro_rules! define_opaque_mask {
impl<const LANES: usize> core::ops::BitOr<$name<LANES>> for bool
where
$bits_ty: LanesAtMost32,
$bits_ty<LANES>: LanesAtMost32,
{
type Output = $name<LANES>;
#[inline]
@ -243,7 +251,7 @@ macro_rules! define_opaque_mask {
impl<const LANES: usize> core::ops::BitXor for $name<LANES>
where
$bits_ty: LanesAtMost32,
$bits_ty<LANES>: LanesAtMost32,
{
type Output = Self;
#[inline]
@ -254,7 +262,7 @@ macro_rules! define_opaque_mask {
impl<const LANES: usize> core::ops::BitXor<bool> for $name<LANES>
where
$bits_ty: LanesAtMost32,
$bits_ty<LANES>: LanesAtMost32,
{
type Output = Self;
#[inline]
@ -265,7 +273,7 @@ macro_rules! define_opaque_mask {
impl<const LANES: usize> core::ops::BitXor<$name<LANES>> for bool
where
$bits_ty: LanesAtMost32,
$bits_ty<LANES>: LanesAtMost32,
{
type Output = $name<LANES>;
#[inline]
@ -276,7 +284,7 @@ macro_rules! define_opaque_mask {
impl<const LANES: usize> core::ops::Not for $name<LANES>
where
$bits_ty: LanesAtMost32,
$bits_ty<LANES>: LanesAtMost32,
{
type Output = $name<LANES>;
#[inline]
@ -287,7 +295,7 @@ macro_rules! define_opaque_mask {
impl<const LANES: usize> core::ops::BitAndAssign for $name<LANES>
where
$bits_ty: LanesAtMost32,
$bits_ty<LANES>: LanesAtMost32,
{
#[inline]
fn bitand_assign(&mut self, rhs: Self) {
@ -297,7 +305,7 @@ macro_rules! define_opaque_mask {
impl<const LANES: usize> core::ops::BitAndAssign<bool> for $name<LANES>
where
$bits_ty: LanesAtMost32,
$bits_ty<LANES>: LanesAtMost32,
{
#[inline]
fn bitand_assign(&mut self, rhs: bool) {
@ -307,7 +315,7 @@ macro_rules! define_opaque_mask {
impl<const LANES: usize> core::ops::BitOrAssign for $name<LANES>
where
$bits_ty: LanesAtMost32,
$bits_ty<LANES>: LanesAtMost32,
{
#[inline]
fn bitor_assign(&mut self, rhs: Self) {
@ -317,7 +325,7 @@ macro_rules! define_opaque_mask {
impl<const LANES: usize> core::ops::BitOrAssign<bool> for $name<LANES>
where
$bits_ty: LanesAtMost32,
$bits_ty<LANES>: LanesAtMost32,
{
#[inline]
fn bitor_assign(&mut self, rhs: bool) {
@ -327,7 +335,7 @@ macro_rules! define_opaque_mask {
impl<const LANES: usize> core::ops::BitXorAssign for $name<LANES>
where
$bits_ty: LanesAtMost32,
$bits_ty<LANES>: LanesAtMost32,
{
#[inline]
fn bitxor_assign(&mut self, rhs: Self) {
@ -337,7 +345,7 @@ macro_rules! define_opaque_mask {
impl<const LANES: usize> core::ops::BitXorAssign<bool> for $name<LANES>
where
$bits_ty: LanesAtMost32,
$bits_ty<LANES>: LanesAtMost32,
{
#[inline]
fn bitxor_assign(&mut self, rhs: bool) {
@ -352,7 +360,7 @@ define_opaque_mask! {
///
/// The layout of this type is unspecified.
struct Mask8<const LANES: usize>(SimdMask8<LANES>);
@bits crate::SimdI8<LANES>
@bits SimdI8
}
define_opaque_mask! {
@ -360,7 +368,7 @@ define_opaque_mask! {
///
/// The layout of this type is unspecified.
struct Mask16<const LANES: usize>(SimdMask16<LANES>);
@bits crate::SimdI16<LANES>
@bits SimdI16
}
define_opaque_mask! {
@ -368,7 +376,7 @@ define_opaque_mask! {
///
/// The layout of this type is unspecified.
struct Mask32<const LANES: usize>(SimdMask32<LANES>);
@bits crate::SimdI32<LANES>
@bits SimdI32
}
define_opaque_mask! {
@ -376,7 +384,7 @@ define_opaque_mask! {
///
/// The layout of this type is unspecified.
struct Mask64<const LANES: usize>(SimdMask64<LANES>);
@bits crate::SimdI64<LANES>
@bits SimdI64
}
define_opaque_mask! {
@ -384,7 +392,7 @@ define_opaque_mask! {
///
/// The layout of this type is unspecified.
struct Mask128<const LANES: usize>(SimdMask128<LANES>);
@bits crate::SimdI128<LANES>
@bits SimdI128
}
define_opaque_mask! {
@ -392,7 +400,7 @@ define_opaque_mask! {
///
/// The layout of this type is unspecified.
struct MaskSize<const LANES: usize>(SimdMaskSize<LANES>);
@bits crate::SimdIsize<LANES>
@bits SimdIsize
}
/// Vector of eight 8-bit masks

162
crates/core_simd/src/reduction.rs Normal file
View File

@ -0,0 +1,162 @@
macro_rules! impl_integer_reductions {
{ $name:ident, $scalar:ty } => {
impl<const LANES: usize> crate::$name<LANES>
where
Self: crate::LanesAtMost32
{
/// Horizontal wrapping add. Returns the sum of the lanes of the vector, with wrapping addition.
#[inline]
pub fn horizontal_sum(self) -> $scalar {
unsafe { crate::intrinsics::simd_reduce_add_ordered(self, 0) }
}
/// Horizontal wrapping multiply. Returns the product of the lanes of the vector, with wrapping multiplication.
#[inline]
pub fn horizontal_product(self) -> $scalar {
unsafe { crate::intrinsics::simd_reduce_mul_ordered(self, 1) }
}
/// Horizontal bitwise "and". Returns the cumulative bitwise "and" across the lanes of
/// the vector.
#[inline]
pub fn horizontal_and(self) -> $scalar {
unsafe { crate::intrinsics::simd_reduce_and(self) }
}
/// Horizontal bitwise "or". Returns the cumulative bitwise "or" across the lanes of
/// the vector.
#[inline]
pub fn horizontal_or(self) -> $scalar {
unsafe { crate::intrinsics::simd_reduce_or(self) }
}
/// Horizontal bitwise "xor". Returns the cumulative bitwise "xor" across the lanes of
/// the vector.
#[inline]
pub fn horizontal_xor(self) -> $scalar {
unsafe { crate::intrinsics::simd_reduce_xor(self) }
}
/// Horizontal maximum. Returns the maximum lane in the vector.
#[inline]
pub fn horizontal_max(self) -> $scalar {
unsafe { crate::intrinsics::simd_reduce_max(self) }
}
/// Horizontal minimum. Returns the minimum lane in the vector.
#[inline]
pub fn horizontal_min(self) -> $scalar {
unsafe { crate::intrinsics::simd_reduce_min(self) }
}
}
}
}
macro_rules! impl_float_reductions {
{ $name:ident, $scalar:ty } => {
impl<const LANES: usize> crate::$name<LANES>
where
Self: crate::LanesAtMost32
{
/// Horizontal add. Returns the sum of the lanes of the vector.
#[inline]
pub fn horizontal_sum(self) -> $scalar {
// LLVM sum is inaccurate on i586
if cfg!(all(target_arch = "x86", not(target_feature = "sse2"))) {
self.as_slice().iter().sum()
} else {
unsafe { crate::intrinsics::simd_reduce_add_ordered(self, 0.) }
}
}
/// Horizontal multiply. Returns the product of the lanes of the vector.
#[inline]
pub fn horizontal_product(self) -> $scalar {
// LLVM product is inaccurate on i586
if cfg!(all(target_arch = "x86", not(target_feature = "sse2"))) {
self.as_slice().iter().product()
} else {
unsafe { crate::intrinsics::simd_reduce_mul_ordered(self, 1.) }
}
}
/// Horizontal maximum. Returns the maximum lane in the vector.
///
/// Returns values based on equality, so a vector containing both `0.` and `-0.` may
/// return either. This function will not return `NaN` unless all lanes are `NaN`.
#[inline]
pub fn horizontal_max(self) -> $scalar {
unsafe { crate::intrinsics::simd_reduce_max(self) }
}
/// Horizontal minimum. Returns the minimum lane in the vector.
///
/// Returns values based on equality, so a vector containing both `0.` and `-0.` may
/// return either. This function will not return `NaN` unless all lanes are `NaN`.
#[inline]
pub fn horizontal_min(self) -> $scalar {
unsafe { crate::intrinsics::simd_reduce_min(self) }
}
}
}
}
macro_rules! impl_full_mask_reductions {
{ $name:ident, $inner:ident } => {
impl<const LANES: usize> crate::$name<LANES>
where
crate::$inner<LANES>: crate::LanesAtMost32
{
/// Returns true if any lane is set, or false otherwise.
#[inline]
pub fn any(self) -> bool {
unsafe { crate::intrinsics::simd_reduce_any(self.to_int()) }
}
/// Returns true if all lanes are set, or false otherwise.
#[inline]
pub fn all(self) -> bool {
unsafe { crate::intrinsics::simd_reduce_all(self.to_int()) }
}
}
}
}
macro_rules! impl_opaque_mask_reductions {
{ $name:ident, $inner:ident, $bits_ty:ident } => {
impl<const LANES: usize> $name<LANES>
where
$bits_ty<LANES>: crate::LanesAtMost32
{
/// Returns true if any lane is set, or false otherwise.
#[inline]
pub fn any(self) -> bool {
self.0.any()
}
/// Returns true if all lanes are set, or false otherwise.
#[inline]
pub fn all(self) -> bool {
self.0.all()
}
}
}
}
impl<const LANES: usize> crate::BitMask<LANES>
where
crate::BitMask<LANES>: crate::LanesAtMost32,
{
/// Returns true if any lane is set, or false otherwise.
#[inline]
pub fn any(self) -> bool {
self != Self::splat(false)
}
/// Returns true if all lanes are set, or false otherwise.
#[inline]
pub fn all(self) -> bool {
self == Self::splat(true)
}
}

1
crates/core_simd/src/vector/float.rs
View File

@ -6,6 +6,7 @@
macro_rules! impl_float_vector {
{ $name:ident, $type:ty, $bits_ty:ident, $mask_ty:ident, $mask_impl_ty:ident } => {
impl_vector! { $name, $type }
impl_float_reductions! { $name, $type }
impl<const LANES: usize> $name<LANES>
where

1
crates/core_simd/src/vector/int.rs
View File

@ -4,6 +4,7 @@
macro_rules! impl_integer_vector {
{ $name:ident, $type:ty, $mask_ty:ident, $mask_impl_ty:ident } => {
impl_vector! { $name, $type }
impl_integer_reductions! { $name, $type }
impl<const LANES: usize> Eq for $name<LANES> where Self: crate::LanesAtMost32 {}

1
crates/core_simd/src/vector/uint.rs
View File

@ -5,6 +5,7 @@
macro_rules! impl_unsigned_vector {
{ $name:ident, $type:ty } => {
impl_vector! { $name, $type }
impl_integer_reductions! { $name, $type }
impl<const LANES: usize> Eq for $name<LANES> where Self: crate::LanesAtMost32 {}

18
crates/core_simd/tests/masks.rs
View File

@ -59,6 +59,24 @@ macro_rules! test_mask_api {
let mask = core_simd::$name::<8>::splat(false);
let _ = mask.test(8);
}
#[test]
fn any() {
assert!(!core_simd::$name::<8>::splat(false).any());
assert!(core_simd::$name::<8>::splat(true).any());
let mut v = core_simd::$name::<8>::splat(false);
v.set(2, true);
assert!(v.any());
}
#[test]
fn all() {
assert!(!core_simd::$name::<8>::splat(false).all());
assert!(core_simd::$name::<8>::splat(true).all());
let mut v = core_simd::$name::<8>::splat(false);
v.set(2, true);
assert!(!v.all());
}
}
}
}

125
crates/core_simd/tests/ops_macros.rs
View File

@ -136,6 +136,83 @@ macro_rules! impl_binary_checked_op_test {
};
}
#[macro_export]
macro_rules! impl_common_integer_tests {
{ $vector:ident, $scalar:ident } => {
test_helpers::test_lanes! {
fn horizontal_sum<const LANES: usize>() {
test_helpers::test_1(&|x| {
test_helpers::prop_assert_biteq! (
$vector::<LANES>::from_array(x).horizontal_sum(),
x.iter().copied().fold(0 as $scalar, $scalar::wrapping_add),
);
Ok(())
});
}
fn horizontal_product<const LANES: usize>() {
test_helpers::test_1(&|x| {
test_helpers::prop_assert_biteq! (
$vector::<LANES>::from_array(x).horizontal_product(),
x.iter().copied().fold(1 as $scalar, $scalar::wrapping_mul),
);
Ok(())
});
}
fn horizontal_and<const LANES: usize>() {
test_helpers::test_1(&|x| {
test_helpers::prop_assert_biteq! (
$vector::<LANES>::from_array(x).horizontal_and(),
x.iter().copied().fold(-1i8 as $scalar, <$scalar as core::ops::BitAnd>::bitand),
);
Ok(())
});
}
fn horizontal_or<const LANES: usize>() {
test_helpers::test_1(&|x| {
test_helpers::prop_assert_biteq! (
$vector::<LANES>::from_array(x).horizontal_or(),
x.iter().copied().fold(0 as $scalar, <$scalar as core::ops::BitOr>::bitor),
);
Ok(())
});
}
fn horizontal_xor<const LANES: usize>() {
test_helpers::test_1(&|x| {
test_helpers::prop_assert_biteq! (
$vector::<LANES>::from_array(x).horizontal_xor(),
x.iter().copied().fold(0 as $scalar, <$scalar as core::ops::BitXor>::bitxor),
);
Ok(())
});
}
fn horizontal_max<const LANES: usize>() {
test_helpers::test_1(&|x| {
test_helpers::prop_assert_biteq! (
$vector::<LANES>::from_array(x).horizontal_max(),
x.iter().copied().max().unwrap(),
);
Ok(())
});
}
fn horizontal_min<const LANES: usize>() {
test_helpers::test_1(&|x| {
test_helpers::prop_assert_biteq! (
$vector::<LANES>::from_array(x).horizontal_min(),
x.iter().copied().min().unwrap(),
);
Ok(())
});
}
}
}
}
/// Implement tests for signed integers.
#[macro_export]
macro_rules! impl_signed_tests {
@ -144,6 +221,8 @@ macro_rules! impl_signed_tests {
type Vector<const LANES: usize> = core_simd::$vector<LANES>;
type Scalar = $scalar;
impl_common_integer_tests! { Vector, Scalar }
test_helpers::test_lanes! {
fn neg<const LANES: usize>() {
test_helpers::test_unary_elementwise(
@ -241,6 +320,8 @@ macro_rules! impl_unsigned_tests {
type Vector<const LANES: usize> = core_simd::$vector<LANES>;
type Scalar = $scalar;
impl_common_integer_tests! { Vector, Scalar }
test_helpers::test_lanes_panic! {
fn rem_zero_panic<const LANES: usize>() {
let a = Vector::<LANES>::splat(42);
@ -397,6 +478,50 @@ macro_rules! impl_float_tests {
},
).unwrap();
}
fn horizontal_sum<const LANES: usize>() {
test_helpers::test_1(&|x| {
test_helpers::prop_assert_biteq! (
Vector::<LANES>::from_array(x).horizontal_sum(),
x.iter().sum(),
);
Ok(())
});
}
fn horizontal_product<const LANES: usize>() {
test_helpers::test_1(&|x| {
test_helpers::prop_assert_biteq! (
Vector::<LANES>::from_array(x).horizontal_product(),
x.iter().product(),
);
Ok(())
});
}
fn horizontal_max<const LANES: usize>() {
test_helpers::test_1(&|x| {
let vmax = Vector::<LANES>::from_array(x).horizontal_max();
let smax = x.iter().copied().fold(Scalar::NAN, Scalar::max);
// 0 and -0 are treated the same
if !(x.contains(&0.) && x.contains(&-0.) && vmax.abs() == 0. && smax.abs() == 0.) {
test_helpers::prop_assert_biteq!(vmax, smax);
}
Ok(())
});
}
fn horizontal_min<const LANES: usize>() {
test_helpers::test_1(&|x| {
let vmax = Vector::<LANES>::from_array(x).horizontal_min();
let smax = x.iter().copied().fold(Scalar::NAN, Scalar::min);
// 0 and -0 are treated the same
if !(x.contains(&0.) && x.contains(&-0.) && vmax.abs() == 0. && smax.abs() == 0.) {
test_helpers::prop_assert_biteq!(vmax, smax);
}
Ok(())
});
}
}
}
}

2
crates/test_helpers/src/biteq.rs
View File

@ -95,7 +95,7 @@ impl<T: BitEq> core::fmt::Debug for BitEqWrapper<'_, T> {
#[macro_export]
macro_rules! prop_assert_biteq {
{ $a:expr, $b:expr } => {
{ $a:expr, $b:expr $(,)? } => {
{
use $crate::biteq::BitEqWrapper;
let a = $a;