diff --git a/library/portable-simd/.github/workflows/ci.yml b/library/portable-simd/.github/workflows/ci.yml index 1ff377fce34..90543044ea8 100644 --- a/library/portable-simd/.github/workflows/ci.yml +++ b/library/portable-simd/.github/workflows/ci.yml @@ -167,40 +167,33 @@ jobs: RUSTFLAGS: ${{ matrix.rustflags }} cross-tests: - name: "${{ matrix.target }} (via cross)" + name: "${{ matrix.target_feature }} on ${{ matrix.target }} (via cross)" runs-on: ubuntu-latest strategy: fail-fast: false - # TODO: Sadly, we cant configure target-feature in a meaningful way - # because `cross` doesn't tell qemu to enable any non-default cpu - # features, nor does it give us a way to do so. - # - # Ultimately, we'd like to do something like [rust-lang/stdarch][stdarch]. - # This is a lot more complex... but in practice it's likely that we can just - # snarf the docker config from around [here][1000-dockerfiles]. - # - # [stdarch]: https://github.com/rust-lang/stdarch/blob/a5db4eaf/.github/workflows/main.yml#L67 - # [1000-dockerfiles]: https://github.com/rust-lang/stdarch/tree/a5db4eaf/ci/docker matrix: target: - - i586-unknown-linux-gnu - # 32-bit arm has a few idiosyncracies like having subnormal flushing - # to zero on by default. Ideally we'd set - armv7-unknown-linux-gnueabihf - - aarch64-unknown-linux-gnu - # Note: The issue above means neither of these mips targets will use - # MSA (mips simd) but MIPS uses a nonstandard binary representation - # for NaNs which makes it worth testing on despite that. + - thumbv7neon-unknown-linux-gnueabihf # includes neon by default + - aarch64-unknown-linux-gnu # includes neon by default + - powerpc-unknown-linux-gnu + - powerpc64le-unknown-linux-gnu # includes altivec by default + - riscv64gc-unknown-linux-gnu + # MIPS uses a nonstandard binary representation for NaNs which makes it worth testing + # non-nightly since https://github.com/rust-lang/rust/pull/113274 # - mips-unknown-linux-gnu # - mips64-unknown-linux-gnuabi64 - - riscv64gc-unknown-linux-gnu - # TODO this test works, but it appears to time out - # - powerpc-unknown-linux-gnu - # TODO this test is broken, but it appears to be a problem with QEMU, not us. - # - powerpc64le-unknown-linux-gnu - # TODO enable this once a new version of cross is released + # Lots of errors in QEMU and no real hardware to test on. Not clear if it's QEMU or bad codegen. # - powerpc64-unknown-linux-gnu + target_feature: [default] + include: + - { target: powerpc64le-unknown-linux-gnu, target_feature: "+vsx" } + # Fails due to QEMU floating point errors, probably handling subnormals incorrectly. + # This target is somewhat redundant, since ppc64le has altivec as well. + # - { target: powerpc-unknown-linux-gnu, target_feature: "+altivec" } + # We should test this, but cross currently can't run it + # - { target: riscv64gc-unknown-linux-gnu, target_feature: "+v,+zvl128b" } steps: - uses: actions/checkout@v2 @@ -217,11 +210,27 @@ jobs: # being part of the tarball means we can't just use the download/latest # URL :( run: | - CROSS_URL=https://github.com/rust-embedded/cross/releases/download/v0.2.1/cross-v0.2.1-x86_64-unknown-linux-gnu.tar.gz + CROSS_URL=https://github.com/cross-rs/cross/releases/download/v0.2.5/cross-x86_64-unknown-linux-gnu.tar.gz mkdir -p "$HOME/.bin" curl -sfSL --retry-delay 10 --retry 5 "${CROSS_URL}" | tar zxf - -C "$HOME/.bin" echo "$HOME/.bin" >> $GITHUB_PATH + - name: Configure Emulated CPUs + run: | + echo "CARGO_TARGET_POWERPC_UNKNOWN_LINUX_GNU_RUNNER=qemu-ppc -cpu e600" >> $GITHUB_ENV + # echo "CARGO_TARGET_RISCV64GC_UNKNOWN_LINUX_GNU_RUNNER=qemu-riscv64 -cpu rv64,zba=true,zbb=true,v=true,vlen=256,vext_spec=v1.0" >> $GITHUB_ENV + + - name: Configure RUSTFLAGS + shell: bash + run: | + case "${{ matrix.target_feature }}" in + default) + echo "RUSTFLAGS=" >> $GITHUB_ENV;; + *) + echo "RUSTFLAGS=-Ctarget-feature=${{ matrix.target_feature }}" >> $GITHUB_ENV + ;; + esac + - name: Test (debug) run: cross test --verbose --target=${{ matrix.target }} @@ -229,7 +238,7 @@ jobs: run: cross test --verbose --target=${{ matrix.target }} --release features: - name: "Check cargo features (${{ matrix.simd }} × ${{ matrix.features }})" + name: "Test cargo features (${{ matrix.simd }} × ${{ matrix.features }})" runs-on: ubuntu-latest strategy: fail-fast: false @@ -240,12 +249,8 @@ jobs: features: - "" - "--features std" - - "--features generic_const_exprs" - - "--features std --features generic_const_exprs" - "--features all_lane_counts" - - "--features all_lane_counts --features std" - - "--features all_lane_counts --features generic_const_exprs" - - "--features all_lane_counts --features std --features generic_const_exprs" + - "--all-features" steps: - uses: actions/checkout@v2 @@ -257,9 +262,9 @@ jobs: run: echo "CPU_FEATURE=$(lscpu | grep -o avx512[a-z]* | sed s/avx/+avx/ | tr '\n' ',' )" >> $GITHUB_ENV - name: Check build if: ${{ matrix.simd == '' }} - run: RUSTFLAGS="-Dwarnings" cargo check --all-targets --no-default-features ${{ matrix.features }} + run: RUSTFLAGS="-Dwarnings" cargo test --all-targets --no-default-features ${{ matrix.features }} - name: Check AVX if: ${{ matrix.simd == 'avx512' && contains(env.CPU_FEATURE, 'avx512') }} run: | echo "Found AVX features: $CPU_FEATURE" - RUSTFLAGS="-Dwarnings -Ctarget-feature=$CPU_FEATURE" cargo check --all-targets --no-default-features ${{ matrix.features }} + RUSTFLAGS="-Dwarnings -Ctarget-feature=$CPU_FEATURE" cargo test --all-targets --no-default-features ${{ matrix.features }} diff --git a/library/portable-simd/.gitignore b/library/portable-simd/.gitignore index 96ef6c0b944..ea8c4bf7f35 100644 --- a/library/portable-simd/.gitignore +++ b/library/portable-simd/.gitignore @@ -1,2 +1 @@ /target -Cargo.lock diff --git a/library/portable-simd/Cargo.lock b/library/portable-simd/Cargo.lock new file mode 100644 index 00000000000..46312c09657 --- /dev/null +++ b/library/portable-simd/Cargo.lock @@ -0,0 +1,304 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bumpalo" +version = "3.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" + +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "console_error_panic_hook" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06aeb73f470f66dcdbf7223caeebb85984942f22f1adb2a088cf9668146bbbc" +dependencies = [ + "cfg-if", + "wasm-bindgen", +] + +[[package]] +name = "core_simd" +version = "0.1.0" +dependencies = [ + "proptest", + "std_float", + "test_helpers", + "wasm-bindgen", + "wasm-bindgen-test", +] + +[[package]] +name = "js-sys" +version = "0.3.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "log" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" + +[[package]] +name = "num-traits" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" + +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + +[[package]] +name = "proc-macro2" +version = "1.0.66" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "proptest" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12e6c80c1139113c28ee4670dc50cc42915228b51f56a9e407f0ec60f966646f" +dependencies = [ + "bitflags", + "byteorder", + "num-traits", + "rand", + "rand_chacha", + "rand_xorshift", +] + +[[package]] +name = "quote" +version = "1.0.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "rand_chacha", + "rand_core", + "rand_hc", +] + +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" + +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core", +] + +[[package]] +name = "rand_xorshift" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77d416b86801d23dde1aa643023b775c3a462efc0ed96443add11546cdf1dca8" +dependencies = [ + "rand_core", +] + +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + +[[package]] +name = "std_float" +version = "0.1.0" +dependencies = [ + "core_simd", +] + +[[package]] +name = "syn" +version = "2.0.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c324c494eba9d92503e6f1ef2e6df781e78f6a7705a0202d9801b198807d518a" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "test_helpers" +version = "0.1.0" +dependencies = [ + "proptest", +] + +[[package]] +name = "unicode-ident" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" + +[[package]] +name = "wasm-bindgen" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c02dbc21516f9f1f04f187958890d7e6026df8d16540b7ad9492bc34a67cea03" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" + +[[package]] +name = "wasm-bindgen-test" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e6e302a7ea94f83a6d09e78e7dc7d9ca7b186bc2829c24a22d0753efd680671" +dependencies = [ + "console_error_panic_hook", + "js-sys", + "scoped-tls", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-bindgen-test-macro", +] + +[[package]] +name = "wasm-bindgen-test-macro" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecb993dd8c836930ed130e020e77d9b2e65dd0fbab1b67c790b0f5d80b11a575" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "web-sys" +version = "0.3.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" +dependencies = [ + "js-sys", + "wasm-bindgen", +] diff --git a/library/portable-simd/crates/core_simd/Cargo.toml b/library/portable-simd/crates/core_simd/Cargo.toml index d1a3a515a7e..b4a8fd70f4c 100644 --- a/library/portable-simd/crates/core_simd/Cargo.toml +++ b/library/portable-simd/crates/core_simd/Cargo.toml @@ -12,7 +12,6 @@ license = "MIT OR Apache-2.0" default = ["as_crate"] as_crate = [] std = [] -generic_const_exprs = [] all_lane_counts = [] [target.'cfg(target_arch = "wasm32")'.dev-dependencies] diff --git a/library/portable-simd/crates/core_simd/examples/dot_product.rs b/library/portable-simd/crates/core_simd/examples/dot_product.rs index a7973ec7404..f047010a65c 100644 --- a/library/portable-simd/crates/core_simd/examples/dot_product.rs +++ b/library/portable-simd/crates/core_simd/examples/dot_product.rs @@ -6,7 +6,7 @@ #![feature(slice_as_chunks)] // Add these imports to use the stdsimd library #![feature(portable_simd)] -use core_simd::simd::*; +use core_simd::simd::prelude::*; // This is your barebones dot product implementation: // Take 2 vectors, multiply them element wise and *then* diff --git a/library/portable-simd/crates/core_simd/examples/matrix_inversion.rs b/library/portable-simd/crates/core_simd/examples/matrix_inversion.rs index 39f530f68f5..bad86414401 100644 --- a/library/portable-simd/crates/core_simd/examples/matrix_inversion.rs +++ b/library/portable-simd/crates/core_simd/examples/matrix_inversion.rs @@ -2,8 +2,7 @@ // Code ported from the `packed_simd` crate // Run this code with `cargo test --example matrix_inversion` #![feature(array_chunks, portable_simd)] -use core_simd::simd::*; -use Which::*; +use core_simd::simd::prelude::*; // Gotta define our own 4x4 matrix since Rust doesn't ship multidim arrays yet :^) #[derive(Copy, Clone, Debug, PartialEq, PartialOrd)] @@ -164,10 +163,10 @@ pub fn simd_inv4x4(m: Matrix4x4) -> Option { let m_2 = f32x4::from_array(m[2]); let m_3 = f32x4::from_array(m[3]); - const SHUFFLE01: [Which; 4] = [First(0), First(1), Second(0), Second(1)]; - const SHUFFLE02: [Which; 4] = [First(0), First(2), Second(0), Second(2)]; - const SHUFFLE13: [Which; 4] = [First(1), First(3), Second(1), Second(3)]; - const SHUFFLE23: [Which; 4] = [First(2), First(3), Second(2), Second(3)]; + const SHUFFLE01: [usize; 4] = [0, 1, 4, 5]; + const SHUFFLE02: [usize; 4] = [0, 2, 4, 6]; + const SHUFFLE13: [usize; 4] = [1, 3, 5, 7]; + const SHUFFLE23: [usize; 4] = [2, 3, 6, 7]; let tmp = simd_swizzle!(m_0, m_1, SHUFFLE01); let row1 = simd_swizzle!(m_2, m_3, SHUFFLE01); @@ -180,58 +179,58 @@ pub fn simd_inv4x4(m: Matrix4x4) -> Option { let row2 = simd_swizzle!(tmp, row3, SHUFFLE02); let row3 = simd_swizzle!(row3, tmp, SHUFFLE13); - let tmp = (row2 * row3).reverse().rotate_lanes_right::<2>(); + let tmp = (row2 * row3).reverse().rotate_elements_right::<2>(); let minor0 = row1 * tmp; let minor1 = row0 * tmp; - let tmp = tmp.rotate_lanes_right::<2>(); + let tmp = tmp.rotate_elements_right::<2>(); let minor0 = (row1 * tmp) - minor0; let minor1 = (row0 * tmp) - minor1; - let minor1 = minor1.rotate_lanes_right::<2>(); + let minor1 = minor1.rotate_elements_right::<2>(); - let tmp = (row1 * row2).reverse().rotate_lanes_right::<2>(); + let tmp = (row1 * row2).reverse().rotate_elements_right::<2>(); let minor0 = (row3 * tmp) + minor0; let minor3 = row0 * tmp; - let tmp = tmp.rotate_lanes_right::<2>(); + let tmp = tmp.rotate_elements_right::<2>(); let minor0 = minor0 - row3 * tmp; let minor3 = row0 * tmp - minor3; - let minor3 = minor3.rotate_lanes_right::<2>(); + let minor3 = minor3.rotate_elements_right::<2>(); - let tmp = (row3 * row1.rotate_lanes_right::<2>()) + let tmp = (row3 * row1.rotate_elements_right::<2>()) .reverse() - .rotate_lanes_right::<2>(); - let row2 = row2.rotate_lanes_right::<2>(); + .rotate_elements_right::<2>(); + let row2 = row2.rotate_elements_right::<2>(); let minor0 = row2 * tmp + minor0; let minor2 = row0 * tmp; - let tmp = tmp.rotate_lanes_right::<2>(); + let tmp = tmp.rotate_elements_right::<2>(); let minor0 = minor0 - row2 * tmp; let minor2 = row0 * tmp - minor2; - let minor2 = minor2.rotate_lanes_right::<2>(); + let minor2 = minor2.rotate_elements_right::<2>(); - let tmp = (row0 * row1).reverse().rotate_lanes_right::<2>(); + let tmp = (row0 * row1).reverse().rotate_elements_right::<2>(); let minor2 = minor2 + row3 * tmp; let minor3 = row2 * tmp - minor3; - let tmp = tmp.rotate_lanes_right::<2>(); + let tmp = tmp.rotate_elements_right::<2>(); let minor2 = row3 * tmp - minor2; let minor3 = minor3 - row2 * tmp; - let tmp = (row0 * row3).reverse().rotate_lanes_right::<2>(); + let tmp = (row0 * row3).reverse().rotate_elements_right::<2>(); let minor1 = minor1 - row2 * tmp; let minor2 = row1 * tmp + minor2; - let tmp = tmp.rotate_lanes_right::<2>(); + let tmp = tmp.rotate_elements_right::<2>(); let minor1 = row2 * tmp + minor1; let minor2 = minor2 - row1 * tmp; - let tmp = (row0 * row2).reverse().rotate_lanes_right::<2>(); + let tmp = (row0 * row2).reverse().rotate_elements_right::<2>(); let minor1 = row3 * tmp + minor1; let minor3 = minor3 - row1 * tmp; - let tmp = tmp.rotate_lanes_right::<2>(); + let tmp = tmp.rotate_elements_right::<2>(); let minor1 = minor1 - row3 * tmp; let minor3 = row1 * tmp + minor3; let det = row0 * minor0; - let det = det.rotate_lanes_right::<2>() + det; - let det = det.reverse().rotate_lanes_right::<2>() + det; + let det = det.rotate_elements_right::<2>() + det; + let det = det.reverse().rotate_elements_right::<2>() + det; if det.reduce_sum() == 0. { return None; diff --git a/library/portable-simd/crates/core_simd/examples/nbody.rs b/library/portable-simd/crates/core_simd/examples/nbody.rs index df38a00967f..65820d1340b 100644 --- a/library/portable-simd/crates/core_simd/examples/nbody.rs +++ b/library/portable-simd/crates/core_simd/examples/nbody.rs @@ -1,11 +1,12 @@ #![feature(portable_simd)] +#![allow(clippy::excessive_precision)] extern crate std_float; /// Benchmarks game nbody code /// Taken from the `packed_simd` crate /// Run this benchmark with `cargo test --example nbody` mod nbody { - use core_simd::simd::*; + use core_simd::simd::prelude::*; #[allow(unused)] // False positive? use std_float::StdFloat; diff --git a/library/portable-simd/crates/core_simd/examples/spectral_norm.rs b/library/portable-simd/crates/core_simd/examples/spectral_norm.rs index d576bd0ccee..bc7934c2522 100644 --- a/library/portable-simd/crates/core_simd/examples/spectral_norm.rs +++ b/library/portable-simd/crates/core_simd/examples/spectral_norm.rs @@ -1,6 +1,6 @@ #![feature(portable_simd)] -use core_simd::simd::*; +use core_simd::simd::prelude::*; fn a(i: usize, j: usize) -> f64 { ((i + j) * (i + j + 1) / 2 + i + 1) as f64 diff --git a/library/portable-simd/crates/core_simd/src/core_simd_docs.md b/library/portable-simd/crates/core_simd/src/core_simd_docs.md index 15e8ed0253e..fa93155ff5e 100644 --- a/library/portable-simd/crates/core_simd/src/core_simd_docs.md +++ b/library/portable-simd/crates/core_simd/src/core_simd_docs.md @@ -2,3 +2,38 @@ Portable SIMD module. This module offers a portable abstraction for SIMD operations that is not bound to any particular hardware architecture. + +# What is "portable"? + +This module provides a SIMD implementation that is fast and predictable on any target. + +### Portable SIMD works on every target + +Unlike target-specific SIMD in `std::arch`, portable SIMD compiles for every target. +In this regard, it is just like "regular" Rust. + +### Portable SIMD is consistent between targets + +A program using portable SIMD can expect identical behavior on any target. +In most regards, [`Simd`] can be thought of as a parallelized `[T; N]` and operates like a sequence of `T`. + +This has one notable exception: a handful of older architectures (e.g. `armv7` and `powerpc`) flush [subnormal](`f32::is_subnormal`) `f32` values to zero. +On these architectures, subnormal `f32` input values are replaced with zeros, and any operation producing subnormal `f32` values produces zeros instead. +This doesn't affect most architectures or programs. + +### Operations use the best instructions available + +Operations provided by this module compile to the best available SIMD instructions. + +Portable SIMD is not a low-level vendor library, and operations in portable SIMD _do not_ necessarily map to a single instruction. +Instead, they map to a reasonable implementation of the operation for the target. + +Consistency between targets is not compromised to use faster or fewer instructions. +In some cases, `std::arch` will provide a faster function that has slightly different behavior than the `std::simd` equivalent. +For example, [`_mm_min_ps`](`core::arch::x86_64::_mm_min_ps`)[^1] can be slightly faster than [`SimdFloat::simd_min`](`num::SimdFloat::simd_min`), but does not conform to the IEEE standard also used by [`f32::min`]. +When necessary, [`Simd`] can be converted to the types provided by `std::arch` to make use of target-specific functions. + +Many targets simply don't have SIMD, or don't support SIMD for a particular element type. +In those cases, regular scalar operations are generated instead. + +[^1]: `_mm_min_ps(x, y)` is equivalent to `x.simd_lt(y).select(x, y)` diff --git a/library/portable-simd/crates/core_simd/src/fmt.rs b/library/portable-simd/crates/core_simd/src/fmt.rs index b7317969cbb..3a540f5a049 100644 --- a/library/portable-simd/crates/core_simd/src/fmt.rs +++ b/library/portable-simd/crates/core_simd/src/fmt.rs @@ -1,9 +1,9 @@ use crate::simd::{LaneCount, Simd, SimdElement, SupportedLaneCount}; use core::fmt; -impl fmt::Debug for Simd +impl fmt::Debug for Simd where - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, T: SimdElement + fmt::Debug, { /// A `Simd` has a debug format like the one for `[T]`: diff --git a/library/portable-simd/crates/core_simd/src/intrinsics.rs b/library/portable-simd/crates/core_simd/src/intrinsics.rs index dd6698e2ba5..b27893bc729 100644 --- a/library/portable-simd/crates/core_simd/src/intrinsics.rs +++ b/library/portable-simd/crates/core_simd/src/intrinsics.rs @@ -160,4 +160,10 @@ /// convert an exposed address back to a pointer pub(crate) fn simd_from_exposed_addr(addr: T) -> U; + + // Integer operations + pub(crate) fn simd_bswap(x: T) -> T; + pub(crate) fn simd_bitreverse(x: T) -> T; + pub(crate) fn simd_ctlz(x: T) -> T; + pub(crate) fn simd_cttz(x: T) -> T; } diff --git a/library/portable-simd/crates/core_simd/src/iter.rs b/library/portable-simd/crates/core_simd/src/iter.rs index 328c995b81d..b3732fd74d5 100644 --- a/library/portable-simd/crates/core_simd/src/iter.rs +++ b/library/portable-simd/crates/core_simd/src/iter.rs @@ -6,9 +6,9 @@ macro_rules! impl_traits { { $type:ty } => { - impl Sum for Simd<$type, LANES> + impl Sum for Simd<$type, N> where - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { #[inline] fn sum>(iter: I) -> Self { @@ -16,9 +16,9 @@ fn sum>(iter: I) -> Self { } } - impl Product for Simd<$type, LANES> + impl Product for Simd<$type, N> where - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { #[inline] fn product>(iter: I) -> Self { @@ -26,9 +26,9 @@ fn product>(iter: I) -> Self { } } - impl<'a, const LANES: usize> Sum<&'a Self> for Simd<$type, LANES> + impl<'a, const N: usize> Sum<&'a Self> for Simd<$type, N> where - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { #[inline] fn sum>(iter: I) -> Self { @@ -36,9 +36,9 @@ fn sum>(iter: I) -> Self { } } - impl<'a, const LANES: usize> Product<&'a Self> for Simd<$type, LANES> + impl<'a, const N: usize> Product<&'a Self> for Simd<$type, N> where - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { #[inline] fn product>(iter: I) -> Self { diff --git a/library/portable-simd/crates/core_simd/src/lane_count.rs b/library/portable-simd/crates/core_simd/src/lane_count.rs index 2b91eb9e800..4cd7265ed67 100644 --- a/library/portable-simd/crates/core_simd/src/lane_count.rs +++ b/library/portable-simd/crates/core_simd/src/lane_count.rs @@ -4,11 +4,11 @@ pub trait Sealed {} use sealed::Sealed; /// Specifies the number of lanes in a SIMD vector as a type. -pub struct LaneCount; +pub struct LaneCount; -impl LaneCount { +impl LaneCount { /// The number of bytes in a bitmask with this many lanes. - pub const BITMASK_LEN: usize = (LANES + 7) / 8; + pub const BITMASK_LEN: usize = (N + 7) / 8; } /// Statically guarantees that a lane count is marked as supported. @@ -21,7 +21,7 @@ pub trait SupportedLaneCount: Sealed { type BitMask: Copy + Default + AsRef<[u8]> + AsMut<[u8]>; } -impl Sealed for LaneCount {} +impl Sealed for LaneCount {} macro_rules! supported_lane_count { ($($lanes:literal),+) => { diff --git a/library/portable-simd/crates/core_simd/src/lib.rs b/library/portable-simd/crates/core_simd/src/lib.rs index fde406bda70..64ba9705ef5 100644 --- a/library/portable-simd/crates/core_simd/src/lib.rs +++ b/library/portable-simd/crates/core_simd/src/lib.rs @@ -5,6 +5,7 @@ const_mut_refs, convert_float_to_int, decl_macro, + inline_const, intra_doc_pointers, platform_intrinsics, repr_simd, @@ -14,10 +15,9 @@ strict_provenance, ptr_metadata )] -#![cfg_attr(feature = "generic_const_exprs", feature(generic_const_exprs))] -#![cfg_attr(feature = "generic_const_exprs", allow(incomplete_features))] #![warn(missing_docs, clippy::missing_inline_in_public_items)] // basically all items, really #![deny(unsafe_op_in_unsafe_fn, clippy::undocumented_unsafe_blocks)] +#![allow(internal_features)] #![unstable(feature = "portable_simd", issue = "86656")] //! Portable SIMD module. diff --git a/library/portable-simd/crates/core_simd/src/masks.rs b/library/portable-simd/crates/core_simd/src/masks.rs index fea687bdc1a..0623d2bf3d1 100644 --- a/library/portable-simd/crates/core_simd/src/masks.rs +++ b/library/portable-simd/crates/core_simd/src/masks.rs @@ -1,4 +1,4 @@ -//! Types and traits associated with masking lanes of vectors. +//! Types and traits associated with masking elements of vectors. //! Types representing #![allow(non_camel_case_types)] @@ -12,13 +12,9 @@ )] mod mask_impl; -mod to_bitmask; -pub use to_bitmask::ToBitMask; - -#[cfg(feature = "generic_const_exprs")] -pub use to_bitmask::{bitmask_len, ToBitMaskArray}; - -use crate::simd::{intrinsics, LaneCount, Simd, SimdElement, SimdPartialEq, SupportedLaneCount}; +use crate::simd::{ + cmp::SimdPartialEq, intrinsics, LaneCount, Simd, SimdCast, SimdElement, SupportedLaneCount, +}; use core::cmp::Ordering; use core::{fmt, mem}; @@ -32,13 +28,17 @@ mod sealed { /// prevent us from ever removing that bound, or from implementing `MaskElement` on /// non-`PartialEq` types in the future. pub trait Sealed { - fn valid(values: Simd) -> bool + fn valid(values: Simd) -> bool where - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, Self: SimdElement; fn eq(self, other: Self) -> bool; + fn as_usize(self) -> usize; + + type Unsigned: SimdElement; + const TRUE: Self; const FALSE: Self; @@ -50,15 +50,15 @@ fn valid(values: Simd) -> bool /// /// # Safety /// Type must be a signed integer. -pub unsafe trait MaskElement: SimdElement + Sealed {} +pub unsafe trait MaskElement: SimdElement + SimdCast + Sealed {} macro_rules! impl_element { - { $ty:ty } => { + { $ty:ty, $unsigned:ty } => { impl Sealed for $ty { #[inline] - fn valid(value: Simd) -> bool + fn valid(value: Simd) -> bool where - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { (value.simd_eq(Simd::splat(0 as _)) | value.simd_eq(Simd::splat(-1 as _))).all() } @@ -66,6 +66,13 @@ fn valid(value: Simd) -> bool #[inline] fn eq(self, other: Self) -> bool { self == other } + #[inline] + fn as_usize(self) -> usize { + self as usize + } + + type Unsigned = $unsigned; + const TRUE: Self = -1; const FALSE: Self = 0; } @@ -75,36 +82,36 @@ unsafe impl MaskElement for $ty {} } } -impl_element! { i8 } -impl_element! { i16 } -impl_element! { i32 } -impl_element! { i64 } -impl_element! { isize } +impl_element! { i8, u8 } +impl_element! { i16, u16 } +impl_element! { i32, u32 } +impl_element! { i64, u64 } +impl_element! { isize, usize } -/// A SIMD vector mask for `LANES` elements of width specified by `Element`. +/// A SIMD vector mask for `N` elements of width specified by `Element`. /// -/// Masks represent boolean inclusion/exclusion on a per-lane basis. +/// Masks represent boolean inclusion/exclusion on a per-element basis. /// /// The layout of this type is unspecified, and may change between platforms /// and/or Rust versions, and code should not assume that it is equivalent to -/// `[T; LANES]`. -#[cfg_attr(not(doc), repr(transparent))] // work around https://github.com/rust-lang/rust/issues/90435 -pub struct Mask(mask_impl::Mask) +/// `[T; N]`. +#[repr(transparent)] +pub struct Mask(mask_impl::Mask) where T: MaskElement, - LaneCount: SupportedLaneCount; + LaneCount: SupportedLaneCount; -impl Copy for Mask +impl Copy for Mask where T: MaskElement, - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { } -impl Clone for Mask +impl Clone for Mask where T: MaskElement, - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { #[inline] fn clone(&self) -> Self { @@ -112,12 +119,12 @@ fn clone(&self) -> Self { } } -impl Mask +impl Mask where T: MaskElement, - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { - /// Construct a mask by setting all lanes to the given value. + /// Construct a mask by setting all elements to the given value. #[inline] pub fn splat(value: bool) -> Self { Self(mask_impl::Mask::splat(value)) @@ -125,7 +132,7 @@ pub fn splat(value: bool) -> Self { /// Converts an array of bools to a SIMD mask. #[inline] - pub fn from_array(array: [bool; LANES]) -> Self { + pub fn from_array(array: [bool; N]) -> Self { // SAFETY: Rust's bool has a layout of 1 byte (u8) with a value of // true: 0b_0000_0001 // false: 0b_0000_0000 @@ -133,16 +140,15 @@ pub fn splat(value: bool) -> Self { // This would be hypothetically valid as an "in-place" transmute, // but these are "dependently-sized" types, so copy elision it is! unsafe { - let bytes: [u8; LANES] = mem::transmute_copy(&array); - let bools: Simd = - intrinsics::simd_ne(Simd::from_array(bytes), Simd::splat(0u8)); + let bytes: [u8; N] = mem::transmute_copy(&array); + let bools: Simd = intrinsics::simd_ne(Simd::from_array(bytes), Simd::splat(0u8)); Mask::from_int_unchecked(intrinsics::simd_cast(bools)) } } /// Converts a SIMD mask to an array of bools. #[inline] - pub fn to_array(self) -> [bool; LANES] { + pub fn to_array(self) -> [bool; N] { // This follows mostly the same logic as from_array. // SAFETY: Rust's bool has a layout of 1 byte (u8) with a value of // true: 0b_0000_0001 @@ -154,7 +160,7 @@ pub fn splat(value: bool) -> Self { // This would be hypothetically valid as an "in-place" transmute, // but these are "dependently-sized" types, so copy elision it is! unsafe { - let mut bytes: Simd = intrinsics::simd_cast(self.to_int()); + let mut bytes: Simd = intrinsics::simd_cast(self.to_int()); bytes &= Simd::splat(1i8); mem::transmute_copy(&bytes) } @@ -164,10 +170,10 @@ pub fn splat(value: bool) -> Self { /// represents `true`. /// /// # Safety - /// All lanes must be either 0 or -1. + /// All elements must be either 0 or -1. #[inline] #[must_use = "method returns a new mask and does not mutate the original value"] - pub unsafe fn from_int_unchecked(value: Simd) -> Self { + pub unsafe fn from_int_unchecked(value: Simd) -> Self { // Safety: the caller must confirm this invariant unsafe { Self(mask_impl::Mask::from_int_unchecked(value)) } } @@ -176,11 +182,11 @@ pub unsafe fn from_int_unchecked(value: Simd) -> Self { /// represents `true`. /// /// # Panics - /// Panics if any lane is not 0 or -1. + /// Panics if any element is not 0 or -1. #[inline] #[must_use = "method returns a new mask and does not mutate the original value"] #[track_caller] - pub fn from_int(value: Simd) -> Self { + pub fn from_int(value: Simd) -> Self { assert!(T::valid(value), "all values must be either 0 or -1",); // Safety: the validity has been checked unsafe { Self::from_int_unchecked(value) } @@ -190,121 +196,244 @@ pub fn from_int(value: Simd) -> Self { /// represents `true`. #[inline] #[must_use = "method returns a new vector and does not mutate the original value"] - pub fn to_int(self) -> Simd { + pub fn to_int(self) -> Simd { self.0.to_int() } - /// Converts the mask to a mask of any other lane size. + /// Converts the mask to a mask of any other element size. #[inline] #[must_use = "method returns a new mask and does not mutate the original value"] - pub fn cast(self) -> Mask { + pub fn cast(self) -> Mask { Mask(self.0.convert()) } - /// Tests the value of the specified lane. + /// Tests the value of the specified element. /// /// # Safety - /// `lane` must be less than `LANES`. + /// `index` must be less than `self.len()`. #[inline] #[must_use = "method returns a new bool and does not mutate the original value"] - pub unsafe fn test_unchecked(&self, lane: usize) -> bool { + pub unsafe fn test_unchecked(&self, index: usize) -> bool { // Safety: the caller must confirm this invariant - unsafe { self.0.test_unchecked(lane) } + unsafe { self.0.test_unchecked(index) } } - /// Tests the value of the specified lane. + /// Tests the value of the specified element. /// /// # Panics - /// Panics if `lane` is greater than or equal to the number of lanes in the vector. + /// Panics if `index` is greater than or equal to the number of elements in the vector. #[inline] #[must_use = "method returns a new bool and does not mutate the original value"] #[track_caller] - pub fn test(&self, lane: usize) -> bool { - assert!(lane < LANES, "lane index out of range"); - // Safety: the lane index has been checked - unsafe { self.test_unchecked(lane) } + pub fn test(&self, index: usize) -> bool { + assert!(index < N, "element index out of range"); + // Safety: the element index has been checked + unsafe { self.test_unchecked(index) } } - /// Sets the value of the specified lane. + /// Sets the value of the specified element. /// /// # Safety - /// `lane` must be less than `LANES`. + /// `index` must be less than `self.len()`. #[inline] - pub unsafe fn set_unchecked(&mut self, lane: usize, value: bool) { + pub unsafe fn set_unchecked(&mut self, index: usize, value: bool) { // Safety: the caller must confirm this invariant unsafe { - self.0.set_unchecked(lane, value); + self.0.set_unchecked(index, value); } } - /// Sets the value of the specified lane. + /// Sets the value of the specified element. /// /// # Panics - /// Panics if `lane` is greater than or equal to the number of lanes in the vector. + /// Panics if `index` is greater than or equal to the number of elements in the vector. #[inline] #[track_caller] - pub fn set(&mut self, lane: usize, value: bool) { - assert!(lane < LANES, "lane index out of range"); - // Safety: the lane index has been checked + pub fn set(&mut self, index: usize, value: bool) { + assert!(index < N, "element index out of range"); + // Safety: the element index has been checked unsafe { - self.set_unchecked(lane, value); + self.set_unchecked(index, value); } } - /// Returns true if any lane is set, or false otherwise. + /// Returns true if any element is set, or false otherwise. #[inline] #[must_use = "method returns a new bool and does not mutate the original value"] pub fn any(self) -> bool { self.0.any() } - /// Returns true if all lanes are set, or false otherwise. + /// Returns true if all elements are set, or false otherwise. #[inline] #[must_use = "method returns a new bool and does not mutate the original value"] pub fn all(self) -> bool { self.0.all() } + + /// Create a bitmask from a mask. + /// + /// Each bit is set if the corresponding element in the mask is `true`. + /// If the mask contains more than 64 elements, the bitmask is truncated to the first 64. + #[inline] + #[must_use = "method returns a new integer and does not mutate the original value"] + pub fn to_bitmask(self) -> u64 { + self.0.to_bitmask_integer() + } + + /// Create a mask from a bitmask. + /// + /// For each bit, if it is set, the corresponding element in the mask is set to `true`. + /// If the mask contains more than 64 elements, the remainder are set to `false`. + #[inline] + #[must_use = "method returns a new mask and does not mutate the original value"] + pub fn from_bitmask(bitmask: u64) -> Self { + Self(mask_impl::Mask::from_bitmask_integer(bitmask)) + } + + /// Create a bitmask vector from a mask. + /// + /// Each bit is set if the corresponding element in the mask is `true`. + /// The remaining bits are unset. + /// + /// The bits are packed into the first N bits of the vector: + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::mask32x8; + /// let mask = mask32x8::from_array([true, false, true, false, false, false, true, false]); + /// assert_eq!(mask.to_bitmask_vector()[0], 0b01000101); + /// ``` + #[inline] + #[must_use = "method returns a new integer and does not mutate the original value"] + pub fn to_bitmask_vector(self) -> Simd { + self.0.to_bitmask_vector() + } + + /// Create a mask from a bitmask vector. + /// + /// For each bit, if it is set, the corresponding element in the mask is set to `true`. + /// + /// The bits are packed into the first N bits of the vector: + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::{mask32x8, u8x8}; + /// let bitmask = u8x8::from_array([0b01000101, 0, 0, 0, 0, 0, 0, 0]); + /// assert_eq!( + /// mask32x8::from_bitmask_vector(bitmask), + /// mask32x8::from_array([true, false, true, false, false, false, true, false]), + /// ); + /// ``` + #[inline] + #[must_use = "method returns a new mask and does not mutate the original value"] + pub fn from_bitmask_vector(bitmask: Simd) -> Self { + Self(mask_impl::Mask::from_bitmask_vector(bitmask)) + } + + /// Find the index of the first set element. + /// + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::mask32x8; + /// assert_eq!(mask32x8::splat(false).first_set(), None); + /// assert_eq!(mask32x8::splat(true).first_set(), Some(0)); + /// + /// let mask = mask32x8::from_array([false, true, false, false, true, false, false, true]); + /// assert_eq!(mask.first_set(), Some(1)); + /// ``` + #[inline] + #[must_use = "method returns the index and does not mutate the original value"] + pub fn first_set(self) -> Option { + // If bitmasks are efficient, using them is better + if cfg!(target_feature = "sse") && N <= 64 { + let tz = self.to_bitmask().trailing_zeros(); + return if tz == 64 { None } else { Some(tz as usize) }; + } + + // To find the first set index: + // * create a vector 0..N + // * replace unset mask elements in that vector with -1 + // * perform _unsigned_ reduce-min + // * check if the result is -1 or an index + + let index = Simd::from_array( + const { + let mut index = [0; N]; + let mut i = 0; + while i < N { + index[i] = i; + i += 1; + } + index + }, + ); + + // Safety: the input and output are integer vectors + let index: Simd = unsafe { intrinsics::simd_cast(index) }; + + let masked_index = self.select(index, Self::splat(true).to_int()); + + // Safety: the input and output are integer vectors + let masked_index: Simd = unsafe { intrinsics::simd_cast(masked_index) }; + + // Safety: the input is an integer vector + let min_index: T::Unsigned = unsafe { intrinsics::simd_reduce_min(masked_index) }; + + // Safety: the return value is the unsigned version of T + let min_index: T = unsafe { core::mem::transmute_copy(&min_index) }; + + if min_index.eq(T::TRUE) { + None + } else { + Some(min_index.as_usize()) + } + } } // vector/array conversion -impl From<[bool; LANES]> for Mask +impl From<[bool; N]> for Mask where T: MaskElement, - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { #[inline] - fn from(array: [bool; LANES]) -> Self { + fn from(array: [bool; N]) -> Self { Self::from_array(array) } } -impl From> for [bool; LANES] +impl From> for [bool; N] where T: MaskElement, - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { #[inline] - fn from(vector: Mask) -> Self { + fn from(vector: Mask) -> Self { vector.to_array() } } -impl Default for Mask +impl Default for Mask where T: MaskElement, - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { #[inline] - #[must_use = "method returns a defaulted mask with all lanes set to false (0)"] + #[must_use = "method returns a defaulted mask with all elements set to false (0)"] fn default() -> Self { Self::splat(false) } } -impl PartialEq for Mask +impl PartialEq for Mask where T: MaskElement + PartialEq, - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { #[inline] #[must_use = "method returns a new bool and does not mutate the original value"] @@ -313,10 +442,10 @@ fn eq(&self, other: &Self) -> bool { } } -impl PartialOrd for Mask +impl PartialOrd for Mask where T: MaskElement + PartialOrd, - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { #[inline] #[must_use = "method returns a new Ordering and does not mutate the original value"] @@ -325,23 +454,23 @@ fn partial_cmp(&self, other: &Self) -> Option { } } -impl fmt::Debug for Mask +impl fmt::Debug for Mask where T: MaskElement + fmt::Debug, - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { #[inline] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_list() - .entries((0..LANES).map(|lane| self.test(lane))) + .entries((0..N).map(|i| self.test(i))) .finish() } } -impl core::ops::BitAnd for Mask +impl core::ops::BitAnd for Mask where T: MaskElement, - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { type Output = Self; #[inline] @@ -351,10 +480,10 @@ fn bitand(self, rhs: Self) -> Self { } } -impl core::ops::BitAnd for Mask +impl core::ops::BitAnd for Mask where T: MaskElement, - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { type Output = Self; #[inline] @@ -364,23 +493,23 @@ fn bitand(self, rhs: bool) -> Self { } } -impl core::ops::BitAnd> for bool +impl core::ops::BitAnd> for bool where T: MaskElement, - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { - type Output = Mask; + type Output = Mask; #[inline] #[must_use = "method returns a new mask and does not mutate the original value"] - fn bitand(self, rhs: Mask) -> Mask { + fn bitand(self, rhs: Mask) -> Mask { Mask::splat(self) & rhs } } -impl core::ops::BitOr for Mask +impl core::ops::BitOr for Mask where T: MaskElement, - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { type Output = Self; #[inline] @@ -390,10 +519,10 @@ fn bitor(self, rhs: Self) -> Self { } } -impl core::ops::BitOr for Mask +impl core::ops::BitOr for Mask where T: MaskElement, - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { type Output = Self; #[inline] @@ -403,23 +532,23 @@ fn bitor(self, rhs: bool) -> Self { } } -impl core::ops::BitOr> for bool +impl core::ops::BitOr> for bool where T: MaskElement, - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { - type Output = Mask; + type Output = Mask; #[inline] #[must_use = "method returns a new mask and does not mutate the original value"] - fn bitor(self, rhs: Mask) -> Mask { + fn bitor(self, rhs: Mask) -> Mask { Mask::splat(self) | rhs } } -impl core::ops::BitXor for Mask +impl core::ops::BitXor for Mask where T: MaskElement, - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { type Output = Self; #[inline] @@ -429,10 +558,10 @@ fn bitxor(self, rhs: Self) -> Self::Output { } } -impl core::ops::BitXor for Mask +impl core::ops::BitXor for Mask where T: MaskElement, - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { type Output = Self; #[inline] @@ -442,25 +571,25 @@ fn bitxor(self, rhs: bool) -> Self::Output { } } -impl core::ops::BitXor> for bool +impl core::ops::BitXor> for bool where T: MaskElement, - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { - type Output = Mask; + type Output = Mask; #[inline] #[must_use = "method returns a new mask and does not mutate the original value"] - fn bitxor(self, rhs: Mask) -> Self::Output { + fn bitxor(self, rhs: Mask) -> Self::Output { Mask::splat(self) ^ rhs } } -impl core::ops::Not for Mask +impl core::ops::Not for Mask where T: MaskElement, - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { - type Output = Mask; + type Output = Mask; #[inline] #[must_use = "method returns a new mask and does not mutate the original value"] fn not(self) -> Self::Output { @@ -468,10 +597,10 @@ fn not(self) -> Self::Output { } } -impl core::ops::BitAndAssign for Mask +impl core::ops::BitAndAssign for Mask where T: MaskElement, - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { #[inline] fn bitand_assign(&mut self, rhs: Self) { @@ -479,10 +608,10 @@ fn bitand_assign(&mut self, rhs: Self) { } } -impl core::ops::BitAndAssign for Mask +impl core::ops::BitAndAssign for Mask where T: MaskElement, - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { #[inline] fn bitand_assign(&mut self, rhs: bool) { @@ -490,10 +619,10 @@ fn bitand_assign(&mut self, rhs: bool) { } } -impl core::ops::BitOrAssign for Mask +impl core::ops::BitOrAssign for Mask where T: MaskElement, - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { #[inline] fn bitor_assign(&mut self, rhs: Self) { @@ -501,10 +630,10 @@ fn bitor_assign(&mut self, rhs: Self) { } } -impl core::ops::BitOrAssign for Mask +impl core::ops::BitOrAssign for Mask where T: MaskElement, - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { #[inline] fn bitor_assign(&mut self, rhs: bool) { @@ -512,10 +641,10 @@ fn bitor_assign(&mut self, rhs: bool) { } } -impl core::ops::BitXorAssign for Mask +impl core::ops::BitXorAssign for Mask where T: MaskElement, - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { #[inline] fn bitxor_assign(&mut self, rhs: Self) { @@ -523,10 +652,10 @@ fn bitxor_assign(&mut self, rhs: Self) { } } -impl core::ops::BitXorAssign for Mask +impl core::ops::BitXorAssign for Mask where T: MaskElement, - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { #[inline] fn bitxor_assign(&mut self, rhs: bool) { @@ -537,12 +666,12 @@ fn bitxor_assign(&mut self, rhs: bool) { macro_rules! impl_from { { $from:ty => $($to:ty),* } => { $( - impl From> for Mask<$to, LANES> + impl From> for Mask<$to, N> where - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { #[inline] - fn from(value: Mask<$from, LANES>) -> Self { + fn from(value: Mask<$from, N>) -> Self { value.cast() } } diff --git a/library/portable-simd/crates/core_simd/src/masks/bitmask.rs b/library/portable-simd/crates/core_simd/src/masks/bitmask.rs index 20465ba9b07..6ddff07fea2 100644 --- a/library/portable-simd/crates/core_simd/src/masks/bitmask.rs +++ b/library/portable-simd/crates/core_simd/src/masks/bitmask.rs @@ -1,30 +1,30 @@ #![allow(unused_imports)] use super::MaskElement; use crate::simd::intrinsics; -use crate::simd::{LaneCount, Simd, SupportedLaneCount, ToBitMask}; +use crate::simd::{LaneCount, Simd, SupportedLaneCount}; use core::marker::PhantomData; /// A mask where each lane is represented by a single bit. #[repr(transparent)] -pub struct Mask( - as SupportedLaneCount>::BitMask, +pub struct Mask( + as SupportedLaneCount>::BitMask, PhantomData, ) where T: MaskElement, - LaneCount: SupportedLaneCount; + LaneCount: SupportedLaneCount; -impl Copy for Mask +impl Copy for Mask where T: MaskElement, - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { } -impl Clone for Mask +impl Clone for Mask where T: MaskElement, - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { #[inline] fn clone(&self) -> Self { @@ -32,10 +32,10 @@ fn clone(&self) -> Self { } } -impl PartialEq for Mask +impl PartialEq for Mask where T: MaskElement, - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { #[inline] fn eq(&self, other: &Self) -> bool { @@ -43,10 +43,10 @@ fn eq(&self, other: &Self) -> bool { } } -impl PartialOrd for Mask +impl PartialOrd for Mask where T: MaskElement, - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { #[inline] fn partial_cmp(&self, other: &Self) -> Option { @@ -54,17 +54,17 @@ fn partial_cmp(&self, other: &Self) -> Option { } } -impl Eq for Mask +impl Eq for Mask where T: MaskElement, - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { } -impl Ord for Mask +impl Ord for Mask where T: MaskElement, - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { #[inline] fn cmp(&self, other: &Self) -> core::cmp::Ordering { @@ -72,22 +72,22 @@ fn cmp(&self, other: &Self) -> core::cmp::Ordering { } } -impl Mask +impl Mask where T: MaskElement, - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { #[inline] #[must_use = "method returns a new mask and does not mutate the original value"] pub fn splat(value: bool) -> Self { - let mut mask = as SupportedLaneCount>::BitMask::default(); + let mut mask = as SupportedLaneCount>::BitMask::default(); if value { mask.as_mut().fill(u8::MAX) } else { mask.as_mut().fill(u8::MIN) } - if LANES % 8 > 0 { - *mask.as_mut().last_mut().unwrap() &= u8::MAX >> (8 - LANES % 8); + if N % 8 > 0 { + *mask.as_mut().last_mut().unwrap() &= u8::MAX >> (8 - N % 8); } Self(mask, PhantomData) } @@ -107,7 +107,7 @@ pub unsafe fn set_unchecked(&mut self, lane: usize, value: bool) { #[inline] #[must_use = "method returns a new vector and does not mutate the original value"] - pub fn to_int(self) -> Simd { + pub fn to_int(self) -> Simd { unsafe { intrinsics::simd_select_bitmask(self.0, Simd::splat(T::TRUE), Simd::splat(T::FALSE)) } @@ -115,51 +115,47 @@ pub fn to_int(self) -> Simd { #[inline] #[must_use = "method returns a new mask and does not mutate the original value"] - pub unsafe fn from_int_unchecked(value: Simd) -> Self { + pub unsafe fn from_int_unchecked(value: Simd) -> Self { unsafe { Self(intrinsics::simd_bitmask(value), PhantomData) } } - #[cfg(feature = "generic_const_exprs")] #[inline] - #[must_use = "method returns a new array and does not mutate the original value"] - pub fn to_bitmask_array(self) -> [u8; N] { - assert!(core::mem::size_of::() == N); - - // Safety: converting an integer to an array of bytes of the same size is safe - unsafe { core::mem::transmute_copy(&self.0) } - } - - #[cfg(feature = "generic_const_exprs")] - #[inline] - #[must_use = "method returns a new mask and does not mutate the original value"] - pub fn from_bitmask_array(bitmask: [u8; N]) -> Self { - assert!(core::mem::size_of::() == N); - - // Safety: converting an array of bytes to an integer of the same size is safe - Self(unsafe { core::mem::transmute_copy(&bitmask) }, PhantomData) - } - - #[inline] - pub fn to_bitmask_integer(self) -> U - where - super::Mask: ToBitMask, - { - // Safety: these are the same types - unsafe { core::mem::transmute_copy(&self.0) } - } - - #[inline] - pub fn from_bitmask_integer(bitmask: U) -> Self - where - super::Mask: ToBitMask, - { - // Safety: these are the same types - unsafe { Self(core::mem::transmute_copy(&bitmask), PhantomData) } + #[must_use = "method returns a new vector and does not mutate the original value"] + pub fn to_bitmask_vector(self) -> Simd { + let mut bitmask = Simd::splat(0); + bitmask.as_mut_array()[..self.0.as_ref().len()].copy_from_slice(self.0.as_ref()); + bitmask } #[inline] #[must_use = "method returns a new mask and does not mutate the original value"] - pub fn convert(self) -> Mask + pub fn from_bitmask_vector(bitmask: Simd) -> Self { + let mut bytes = as SupportedLaneCount>::BitMask::default(); + let len = bytes.as_ref().len(); + bytes.as_mut().copy_from_slice(&bitmask.as_array()[..len]); + Self(bytes, PhantomData) + } + + #[inline] + pub fn to_bitmask_integer(self) -> u64 { + let mut bitmask = [0u8; 8]; + bitmask[..self.0.as_ref().len()].copy_from_slice(self.0.as_ref()); + u64::from_ne_bytes(bitmask) + } + + #[inline] + pub fn from_bitmask_integer(bitmask: u64) -> Self { + let mut bytes = as SupportedLaneCount>::BitMask::default(); + let len = bytes.as_mut().len(); + bytes + .as_mut() + .copy_from_slice(&bitmask.to_ne_bytes()[..len]); + Self(bytes, PhantomData) + } + + #[inline] + #[must_use = "method returns a new mask and does not mutate the original value"] + pub fn convert(self) -> Mask where U: MaskElement, { @@ -180,11 +176,11 @@ pub fn all(self) -> bool { } } -impl core::ops::BitAnd for Mask +impl core::ops::BitAnd for Mask where T: MaskElement, - LaneCount: SupportedLaneCount, - as SupportedLaneCount>::BitMask: AsRef<[u8]> + AsMut<[u8]>, + LaneCount: SupportedLaneCount, + as SupportedLaneCount>::BitMask: AsRef<[u8]> + AsMut<[u8]>, { type Output = Self; #[inline] @@ -197,11 +193,11 @@ fn bitand(mut self, rhs: Self) -> Self { } } -impl core::ops::BitOr for Mask +impl core::ops::BitOr for Mask where T: MaskElement, - LaneCount: SupportedLaneCount, - as SupportedLaneCount>::BitMask: AsRef<[u8]> + AsMut<[u8]>, + LaneCount: SupportedLaneCount, + as SupportedLaneCount>::BitMask: AsRef<[u8]> + AsMut<[u8]>, { type Output = Self; #[inline] @@ -214,10 +210,10 @@ fn bitor(mut self, rhs: Self) -> Self { } } -impl core::ops::BitXor for Mask +impl core::ops::BitXor for Mask where T: MaskElement, - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { type Output = Self; #[inline] @@ -230,10 +226,10 @@ fn bitxor(mut self, rhs: Self) -> Self::Output { } } -impl core::ops::Not for Mask +impl core::ops::Not for Mask where T: MaskElement, - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { type Output = Self; #[inline] @@ -242,8 +238,8 @@ fn not(mut self) -> Self::Output { for x in self.0.as_mut() { *x = !*x; } - if LANES % 8 > 0 { - *self.0.as_mut().last_mut().unwrap() &= u8::MAX >> (8 - LANES % 8); + if N % 8 > 0 { + *self.0.as_mut().last_mut().unwrap() &= u8::MAX >> (8 - N % 8); } self } diff --git a/library/portable-simd/crates/core_simd/src/masks/full_masks.rs b/library/portable-simd/crates/core_simd/src/masks/full_masks.rs index 1d13c45b8e7..63964f455e0 100644 --- a/library/portable-simd/crates/core_simd/src/masks/full_masks.rs +++ b/library/portable-simd/crates/core_simd/src/masks/full_masks.rs @@ -1,29 +1,25 @@ //! Masks that take up full SIMD vector registers. -use super::MaskElement; use crate::simd::intrinsics; -use crate::simd::{LaneCount, Simd, SupportedLaneCount, ToBitMask}; - -#[cfg(feature = "generic_const_exprs")] -use crate::simd::ToBitMaskArray; +use crate::simd::{LaneCount, MaskElement, Simd, SupportedLaneCount}; #[repr(transparent)] -pub struct Mask(Simd) +pub struct Mask(Simd) where T: MaskElement, - LaneCount: SupportedLaneCount; + LaneCount: SupportedLaneCount; -impl Copy for Mask +impl Copy for Mask where T: MaskElement, - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { } -impl Clone for Mask +impl Clone for Mask where T: MaskElement, - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { #[inline] #[must_use = "method returns a new mask and does not mutate the original value"] @@ -32,10 +28,10 @@ fn clone(&self) -> Self { } } -impl PartialEq for Mask +impl PartialEq for Mask where T: MaskElement + PartialEq, - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { #[inline] fn eq(&self, other: &Self) -> bool { @@ -43,10 +39,10 @@ fn eq(&self, other: &Self) -> bool { } } -impl PartialOrd for Mask +impl PartialOrd for Mask where T: MaskElement + PartialOrd, - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { #[inline] fn partial_cmp(&self, other: &Self) -> Option { @@ -54,17 +50,17 @@ fn partial_cmp(&self, other: &Self) -> Option { } } -impl Eq for Mask +impl Eq for Mask where T: MaskElement + Eq, - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { } -impl Ord for Mask +impl Ord for Mask where T: MaskElement + Ord, - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { #[inline] fn cmp(&self, other: &Self) -> core::cmp::Ordering { @@ -101,10 +97,10 @@ fn reverse_bits(self, n: usize) -> Self { impl_reverse_bits! { u8, u16, u32, u64 } -impl Mask +impl Mask where T: MaskElement, - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { #[inline] #[must_use = "method returns a new mask and does not mutate the original value"] @@ -125,19 +121,19 @@ pub unsafe fn set_unchecked(&mut self, lane: usize, value: bool) { #[inline] #[must_use = "method returns a new vector and does not mutate the original value"] - pub fn to_int(self) -> Simd { + pub fn to_int(self) -> Simd { self.0 } #[inline] #[must_use = "method returns a new mask and does not mutate the original value"] - pub unsafe fn from_int_unchecked(value: Simd) -> Self { + pub unsafe fn from_int_unchecked(value: Simd) -> Self { Self(value) } #[inline] #[must_use = "method returns a new mask and does not mutate the original value"] - pub fn convert(self) -> Mask + pub fn convert(self) -> Mask where U: MaskElement, { @@ -145,62 +141,50 @@ pub fn convert(self) -> Mask unsafe { Mask(intrinsics::simd_cast(self.0)) } } - #[cfg(feature = "generic_const_exprs")] #[inline] - #[must_use = "method returns a new array and does not mutate the original value"] - pub fn to_bitmask_array(self) -> [u8; N] - where - super::Mask: ToBitMaskArray, - [(); as ToBitMaskArray>::BYTES]: Sized, - { - assert_eq!( as ToBitMaskArray>::BYTES, N); + #[must_use = "method returns a new vector and does not mutate the original value"] + pub fn to_bitmask_vector(self) -> Simd { + let mut bitmask = Simd::splat(0); - // Safety: N is the correct bitmask size + // Safety: Bytes is the right size array unsafe { // Compute the bitmask - let bitmask: [u8; as ToBitMaskArray>::BYTES] = + let mut bytes: as SupportedLaneCount>::BitMask = intrinsics::simd_bitmask(self.0); - // Transmute to the return type, previously asserted to be the same size - let mut bitmask: [u8; N] = core::mem::transmute_copy(&bitmask); - // LLVM assumes bit order should match endianness if cfg!(target_endian = "big") { - for x in bitmask.as_mut() { - *x = x.reverse_bits(); + for x in bytes.as_mut() { + *x = x.reverse_bits() } - }; + } - bitmask + bitmask.as_mut_array()[..bytes.as_ref().len()].copy_from_slice(bytes.as_ref()); } + + bitmask } - #[cfg(feature = "generic_const_exprs")] #[inline] #[must_use = "method returns a new mask and does not mutate the original value"] - pub fn from_bitmask_array(mut bitmask: [u8; N]) -> Self - where - super::Mask: ToBitMaskArray, - [(); as ToBitMaskArray>::BYTES]: Sized, - { - assert_eq!( as ToBitMaskArray>::BYTES, N); + pub fn from_bitmask_vector(bitmask: Simd) -> Self { + let mut bytes = as SupportedLaneCount>::BitMask::default(); - // Safety: N is the correct bitmask size + // Safety: Bytes is the right size array unsafe { + let len = bytes.as_ref().len(); + bytes.as_mut().copy_from_slice(&bitmask.as_array()[..len]); + // LLVM assumes bit order should match endianness if cfg!(target_endian = "big") { - for x in bitmask.as_mut() { + for x in bytes.as_mut() { *x = x.reverse_bits(); } } - // Transmute to the bitmask type, previously asserted to be the same size - let bitmask: [u8; as ToBitMaskArray>::BYTES] = - core::mem::transmute_copy(&bitmask); - // Compute the regular mask Self::from_int_unchecked(intrinsics::simd_select_bitmask( - bitmask, + bytes, Self::splat(true).to_int(), Self::splat(false).to_int(), )) @@ -208,40 +192,81 @@ pub fn convert(self) -> Mask } #[inline] - pub(crate) fn to_bitmask_integer(self) -> U + unsafe fn to_bitmask_impl(self) -> U where - super::Mask: ToBitMask, + LaneCount: SupportedLaneCount, { - // Safety: U is required to be the appropriate bitmask type - let bitmask: U = unsafe { intrinsics::simd_bitmask(self.0) }; + let resized = self.to_int().resize::(T::FALSE); + + // Safety: `resized` is an integer vector with length M, which must match T + let bitmask: U = unsafe { intrinsics::simd_bitmask(resized) }; // LLVM assumes bit order should match endianness if cfg!(target_endian = "big") { - bitmask.reverse_bits(LANES) + bitmask.reverse_bits(M) } else { bitmask } } #[inline] - pub(crate) fn from_bitmask_integer(bitmask: U) -> Self + unsafe fn from_bitmask_impl(bitmask: U) -> Self where - super::Mask: ToBitMask, + LaneCount: SupportedLaneCount, { // LLVM assumes bit order should match endianness let bitmask = if cfg!(target_endian = "big") { - bitmask.reverse_bits(LANES) + bitmask.reverse_bits(M) } else { bitmask }; - // Safety: U is required to be the appropriate bitmask type - unsafe { - Self::from_int_unchecked(intrinsics::simd_select_bitmask( + // SAFETY: `mask` is the correct bitmask type for a u64 bitmask + let mask: Simd = unsafe { + intrinsics::simd_select_bitmask( bitmask, - Self::splat(true).to_int(), - Self::splat(false).to_int(), - )) + Simd::::splat(T::TRUE), + Simd::::splat(T::FALSE), + ) + }; + + // SAFETY: `mask` only contains `T::TRUE` or `T::FALSE` + unsafe { Self::from_int_unchecked(mask.resize::(T::FALSE)) } + } + + #[inline] + pub(crate) fn to_bitmask_integer(self) -> u64 { + // TODO modify simd_bitmask to zero-extend output, making this unnecessary + if N <= 8 { + // Safety: bitmask matches length + unsafe { self.to_bitmask_impl::() as u64 } + } else if N <= 16 { + // Safety: bitmask matches length + unsafe { self.to_bitmask_impl::() as u64 } + } else if N <= 32 { + // Safety: bitmask matches length + unsafe { self.to_bitmask_impl::() as u64 } + } else { + // Safety: bitmask matches length + unsafe { self.to_bitmask_impl::() } + } + } + + #[inline] + pub(crate) fn from_bitmask_integer(bitmask: u64) -> Self { + // TODO modify simd_bitmask_select to truncate input, making this unnecessary + if N <= 8 { + // Safety: bitmask matches length + unsafe { Self::from_bitmask_impl::(bitmask as u8) } + } else if N <= 16 { + // Safety: bitmask matches length + unsafe { Self::from_bitmask_impl::(bitmask as u16) } + } else if N <= 32 { + // Safety: bitmask matches length + unsafe { Self::from_bitmask_impl::(bitmask as u32) } + } else { + // Safety: bitmask matches length + unsafe { Self::from_bitmask_impl::(bitmask) } } } @@ -260,21 +285,21 @@ pub fn all(self) -> bool { } } -impl From> for Simd +impl From> for Simd where T: MaskElement, - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { #[inline] - fn from(value: Mask) -> Self { + fn from(value: Mask) -> Self { value.0 } } -impl core::ops::BitAnd for Mask +impl core::ops::BitAnd for Mask where T: MaskElement, - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { type Output = Self; #[inline] @@ -285,10 +310,10 @@ fn bitand(self, rhs: Self) -> Self { } } -impl core::ops::BitOr for Mask +impl core::ops::BitOr for Mask where T: MaskElement, - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { type Output = Self; #[inline] @@ -299,10 +324,10 @@ fn bitor(self, rhs: Self) -> Self { } } -impl core::ops::BitXor for Mask +impl core::ops::BitXor for Mask where T: MaskElement, - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { type Output = Self; #[inline] @@ -313,10 +338,10 @@ fn bitxor(self, rhs: Self) -> Self { } } -impl core::ops::Not for Mask +impl core::ops::Not for Mask where T: MaskElement, - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { type Output = Self; #[inline] diff --git a/library/portable-simd/crates/core_simd/src/masks/to_bitmask.rs b/library/portable-simd/crates/core_simd/src/masks/to_bitmask.rs deleted file mode 100644 index fc7d6b781f2..00000000000 --- a/library/portable-simd/crates/core_simd/src/masks/to_bitmask.rs +++ /dev/null @@ -1,97 +0,0 @@ -use super::{mask_impl, Mask, MaskElement}; -use crate::simd::{LaneCount, SupportedLaneCount}; - -mod sealed { - pub trait Sealed {} -} -pub use sealed::Sealed; - -impl Sealed for Mask -where - T: MaskElement, - LaneCount: SupportedLaneCount, -{ -} - -/// Converts masks to and from integer bitmasks. -/// -/// Each bit of the bitmask corresponds to a mask lane, starting with the LSB. -pub trait ToBitMask: Sealed { - /// The integer bitmask type. - type BitMask; - - /// Converts a mask to a bitmask. - fn to_bitmask(self) -> Self::BitMask; - - /// Converts a bitmask to a mask. - fn from_bitmask(bitmask: Self::BitMask) -> Self; -} - -/// Converts masks to and from byte array bitmasks. -/// -/// Each bit of the bitmask corresponds to a mask lane, starting with the LSB of the first byte. -#[cfg(feature = "generic_const_exprs")] -pub trait ToBitMaskArray: Sealed { - /// The length of the bitmask array. - const BYTES: usize; - - /// Converts a mask to a bitmask. - fn to_bitmask_array(self) -> [u8; Self::BYTES]; - - /// Converts a bitmask to a mask. - fn from_bitmask_array(bitmask: [u8; Self::BYTES]) -> Self; -} - -macro_rules! impl_integer_intrinsic { - { $(impl ToBitMask for Mask<_, $lanes:literal>)* } => { - $( - impl ToBitMask for Mask { - type BitMask = $int; - - #[inline] - fn to_bitmask(self) -> $int { - self.0.to_bitmask_integer() - } - - #[inline] - fn from_bitmask(bitmask: $int) -> Self { - Self(mask_impl::Mask::from_bitmask_integer(bitmask)) - } - } - )* - } -} - -impl_integer_intrinsic! { - impl ToBitMask for Mask<_, 1> - impl ToBitMask for Mask<_, 2> - impl ToBitMask for Mask<_, 4> - impl ToBitMask for Mask<_, 8> - impl ToBitMask for Mask<_, 16> - impl ToBitMask for Mask<_, 32> - impl ToBitMask for Mask<_, 64> -} - -/// Returns the minimum number of bytes in a bitmask with `lanes` lanes. -#[cfg(feature = "generic_const_exprs")] -pub const fn bitmask_len(lanes: usize) -> usize { - (lanes + 7) / 8 -} - -#[cfg(feature = "generic_const_exprs")] -impl ToBitMaskArray for Mask -where - LaneCount: SupportedLaneCount, -{ - const BYTES: usize = bitmask_len(LANES); - - #[inline] - fn to_bitmask_array(self) -> [u8; Self::BYTES] { - self.0.to_bitmask_array() - } - - #[inline] - fn from_bitmask_array(bitmask: [u8; Self::BYTES]) -> Self { - Mask(mask_impl::Mask::from_bitmask_array(bitmask)) - } -} diff --git a/library/portable-simd/crates/core_simd/src/mod.rs b/library/portable-simd/crates/core_simd/src/mod.rs index 19426769858..fd016f1c6f7 100644 --- a/library/portable-simd/crates/core_simd/src/mod.rs +++ b/library/portable-simd/crates/core_simd/src/mod.rs @@ -3,37 +3,37 @@ pub(crate) mod intrinsics; -#[cfg(feature = "generic_const_exprs")] -mod to_bytes; - mod alias; mod cast; -mod elements; -mod eq; mod fmt; mod iter; mod lane_count; mod masks; mod ops; -mod ord; mod select; mod swizzle_dyn; +mod to_bytes; mod vector; mod vendor; -#[doc = include_str!("core_simd_docs.md")] pub mod simd { + #![doc = include_str!("core_simd_docs.md")] + pub mod prelude; + pub mod num; + + pub mod ptr; + + pub mod cmp; + pub(crate) use crate::core_simd::intrinsics; pub use crate::core_simd::alias::*; pub use crate::core_simd::cast::*; - pub use crate::core_simd::elements::*; - pub use crate::core_simd::eq::*; pub use crate::core_simd::lane_count::{LaneCount, SupportedLaneCount}; pub use crate::core_simd::masks::*; - pub use crate::core_simd::ord::*; pub use crate::core_simd::swizzle::*; + pub use crate::core_simd::to_bytes::ToBytes; pub use crate::core_simd::vector::*; } diff --git a/library/portable-simd/crates/core_simd/src/ops.rs b/library/portable-simd/crates/core_simd/src/ops.rs index b007456cf2c..8a1b083f039 100644 --- a/library/portable-simd/crates/core_simd/src/ops.rs +++ b/library/portable-simd/crates/core_simd/src/ops.rs @@ -1,4 +1,4 @@ -use crate::simd::{LaneCount, Simd, SimdElement, SimdPartialEq, SupportedLaneCount}; +use crate::simd::{cmp::SimdPartialEq, LaneCount, Simd, SimdElement, SupportedLaneCount}; use core::ops::{Add, Mul}; use core::ops::{BitAnd, BitOr, BitXor}; use core::ops::{Div, Rem, Sub}; @@ -6,12 +6,13 @@ mod assign; mod deref; +mod shift_scalar; mod unary; -impl core::ops::Index for Simd +impl core::ops::Index for Simd where T: SimdElement, - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, I: core::slice::SliceIndex<[T]>, { type Output = I::Output; @@ -21,10 +22,10 @@ fn index(&self, index: I) -> &Self::Output { } } -impl core::ops::IndexMut for Simd +impl core::ops::IndexMut for Simd where T: SimdElement, - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, I: core::slice::SliceIndex<[T]>, { #[inline] diff --git a/library/portable-simd/crates/core_simd/src/ops/assign.rs b/library/portable-simd/crates/core_simd/src/ops/assign.rs index d2b48614fc9..0e87785025a 100644 --- a/library/portable-simd/crates/core_simd/src/ops/assign.rs +++ b/library/portable-simd/crates/core_simd/src/ops/assign.rs @@ -8,7 +8,7 @@ // Arithmetic macro_rules! assign_ops { - ($(impl $assignTrait:ident for Simd + ($(impl $assignTrait:ident for Simd where Self: $trait:ident, { @@ -16,11 +16,11 @@ fn $assign_call:ident(rhs: U) { $call:ident } })*) => { - $(impl $assignTrait for Simd + $(impl $assignTrait for Simd where Self: $trait, T: SimdElement, - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { #[inline] fn $assign_call(&mut self, rhs: U) { @@ -32,7 +32,7 @@ fn $assign_call(&mut self, rhs: U) { assign_ops! { // Arithmetic - impl AddAssign for Simd + impl AddAssign for Simd where Self: Add, { @@ -41,7 +41,7 @@ fn add_assign(rhs: U) { } } - impl MulAssign for Simd + impl MulAssign for Simd where Self: Mul, { @@ -50,7 +50,7 @@ fn mul_assign(rhs: U) { } } - impl SubAssign for Simd + impl SubAssign for Simd where Self: Sub, { @@ -59,7 +59,7 @@ fn sub_assign(rhs: U) { } } - impl DivAssign for Simd + impl DivAssign for Simd where Self: Div, { @@ -67,7 +67,7 @@ fn div_assign(rhs: U) { div } } - impl RemAssign for Simd + impl RemAssign for Simd where Self: Rem, { @@ -77,7 +77,7 @@ fn rem_assign(rhs: U) { } // Bitops - impl BitAndAssign for Simd + impl BitAndAssign for Simd where Self: BitAnd, { @@ -86,7 +86,7 @@ fn bitand_assign(rhs: U) { } } - impl BitOrAssign for Simd + impl BitOrAssign for Simd where Self: BitOr, { @@ -95,7 +95,7 @@ fn bitor_assign(rhs: U) { } } - impl BitXorAssign for Simd + impl BitXorAssign for Simd where Self: BitXor, { @@ -104,7 +104,7 @@ fn bitxor_assign(rhs: U) { } } - impl ShlAssign for Simd + impl ShlAssign for Simd where Self: Shl, { @@ -113,7 +113,7 @@ fn shl_assign(rhs: U) { } } - impl ShrAssign for Simd + impl ShrAssign for Simd where Self: Shr, { diff --git a/library/portable-simd/crates/core_simd/src/ops/deref.rs b/library/portable-simd/crates/core_simd/src/ops/deref.rs index 302bf148bd3..89a60ba1141 100644 --- a/library/portable-simd/crates/core_simd/src/ops/deref.rs +++ b/library/portable-simd/crates/core_simd/src/ops/deref.rs @@ -5,16 +5,16 @@ use super::*; macro_rules! deref_lhs { - (impl $trait:ident for $simd:ty { + (impl $trait:ident for $simd:ty { fn $call:ident }) => { - impl $trait<$simd> for &$simd + impl $trait<$simd> for &$simd where T: SimdElement, $simd: $trait<$simd, Output = $simd>, - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { - type Output = Simd; + type Output = Simd; #[inline] #[must_use = "operator returns a new vector without mutating the inputs"] @@ -26,16 +26,16 @@ fn $call(self, rhs: $simd) -> Self::Output { } macro_rules! deref_rhs { - (impl $trait:ident for $simd:ty { + (impl $trait:ident for $simd:ty { fn $call:ident }) => { - impl $trait<&$simd> for $simd + impl $trait<&$simd> for $simd where T: SimdElement, $simd: $trait<$simd, Output = $simd>, - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { - type Output = Simd; + type Output = Simd; #[inline] #[must_use = "operator returns a new vector without mutating the inputs"] @@ -47,25 +47,25 @@ fn $call(self, rhs: &$simd) -> Self::Output { } macro_rules! deref_ops { - ($(impl $trait:ident for $simd:ty { + ($(impl $trait:ident for $simd:ty { fn $call:ident })*) => { $( deref_rhs! { - impl $trait for $simd { + impl $trait for $simd { fn $call } } deref_lhs! { - impl $trait for $simd { + impl $trait for $simd { fn $call } } - impl<'lhs, 'rhs, T, const LANES: usize> $trait<&'rhs $simd> for &'lhs $simd + impl<'lhs, 'rhs, T, const N: usize> $trait<&'rhs $simd> for &'lhs $simd where T: SimdElement, $simd: $trait<$simd, Output = $simd>, - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { type Output = $simd; @@ -81,44 +81,44 @@ fn $call(self, rhs: &'rhs $simd) -> Self::Output { deref_ops! { // Arithmetic - impl Add for Simd { + impl Add for Simd { fn add } - impl Mul for Simd { + impl Mul for Simd { fn mul } - impl Sub for Simd { + impl Sub for Simd { fn sub } - impl Div for Simd { + impl Div for Simd { fn div } - impl Rem for Simd { + impl Rem for Simd { fn rem } // Bitops - impl BitAnd for Simd { + impl BitAnd for Simd { fn bitand } - impl BitOr for Simd { + impl BitOr for Simd { fn bitor } - impl BitXor for Simd { + impl BitXor for Simd { fn bitxor } - impl Shl for Simd { + impl Shl for Simd { fn shl } - impl Shr for Simd { + impl Shr for Simd { fn shr } } diff --git a/library/portable-simd/crates/core_simd/src/ops/shift_scalar.rs b/library/portable-simd/crates/core_simd/src/ops/shift_scalar.rs new file mode 100644 index 00000000000..f5115a5a5e9 --- /dev/null +++ b/library/portable-simd/crates/core_simd/src/ops/shift_scalar.rs @@ -0,0 +1,62 @@ +// Shift operations uniquely typically only have a scalar on the right-hand side. +// Here, we implement shifts for scalar RHS arguments. + +use crate::simd::{LaneCount, Simd, SupportedLaneCount}; + +macro_rules! impl_splatted_shifts { + { impl $trait:ident :: $trait_fn:ident for $ty:ty } => { + impl core::ops::$trait<$ty> for Simd<$ty, N> + where + LaneCount: SupportedLaneCount, + { + type Output = Self; + #[inline] + fn $trait_fn(self, rhs: $ty) -> Self::Output { + self.$trait_fn(Simd::splat(rhs)) + } + } + + impl core::ops::$trait<&$ty> for Simd<$ty, N> + where + LaneCount: SupportedLaneCount, + { + type Output = Self; + #[inline] + fn $trait_fn(self, rhs: &$ty) -> Self::Output { + self.$trait_fn(Simd::splat(*rhs)) + } + } + + impl<'lhs, const N: usize> core::ops::$trait<$ty> for &'lhs Simd<$ty, N> + where + LaneCount: SupportedLaneCount, + { + type Output = Simd<$ty, N>; + #[inline] + fn $trait_fn(self, rhs: $ty) -> Self::Output { + self.$trait_fn(Simd::splat(rhs)) + } + } + + impl<'lhs, const N: usize> core::ops::$trait<&$ty> for &'lhs Simd<$ty, N> + where + LaneCount: SupportedLaneCount, + { + type Output = Simd<$ty, N>; + #[inline] + fn $trait_fn(self, rhs: &$ty) -> Self::Output { + self.$trait_fn(Simd::splat(*rhs)) + } + } + }; + { $($ty:ty),* } => { + $( + impl_splatted_shifts! { impl Shl::shl for $ty } + impl_splatted_shifts! { impl Shr::shr for $ty } + )* + } +} + +// In the past there were inference issues when generically splatting arguments. +// Enumerate them instead. +impl_splatted_shifts! { i8, i16, i32, i64, isize, u8, u16, u32, u64, usize } diff --git a/library/portable-simd/crates/core_simd/src/ops/unary.rs b/library/portable-simd/crates/core_simd/src/ops/unary.rs index 4ad02215034..a651aa73e95 100644 --- a/library/portable-simd/crates/core_simd/src/ops/unary.rs +++ b/library/portable-simd/crates/core_simd/src/ops/unary.rs @@ -3,11 +3,11 @@ use core::ops::{Neg, Not}; // unary ops macro_rules! neg { - ($(impl Neg for Simd<$scalar:ty, LANES>)*) => { - $(impl Neg for Simd<$scalar, LANES> + ($(impl Neg for Simd<$scalar:ty, N>)*) => { + $(impl Neg for Simd<$scalar, N> where $scalar: SimdElement, - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { type Output = Self; @@ -22,27 +22,27 @@ fn neg(self) -> Self::Output { } neg! { - impl Neg for Simd + impl Neg for Simd - impl Neg for Simd + impl Neg for Simd - impl Neg for Simd + impl Neg for Simd - impl Neg for Simd + impl Neg for Simd - impl Neg for Simd + impl Neg for Simd - impl Neg for Simd + impl Neg for Simd - impl Neg for Simd + impl Neg for Simd } macro_rules! not { - ($(impl Not for Simd<$scalar:ty, LANES>)*) => { - $(impl Not for Simd<$scalar, LANES> + ($(impl Not for Simd<$scalar:ty, N>)*) => { + $(impl Not for Simd<$scalar, N> where $scalar: SimdElement, - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { type Output = Self; @@ -56,23 +56,23 @@ fn not(self) -> Self::Output { } not! { - impl Not for Simd + impl Not for Simd - impl Not for Simd + impl Not for Simd - impl Not for Simd + impl Not for Simd - impl Not for Simd + impl Not for Simd - impl Not for Simd + impl Not for Simd - impl Not for Simd + impl Not for Simd - impl Not for Simd + impl Not for Simd - impl Not for Simd + impl Not for Simd - impl Not for Simd + impl Not for Simd - impl Not for Simd + impl Not for Simd } diff --git a/library/portable-simd/crates/core_simd/src/select.rs b/library/portable-simd/crates/core_simd/src/select.rs index 065c5987d3f..cdcf8eeec81 100644 --- a/library/portable-simd/crates/core_simd/src/select.rs +++ b/library/portable-simd/crates/core_simd/src/select.rs @@ -1,15 +1,15 @@ use crate::simd::intrinsics; use crate::simd::{LaneCount, Mask, MaskElement, Simd, SimdElement, SupportedLaneCount}; -impl Mask +impl Mask where T: MaskElement, - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { - /// Choose lanes from two vectors. + /// Choose elements from two vectors. /// - /// For each lane in the mask, choose the corresponding lane from `true_values` if - /// that lane mask is true, and `false_values` if that lane mask is false. + /// For each element in the mask, choose the corresponding element from `true_values` if + /// that element mask is true, and `false_values` if that element mask is false. /// /// # Examples /// ``` @@ -23,11 +23,7 @@ impl Mask /// ``` #[inline] #[must_use = "method returns a new vector and does not mutate the original inputs"] - pub fn select( - self, - true_values: Simd, - false_values: Simd, - ) -> Simd + pub fn select(self, true_values: Simd, false_values: Simd) -> Simd where U: SimdElement, { @@ -36,10 +32,10 @@ pub fn select( unsafe { intrinsics::simd_select(self.to_int(), true_values, false_values) } } - /// Choose lanes from two masks. + /// Choose elements from two masks. /// - /// For each lane in the mask, choose the corresponding lane from `true_values` if - /// that lane mask is true, and `false_values` if that lane mask is false. + /// For each element in the mask, choose the corresponding element from `true_values` if + /// that element mask is true, and `false_values` if that element mask is false. /// /// # Examples /// ``` diff --git a/library/portable-simd/crates/core_simd/src/simd/cmp.rs b/library/portable-simd/crates/core_simd/src/simd/cmp.rs new file mode 100644 index 00000000000..a8d81dbf20f --- /dev/null +++ b/library/portable-simd/crates/core_simd/src/simd/cmp.rs @@ -0,0 +1,7 @@ +//! Traits for comparing and ordering vectors. + +mod eq; +mod ord; + +pub use eq::*; +pub use ord::*; diff --git a/library/portable-simd/crates/core_simd/src/eq.rs b/library/portable-simd/crates/core_simd/src/simd/cmp/eq.rs similarity index 74% rename from library/portable-simd/crates/core_simd/src/eq.rs rename to library/portable-simd/crates/core_simd/src/simd/cmp/eq.rs index 80763c07272..f132fa2cc0c 100644 --- a/library/portable-simd/crates/core_simd/src/eq.rs +++ b/library/portable-simd/crates/core_simd/src/simd/cmp/eq.rs @@ -1,5 +1,7 @@ use crate::simd::{ - intrinsics, LaneCount, Mask, Simd, SimdConstPtr, SimdElement, SimdMutPtr, SupportedLaneCount, + intrinsics, + ptr::{SimdConstPtr, SimdMutPtr}, + LaneCount, Mask, Simd, SimdElement, SupportedLaneCount, }; /// Parallel `PartialEq`. @@ -7,11 +9,11 @@ pub trait SimdPartialEq { /// The mask type returned by each comparison. type Mask; - /// Test if each lane is equal to the corresponding lane in `other`. + /// Test if each element is equal to the corresponding element in `other`. #[must_use = "method returns a new mask and does not mutate the original value"] fn simd_eq(self, other: Self) -> Self::Mask; - /// Test if each lane is equal to the corresponding lane in `other`. + /// Test if each element is equal to the corresponding element in `other`. #[must_use = "method returns a new mask and does not mutate the original value"] fn simd_ne(self, other: Self) -> Self::Mask; } @@ -19,11 +21,11 @@ pub trait SimdPartialEq { macro_rules! impl_number { { $($number:ty),* } => { $( - impl SimdPartialEq for Simd<$number, LANES> + impl SimdPartialEq for Simd<$number, N> where - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { - type Mask = Mask<<$number as SimdElement>::Mask, LANES>; + type Mask = Mask<<$number as SimdElement>::Mask, N>; #[inline] fn simd_eq(self, other: Self) -> Self::Mask { @@ -48,9 +50,9 @@ fn simd_ne(self, other: Self) -> Self::Mask { macro_rules! impl_mask { { $($integer:ty),* } => { $( - impl SimdPartialEq for Mask<$integer, LANES> + impl SimdPartialEq for Mask<$integer, N> where - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { type Mask = Self; @@ -74,11 +76,11 @@ fn simd_ne(self, other: Self) -> Self::Mask { impl_mask! { i8, i16, i32, i64, isize } -impl SimdPartialEq for Simd<*const T, LANES> +impl SimdPartialEq for Simd<*const T, N> where - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { - type Mask = Mask; + type Mask = Mask; #[inline] fn simd_eq(self, other: Self) -> Self::Mask { @@ -91,11 +93,11 @@ fn simd_ne(self, other: Self) -> Self::Mask { } } -impl SimdPartialEq for Simd<*mut T, LANES> +impl SimdPartialEq for Simd<*mut T, N> where - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { - type Mask = Mask; + type Mask = Mask; #[inline] fn simd_eq(self, other: Self) -> Self::Mask { diff --git a/library/portable-simd/crates/core_simd/src/ord.rs b/library/portable-simd/crates/core_simd/src/simd/cmp/ord.rs similarity index 79% rename from library/portable-simd/crates/core_simd/src/ord.rs rename to library/portable-simd/crates/core_simd/src/simd/cmp/ord.rs index b2455190e82..4e9d49ea221 100644 --- a/library/portable-simd/crates/core_simd/src/ord.rs +++ b/library/portable-simd/crates/core_simd/src/simd/cmp/ord.rs @@ -1,44 +1,47 @@ use crate::simd::{ - intrinsics, LaneCount, Mask, Simd, SimdConstPtr, SimdMutPtr, SimdPartialEq, SupportedLaneCount, + cmp::SimdPartialEq, + intrinsics, + ptr::{SimdConstPtr, SimdMutPtr}, + LaneCount, Mask, Simd, SupportedLaneCount, }; /// Parallel `PartialOrd`. pub trait SimdPartialOrd: SimdPartialEq { - /// Test if each lane is less than the corresponding lane in `other`. + /// Test if each element is less than the corresponding element in `other`. #[must_use = "method returns a new mask and does not mutate the original value"] fn simd_lt(self, other: Self) -> Self::Mask; - /// Test if each lane is less than or equal to the corresponding lane in `other`. + /// Test if each element is less than or equal to the corresponding element in `other`. #[must_use = "method returns a new mask and does not mutate the original value"] fn simd_le(self, other: Self) -> Self::Mask; - /// Test if each lane is greater than the corresponding lane in `other`. + /// Test if each element is greater than the corresponding element in `other`. #[must_use = "method returns a new mask and does not mutate the original value"] fn simd_gt(self, other: Self) -> Self::Mask; - /// Test if each lane is greater than or equal to the corresponding lane in `other`. + /// Test if each element is greater than or equal to the corresponding element in `other`. #[must_use = "method returns a new mask and does not mutate the original value"] fn simd_ge(self, other: Self) -> Self::Mask; } /// Parallel `Ord`. pub trait SimdOrd: SimdPartialOrd { - /// Returns the lane-wise maximum with `other`. + /// Returns the element-wise maximum with `other`. #[must_use = "method returns a new vector and does not mutate the original value"] fn simd_max(self, other: Self) -> Self; - /// Returns the lane-wise minimum with `other`. + /// Returns the element-wise minimum with `other`. #[must_use = "method returns a new vector and does not mutate the original value"] fn simd_min(self, other: Self) -> Self; - /// Restrict each lane to a certain interval. + /// Restrict each element to a certain interval. /// - /// For each lane, returns `max` if `self` is greater than `max`, and `min` if `self` is + /// For each element, returns `max` if `self` is greater than `max`, and `min` if `self` is /// less than `min`. Otherwise returns `self`. /// /// # Panics /// - /// Panics if `min > max` on any lane. + /// Panics if `min > max` on any element. #[must_use = "method returns a new vector and does not mutate the original value"] fn simd_clamp(self, min: Self, max: Self) -> Self; } @@ -46,9 +49,9 @@ pub trait SimdOrd: SimdPartialOrd { macro_rules! impl_integer { { $($integer:ty),* } => { $( - impl SimdPartialOrd for Simd<$integer, LANES> + impl SimdPartialOrd for Simd<$integer, N> where - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { #[inline] fn simd_lt(self, other: Self) -> Self::Mask { @@ -79,9 +82,9 @@ fn simd_ge(self, other: Self) -> Self::Mask { } } - impl SimdOrd for Simd<$integer, LANES> + impl SimdOrd for Simd<$integer, N> where - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { #[inline] fn simd_max(self, other: Self) -> Self { @@ -98,7 +101,7 @@ fn simd_min(self, other: Self) -> Self { fn simd_clamp(self, min: Self, max: Self) -> Self { assert!( min.simd_le(max).all(), - "each lane in `min` must be less than or equal to the corresponding lane in `max`", + "each element in `min` must be less than or equal to the corresponding element in `max`", ); self.simd_max(min).simd_min(max) } @@ -112,9 +115,9 @@ fn simd_clamp(self, min: Self, max: Self) -> Self { macro_rules! impl_float { { $($float:ty),* } => { $( - impl SimdPartialOrd for Simd<$float, LANES> + impl SimdPartialOrd for Simd<$float, N> where - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { #[inline] fn simd_lt(self, other: Self) -> Self::Mask { @@ -153,9 +156,9 @@ fn simd_ge(self, other: Self) -> Self::Mask { macro_rules! impl_mask { { $($integer:ty),* } => { $( - impl SimdPartialOrd for Mask<$integer, LANES> + impl SimdPartialOrd for Mask<$integer, N> where - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { #[inline] fn simd_lt(self, other: Self) -> Self::Mask { @@ -186,9 +189,9 @@ fn simd_ge(self, other: Self) -> Self::Mask { } } - impl SimdOrd for Mask<$integer, LANES> + impl SimdOrd for Mask<$integer, N> where - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { #[inline] fn simd_max(self, other: Self) -> Self { @@ -205,7 +208,7 @@ fn simd_min(self, other: Self) -> Self { fn simd_clamp(self, min: Self, max: Self) -> Self { assert!( min.simd_le(max).all(), - "each lane in `min` must be less than or equal to the corresponding lane in `max`", + "each element in `min` must be less than or equal to the corresponding element in `max`", ); self.simd_max(min).simd_min(max) } @@ -216,9 +219,9 @@ fn simd_clamp(self, min: Self, max: Self) -> Self { impl_mask! { i8, i16, i32, i64, isize } -impl SimdPartialOrd for Simd<*const T, LANES> +impl SimdPartialOrd for Simd<*const T, N> where - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { #[inline] fn simd_lt(self, other: Self) -> Self::Mask { @@ -241,9 +244,9 @@ fn simd_ge(self, other: Self) -> Self::Mask { } } -impl SimdOrd for Simd<*const T, LANES> +impl SimdOrd for Simd<*const T, N> where - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { #[inline] fn simd_max(self, other: Self) -> Self { @@ -260,15 +263,15 @@ fn simd_min(self, other: Self) -> Self { fn simd_clamp(self, min: Self, max: Self) -> Self { assert!( min.simd_le(max).all(), - "each lane in `min` must be less than or equal to the corresponding lane in `max`", + "each element in `min` must be less than or equal to the corresponding element in `max`", ); self.simd_max(min).simd_min(max) } } -impl SimdPartialOrd for Simd<*mut T, LANES> +impl SimdPartialOrd for Simd<*mut T, N> where - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { #[inline] fn simd_lt(self, other: Self) -> Self::Mask { @@ -291,9 +294,9 @@ fn simd_ge(self, other: Self) -> Self::Mask { } } -impl SimdOrd for Simd<*mut T, LANES> +impl SimdOrd for Simd<*mut T, N> where - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { #[inline] fn simd_max(self, other: Self) -> Self { @@ -310,7 +313,7 @@ fn simd_min(self, other: Self) -> Self { fn simd_clamp(self, min: Self, max: Self) -> Self { assert!( min.simd_le(max).all(), - "each lane in `min` must be less than or equal to the corresponding lane in `max`", + "each element in `min` must be less than or equal to the corresponding element in `max`", ); self.simd_max(min).simd_min(max) } diff --git a/library/portable-simd/crates/core_simd/src/elements.rs b/library/portable-simd/crates/core_simd/src/simd/num.rs similarity index 63% rename from library/portable-simd/crates/core_simd/src/elements.rs rename to library/portable-simd/crates/core_simd/src/simd/num.rs index dc7f52a4d57..22a4802ec6c 100644 --- a/library/portable-simd/crates/core_simd/src/elements.rs +++ b/library/portable-simd/crates/core_simd/src/simd/num.rs @@ -1,15 +1,13 @@ -mod const_ptr; +//! Traits for vectors with numeric elements. + mod float; mod int; -mod mut_ptr; mod uint; mod sealed { pub trait Sealed {} } -pub use const_ptr::*; pub use float::*; pub use int::*; -pub use mut_ptr::*; pub use uint::*; diff --git a/library/portable-simd/crates/core_simd/src/elements/float.rs b/library/portable-simd/crates/core_simd/src/simd/num/float.rs similarity index 80% rename from library/portable-simd/crates/core_simd/src/elements/float.rs rename to library/portable-simd/crates/core_simd/src/simd/num/float.rs index 501c1c5ddd3..fc0b99e87a6 100644 --- a/library/portable-simd/crates/core_simd/src/elements/float.rs +++ b/library/portable-simd/crates/core_simd/src/simd/num/float.rs @@ -1,7 +1,7 @@ use super::sealed::Sealed; use crate::simd::{ - intrinsics, LaneCount, Mask, Simd, SimdCast, SimdElement, SimdPartialEq, SimdPartialOrd, - SupportedLaneCount, + cmp::{SimdPartialEq, SimdPartialOrd}, + intrinsics, LaneCount, Mask, Simd, SimdCast, SimdElement, SupportedLaneCount, }; /// Operations on SIMD vectors of floats. @@ -28,7 +28,7 @@ pub trait SimdFloat: Copy + Sealed { /// # #![feature(portable_simd)] /// # #[cfg(feature = "as_crate")] use core_simd::simd; /// # #[cfg(not(feature = "as_crate"))] use core::simd; - /// # use simd::{SimdFloat, SimdInt, Simd}; + /// # use simd::prelude::*; /// let floats: Simd = Simd::from_array([1.9, -4.5, f32::INFINITY, f32::NAN]); /// let ints = floats.cast::(); /// assert_eq!(ints, Simd::from_array([1, -4, i32::MAX, 0])); @@ -63,64 +63,64 @@ unsafe fn to_int_unchecked(self) -> Self::Cast Self::Scalar: core::convert::FloatToInt; /// Raw transmutation to an unsigned integer vector type with the - /// same size and number of lanes. + /// same size and number of elements. #[must_use = "method returns a new vector and does not mutate the original value"] fn to_bits(self) -> Self::Bits; /// Raw transmutation from an unsigned integer vector type with the - /// same size and number of lanes. + /// same size and number of elements. #[must_use = "method returns a new vector and does not mutate the original value"] fn from_bits(bits: Self::Bits) -> Self; - /// Produces a vector where every lane has the absolute value of the - /// equivalently-indexed lane in `self`. + /// Produces a vector where every element has the absolute value of the + /// equivalently-indexed element in `self`. #[must_use = "method returns a new vector and does not mutate the original value"] fn abs(self) -> Self; - /// Takes the reciprocal (inverse) of each lane, `1/x`. + /// Takes the reciprocal (inverse) of each element, `1/x`. #[must_use = "method returns a new vector and does not mutate the original value"] fn recip(self) -> Self; - /// Converts each lane from radians to degrees. + /// Converts each element from radians to degrees. #[must_use = "method returns a new vector and does not mutate the original value"] fn to_degrees(self) -> Self; - /// Converts each lane from degrees to radians. + /// Converts each element from degrees to radians. #[must_use = "method returns a new vector and does not mutate the original value"] fn to_radians(self) -> Self; - /// Returns true for each lane if it has a positive sign, including + /// Returns true for each element if it has a positive sign, including /// `+0.0`, `NaN`s with positive sign bit and positive infinity. #[must_use = "method returns a new mask and does not mutate the original value"] fn is_sign_positive(self) -> Self::Mask; - /// Returns true for each lane if it has a negative sign, including + /// Returns true for each element if it has a negative sign, including /// `-0.0`, `NaN`s with negative sign bit and negative infinity. #[must_use = "method returns a new mask and does not mutate the original value"] fn is_sign_negative(self) -> Self::Mask; - /// Returns true for each lane if its value is `NaN`. + /// Returns true for each element if its value is `NaN`. #[must_use = "method returns a new mask and does not mutate the original value"] fn is_nan(self) -> Self::Mask; - /// Returns true for each lane if its value is positive infinity or negative infinity. + /// Returns true for each element if its value is positive infinity or negative infinity. #[must_use = "method returns a new mask and does not mutate the original value"] fn is_infinite(self) -> Self::Mask; - /// Returns true for each lane if its value is neither infinite nor `NaN`. + /// Returns true for each element if its value is neither infinite nor `NaN`. #[must_use = "method returns a new mask and does not mutate the original value"] fn is_finite(self) -> Self::Mask; - /// Returns true for each lane if its value is subnormal. + /// Returns true for each element if its value is subnormal. #[must_use = "method returns a new mask and does not mutate the original value"] fn is_subnormal(self) -> Self::Mask; - /// Returns true for each lane if its value is neither zero, infinite, + /// Returns true for each element if its value is neither zero, infinite, /// subnormal, nor `NaN`. #[must_use = "method returns a new mask and does not mutate the original value"] fn is_normal(self) -> Self::Mask; - /// Replaces each lane with a number that represents its sign. + /// Replaces each element with a number that represents its sign. /// /// * `1.0` if the number is positive, `+0.0`, or `INFINITY` /// * `-1.0` if the number is negative, `-0.0`, or `NEG_INFINITY` @@ -128,33 +128,33 @@ unsafe fn to_int_unchecked(self) -> Self::Cast #[must_use = "method returns a new vector and does not mutate the original value"] fn signum(self) -> Self; - /// Returns each lane with the magnitude of `self` and the sign of `sign`. + /// Returns each element with the magnitude of `self` and the sign of `sign`. /// - /// For any lane containing a `NAN`, a `NAN` with the sign of `sign` is returned. + /// For any element containing a `NAN`, a `NAN` with the sign of `sign` is returned. #[must_use = "method returns a new vector and does not mutate the original value"] fn copysign(self, sign: Self) -> Self; - /// Returns the minimum of each lane. + /// Returns the minimum of each element. /// /// If one of the values is `NAN`, then the other value is returned. #[must_use = "method returns a new vector and does not mutate the original value"] fn simd_min(self, other: Self) -> Self; - /// Returns the maximum of each lane. + /// Returns the maximum of each element. /// /// If one of the values is `NAN`, then the other value is returned. #[must_use = "method returns a new vector and does not mutate the original value"] fn simd_max(self, other: Self) -> Self; - /// Restrict each lane to a certain interval unless it is NaN. + /// Restrict each element to a certain interval unless it is NaN. /// - /// For each lane in `self`, returns the corresponding lane in `max` if the lane is - /// greater than `max`, and the corresponding lane in `min` if the lane is less - /// than `min`. Otherwise returns the lane in `self`. + /// For each element in `self`, returns the corresponding element in `max` if the element is + /// greater than `max`, and the corresponding element in `min` if the element is less + /// than `min`. Otherwise returns the element in `self`. #[must_use = "method returns a new vector and does not mutate the original value"] fn simd_clamp(self, min: Self, max: Self) -> Self; - /// Returns the sum of the lanes of the vector. + /// Returns the sum of the elements of the vector. /// /// # Examples /// @@ -162,13 +162,13 @@ unsafe fn to_int_unchecked(self) -> Self::Cast /// # #![feature(portable_simd)] /// # #[cfg(feature = "as_crate")] use core_simd::simd; /// # #[cfg(not(feature = "as_crate"))] use core::simd; - /// # use simd::{f32x2, SimdFloat}; + /// # use simd::prelude::*; /// let v = f32x2::from_array([1., 2.]); /// assert_eq!(v.reduce_sum(), 3.); /// ``` fn reduce_sum(self) -> Self::Scalar; - /// Reducing multiply. Returns the product of the lanes of the vector. + /// Reducing multiply. Returns the product of the elements of the vector. /// /// # Examples /// @@ -176,18 +176,18 @@ unsafe fn to_int_unchecked(self) -> Self::Cast /// # #![feature(portable_simd)] /// # #[cfg(feature = "as_crate")] use core_simd::simd; /// # #[cfg(not(feature = "as_crate"))] use core::simd; - /// # use simd::{f32x2, SimdFloat}; + /// # use simd::prelude::*; /// let v = f32x2::from_array([3., 4.]); /// assert_eq!(v.reduce_product(), 12.); /// ``` fn reduce_product(self) -> Self::Scalar; - /// Returns the maximum lane in the vector. + /// Returns the maximum element in the vector. /// /// Returns values based on equality, so a vector containing both `0.` and `-0.` may /// return either. /// - /// This function will not return `NaN` unless all lanes are `NaN`. + /// This function will not return `NaN` unless all elements are `NaN`. /// /// # Examples /// @@ -195,7 +195,7 @@ unsafe fn to_int_unchecked(self) -> Self::Cast /// # #![feature(portable_simd)] /// # #[cfg(feature = "as_crate")] use core_simd::simd; /// # #[cfg(not(feature = "as_crate"))] use core::simd; - /// # use simd::{f32x2, SimdFloat}; + /// # use simd::prelude::*; /// let v = f32x2::from_array([1., 2.]); /// assert_eq!(v.reduce_max(), 2.); /// @@ -209,12 +209,12 @@ unsafe fn to_int_unchecked(self) -> Self::Cast /// ``` fn reduce_max(self) -> Self::Scalar; - /// Returns the minimum lane in the vector. + /// Returns the minimum element in the vector. /// /// Returns values based on equality, so a vector containing both `0.` and `-0.` may /// return either. /// - /// This function will not return `NaN` unless all lanes are `NaN`. + /// This function will not return `NaN` unless all elements are `NaN`. /// /// # Examples /// @@ -222,7 +222,7 @@ unsafe fn to_int_unchecked(self) -> Self::Cast /// # #![feature(portable_simd)] /// # #[cfg(feature = "as_crate")] use core_simd::simd; /// # #[cfg(not(feature = "as_crate"))] use core::simd; - /// # use simd::{f32x2, SimdFloat}; + /// # use simd::prelude::*; /// let v = f32x2::from_array([3., 7.]); /// assert_eq!(v.reduce_min(), 3.); /// @@ -240,20 +240,20 @@ unsafe fn to_int_unchecked(self) -> Self::Cast macro_rules! impl_trait { { $($ty:ty { bits: $bits_ty:ty, mask: $mask_ty:ty }),* } => { $( - impl Sealed for Simd<$ty, LANES> + impl Sealed for Simd<$ty, N> where - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { } - impl SimdFloat for Simd<$ty, LANES> + impl SimdFloat for Simd<$ty, N> where - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { - type Mask = Mask<<$mask_ty as SimdElement>::Mask, LANES>; + type Mask = Mask<<$mask_ty as SimdElement>::Mask, N>; type Scalar = $ty; - type Bits = Simd<$bits_ty, LANES>; - type Cast = Simd; + type Bits = Simd<$bits_ty, N>; + type Cast = Simd; #[inline] fn cast(self) -> Self::Cast @@ -273,14 +273,14 @@ unsafe fn to_int_unchecked(self) -> Self::Cast } #[inline] - fn to_bits(self) -> Simd<$bits_ty, LANES> { + fn to_bits(self) -> Simd<$bits_ty, N> { assert_eq!(core::mem::size_of::(), core::mem::size_of::()); // Safety: transmuting between vector types is safe unsafe { core::mem::transmute_copy(&self) } } #[inline] - fn from_bits(bits: Simd<$bits_ty, LANES>) -> Self { + fn from_bits(bits: Simd<$bits_ty, N>) -> Self { assert_eq!(core::mem::size_of::(), core::mem::size_of::()); // Safety: transmuting between vector types is safe unsafe { core::mem::transmute_copy(&bits) } @@ -336,7 +336,10 @@ fn is_finite(self) -> Self::Mask { #[inline] fn is_subnormal(self) -> Self::Mask { - self.abs().simd_ne(Self::splat(0.0)) & (self.to_bits() & Self::splat(Self::Scalar::INFINITY).to_bits()).simd_eq(Simd::splat(0)) + // On some architectures (e.g. armv7 and some ppc) subnormals are flushed to zero, + // so this comparison must be done with integers. + let not_zero = self.abs().to_bits().simd_ne(Self::splat(0.0).to_bits()); + not_zero & (self.to_bits() & Self::splat(Self::Scalar::INFINITY).to_bits()).simd_eq(Simd::splat(0)) } #[inline] @@ -373,7 +376,7 @@ fn simd_max(self, other: Self) -> Self { fn simd_clamp(self, min: Self, max: Self) -> Self { assert!( min.simd_le(max).all(), - "each lane in `min` must be less than or equal to the corresponding lane in `max`", + "each element in `min` must be less than or equal to the corresponding element in `max`", ); let mut x = self; x = x.simd_lt(min).select(min, x); diff --git a/library/portable-simd/crates/core_simd/src/elements/int.rs b/library/portable-simd/crates/core_simd/src/simd/num/int.rs similarity index 71% rename from library/portable-simd/crates/core_simd/src/elements/int.rs rename to library/portable-simd/crates/core_simd/src/simd/num/int.rs index 6db89ff9a65..1f1aa272782 100644 --- a/library/portable-simd/crates/core_simd/src/elements/int.rs +++ b/library/portable-simd/crates/core_simd/src/simd/num/int.rs @@ -1,6 +1,7 @@ use super::sealed::Sealed; use crate::simd::{ - intrinsics, LaneCount, Mask, Simd, SimdCast, SimdElement, SimdPartialOrd, SupportedLaneCount, + cmp::SimdPartialOrd, intrinsics, num::SimdUint, LaneCount, Mask, Simd, SimdCast, SimdElement, + SupportedLaneCount, }; /// Operations on SIMD vectors of signed integers. @@ -11,6 +12,9 @@ pub trait SimdInt: Copy + Sealed { /// Scalar type contained by this SIMD vector type. type Scalar; + /// A SIMD vector of unsigned integers with the same element size. + type Unsigned; + /// A SIMD vector with a different element type. type Cast; @@ -28,7 +32,7 @@ pub trait SimdInt: Copy + Sealed { /// # #![feature(portable_simd)] /// # #[cfg(feature = "as_crate")] use core_simd::simd; /// # #[cfg(not(feature = "as_crate"))] use core::simd; - /// # use simd::{Simd, SimdInt}; + /// # use simd::prelude::*; /// use core::i32::{MIN, MAX}; /// let x = Simd::from_array([MIN, 0, 1, MAX]); /// let max = Simd::splat(MAX); @@ -46,7 +50,7 @@ pub trait SimdInt: Copy + Sealed { /// # #![feature(portable_simd)] /// # #[cfg(feature = "as_crate")] use core_simd::simd; /// # #[cfg(not(feature = "as_crate"))] use core::simd; - /// # use simd::{Simd, SimdInt}; + /// # use simd::prelude::*; /// use core::i32::{MIN, MAX}; /// let x = Simd::from_array([MIN, -2, -1, MAX]); /// let max = Simd::splat(MAX); @@ -57,14 +61,14 @@ pub trait SimdInt: Copy + Sealed { fn saturating_sub(self, second: Self) -> Self; /// Lanewise absolute value, implemented in Rust. - /// Every lane becomes its absolute value. + /// Every element becomes its absolute value. /// /// # Examples /// ``` /// # #![feature(portable_simd)] /// # #[cfg(feature = "as_crate")] use core_simd::simd; /// # #[cfg(not(feature = "as_crate"))] use core::simd; - /// # use simd::{Simd, SimdInt}; + /// # use simd::prelude::*; /// use core::i32::{MIN, MAX}; /// let xs = Simd::from_array([MIN, MIN +1, -5, 0]); /// assert_eq!(xs.abs(), Simd::from_array([MIN, MAX, 5, 0])); @@ -79,7 +83,7 @@ pub trait SimdInt: Copy + Sealed { /// # #![feature(portable_simd)] /// # #[cfg(feature = "as_crate")] use core_simd::simd; /// # #[cfg(not(feature = "as_crate"))] use core::simd; - /// # use simd::{Simd, SimdInt}; + /// # use simd::prelude::*; /// use core::i32::{MIN, MAX}; /// let xs = Simd::from_array([MIN, -2, 0, 3]); /// let unsat = xs.abs(); @@ -97,7 +101,7 @@ pub trait SimdInt: Copy + Sealed { /// # #![feature(portable_simd)] /// # #[cfg(feature = "as_crate")] use core_simd::simd; /// # #[cfg(not(feature = "as_crate"))] use core::simd; - /// # use simd::{Simd, SimdInt}; + /// # use simd::prelude::*; /// use core::i32::{MIN, MAX}; /// let x = Simd::from_array([MIN, -2, 3, MAX]); /// let unsat = -x; @@ -107,19 +111,19 @@ pub trait SimdInt: Copy + Sealed { /// ``` fn saturating_neg(self) -> Self; - /// Returns true for each positive lane and false if it is zero or negative. + /// Returns true for each positive element and false if it is zero or negative. fn is_positive(self) -> Self::Mask; - /// Returns true for each negative lane and false if it is zero or positive. + /// Returns true for each negative element and false if it is zero or positive. fn is_negative(self) -> Self::Mask; - /// Returns numbers representing the sign of each lane. + /// Returns numbers representing the sign of each element. /// * `0` if the number is zero /// * `1` if the number is positive /// * `-1` if the number is negative fn signum(self) -> Self; - /// Returns the sum of the lanes of the vector, with wrapping addition. + /// Returns the sum of the elements of the vector, with wrapping addition. /// /// # Examples /// @@ -127,7 +131,7 @@ pub trait SimdInt: Copy + Sealed { /// # #![feature(portable_simd)] /// # #[cfg(feature = "as_crate")] use core_simd::simd; /// # #[cfg(not(feature = "as_crate"))] use core::simd; - /// # use simd::{i32x4, SimdInt}; + /// # use simd::prelude::*; /// let v = i32x4::from_array([1, 2, 3, 4]); /// assert_eq!(v.reduce_sum(), 10); /// @@ -137,7 +141,7 @@ pub trait SimdInt: Copy + Sealed { /// ``` fn reduce_sum(self) -> Self::Scalar; - /// Returns the product of the lanes of the vector, with wrapping multiplication. + /// Returns the product of the elements of the vector, with wrapping multiplication. /// /// # Examples /// @@ -145,7 +149,7 @@ pub trait SimdInt: Copy + Sealed { /// # #![feature(portable_simd)] /// # #[cfg(feature = "as_crate")] use core_simd::simd; /// # #[cfg(not(feature = "as_crate"))] use core::simd; - /// # use simd::{i32x4, SimdInt}; + /// # use simd::prelude::*; /// let v = i32x4::from_array([1, 2, 3, 4]); /// assert_eq!(v.reduce_product(), 24); /// @@ -155,7 +159,7 @@ pub trait SimdInt: Copy + Sealed { /// ``` fn reduce_product(self) -> Self::Scalar; - /// Returns the maximum lane in the vector. + /// Returns the maximum element in the vector. /// /// # Examples /// @@ -163,13 +167,13 @@ pub trait SimdInt: Copy + Sealed { /// # #![feature(portable_simd)] /// # #[cfg(feature = "as_crate")] use core_simd::simd; /// # #[cfg(not(feature = "as_crate"))] use core::simd; - /// # use simd::{i32x4, SimdInt}; + /// # use simd::prelude::*; /// let v = i32x4::from_array([1, 2, 3, 4]); /// assert_eq!(v.reduce_max(), 4); /// ``` fn reduce_max(self) -> Self::Scalar; - /// Returns the minimum lane in the vector. + /// Returns the minimum element in the vector. /// /// # Examples /// @@ -177,38 +181,58 @@ pub trait SimdInt: Copy + Sealed { /// # #![feature(portable_simd)] /// # #[cfg(feature = "as_crate")] use core_simd::simd; /// # #[cfg(not(feature = "as_crate"))] use core::simd; - /// # use simd::{i32x4, SimdInt}; + /// # use simd::prelude::*; /// let v = i32x4::from_array([1, 2, 3, 4]); /// assert_eq!(v.reduce_min(), 1); /// ``` fn reduce_min(self) -> Self::Scalar; - /// Returns the cumulative bitwise "and" across the lanes of the vector. + /// Returns the cumulative bitwise "and" across the elements of the vector. fn reduce_and(self) -> Self::Scalar; - /// Returns the cumulative bitwise "or" across the lanes of the vector. + /// Returns the cumulative bitwise "or" across the elements of the vector. fn reduce_or(self) -> Self::Scalar; - /// Returns the cumulative bitwise "xor" across the lanes of the vector. + /// Returns the cumulative bitwise "xor" across the elements of the vector. fn reduce_xor(self) -> Self::Scalar; + + /// Reverses the byte order of each element. + fn swap_bytes(self) -> Self; + + /// Reverses the order of bits in each elemnent. + /// The least significant bit becomes the most significant bit, second least-significant bit becomes second most-significant bit, etc. + fn reverse_bits(self) -> Self; + + /// Returns the number of leading zeros in the binary representation of each element. + fn leading_zeros(self) -> Self::Unsigned; + + /// Returns the number of trailing zeros in the binary representation of each element. + fn trailing_zeros(self) -> Self::Unsigned; + + /// Returns the number of leading ones in the binary representation of each element. + fn leading_ones(self) -> Self::Unsigned; + + /// Returns the number of trailing ones in the binary representation of each element. + fn trailing_ones(self) -> Self::Unsigned; } macro_rules! impl_trait { - { $($ty:ty),* } => { + { $($ty:ident ($unsigned:ident)),* } => { $( - impl Sealed for Simd<$ty, LANES> + impl Sealed for Simd<$ty, N> where - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { } - impl SimdInt for Simd<$ty, LANES> + impl SimdInt for Simd<$ty, N> where - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { - type Mask = Mask<<$ty as SimdElement>::Mask, LANES>; + type Mask = Mask<<$ty as SimdElement>::Mask, N>; type Scalar = $ty; - type Cast = Simd; + type Unsigned = Simd<$unsigned, N>; + type Cast = Simd; #[inline] fn cast(self) -> Self::Cast { @@ -307,9 +331,41 @@ fn reduce_xor(self) -> Self::Scalar { // Safety: `self` is an integer vector unsafe { intrinsics::simd_reduce_xor(self) } } + + #[inline] + fn swap_bytes(self) -> Self { + // Safety: `self` is an integer vector + unsafe { intrinsics::simd_bswap(self) } + } + + #[inline] + fn reverse_bits(self) -> Self { + // Safety: `self` is an integer vector + unsafe { intrinsics::simd_bitreverse(self) } + } + + #[inline] + fn leading_zeros(self) -> Self::Unsigned { + self.cast::<$unsigned>().leading_zeros() + } + + #[inline] + fn trailing_zeros(self) -> Self::Unsigned { + self.cast::<$unsigned>().trailing_zeros() + } + + #[inline] + fn leading_ones(self) -> Self::Unsigned { + self.cast::<$unsigned>().leading_ones() + } + + #[inline] + fn trailing_ones(self) -> Self::Unsigned { + self.cast::<$unsigned>().trailing_ones() + } } )* } } -impl_trait! { i8, i16, i32, i64, isize } +impl_trait! { i8 (u8), i16 (u16), i32 (u32), i64 (u64), isize (usize) } diff --git a/library/portable-simd/crates/core_simd/src/elements/uint.rs b/library/portable-simd/crates/core_simd/src/simd/num/uint.rs similarity index 58% rename from library/portable-simd/crates/core_simd/src/elements/uint.rs rename to library/portable-simd/crates/core_simd/src/simd/num/uint.rs index 3926c395ec9..c955ee8fe8b 100644 --- a/library/portable-simd/crates/core_simd/src/elements/uint.rs +++ b/library/portable-simd/crates/core_simd/src/simd/num/uint.rs @@ -16,6 +16,12 @@ pub trait SimdUint: Copy + Sealed { #[must_use] fn cast(self) -> Self::Cast; + /// Wrapping negation. + /// + /// Like [`u32::wrapping_neg`], all applications of this function will wrap, with the exception + /// of `-0`. + fn wrapping_neg(self) -> Self; + /// Lanewise saturating add. /// /// # Examples @@ -23,7 +29,7 @@ pub trait SimdUint: Copy + Sealed { /// # #![feature(portable_simd)] /// # #[cfg(feature = "as_crate")] use core_simd::simd; /// # #[cfg(not(feature = "as_crate"))] use core::simd; - /// # use simd::{Simd, SimdUint}; + /// # use simd::prelude::*; /// use core::u32::MAX; /// let x = Simd::from_array([2, 1, 0, MAX]); /// let max = Simd::splat(MAX); @@ -41,7 +47,7 @@ pub trait SimdUint: Copy + Sealed { /// # #![feature(portable_simd)] /// # #[cfg(feature = "as_crate")] use core_simd::simd; /// # #[cfg(not(feature = "as_crate"))] use core::simd; - /// # use simd::{Simd, SimdUint}; + /// # use simd::prelude::*; /// use core::u32::MAX; /// let x = Simd::from_array([2, 1, 0, MAX]); /// let max = Simd::splat(MAX); @@ -51,43 +57,62 @@ pub trait SimdUint: Copy + Sealed { /// assert_eq!(sat, Simd::splat(0)); fn saturating_sub(self, second: Self) -> Self; - /// Returns the sum of the lanes of the vector, with wrapping addition. + /// Returns the sum of the elements of the vector, with wrapping addition. fn reduce_sum(self) -> Self::Scalar; - /// Returns the product of the lanes of the vector, with wrapping multiplication. + /// Returns the product of the elements of the vector, with wrapping multiplication. fn reduce_product(self) -> Self::Scalar; - /// Returns the maximum lane in the vector. + /// Returns the maximum element in the vector. fn reduce_max(self) -> Self::Scalar; - /// Returns the minimum lane in the vector. + /// Returns the minimum element in the vector. fn reduce_min(self) -> Self::Scalar; - /// Returns the cumulative bitwise "and" across the lanes of the vector. + /// Returns the cumulative bitwise "and" across the elements of the vector. fn reduce_and(self) -> Self::Scalar; - /// Returns the cumulative bitwise "or" across the lanes of the vector. + /// Returns the cumulative bitwise "or" across the elements of the vector. fn reduce_or(self) -> Self::Scalar; - /// Returns the cumulative bitwise "xor" across the lanes of the vector. + /// Returns the cumulative bitwise "xor" across the elements of the vector. fn reduce_xor(self) -> Self::Scalar; + + /// Reverses the byte order of each element. + fn swap_bytes(self) -> Self; + + /// Reverses the order of bits in each elemnent. + /// The least significant bit becomes the most significant bit, second least-significant bit becomes second most-significant bit, etc. + fn reverse_bits(self) -> Self; + + /// Returns the number of leading zeros in the binary representation of each element. + fn leading_zeros(self) -> Self; + + /// Returns the number of trailing zeros in the binary representation of each element. + fn trailing_zeros(self) -> Self; + + /// Returns the number of leading ones in the binary representation of each element. + fn leading_ones(self) -> Self; + + /// Returns the number of trailing ones in the binary representation of each element. + fn trailing_ones(self) -> Self; } macro_rules! impl_trait { - { $($ty:ty),* } => { + { $($ty:ident ($signed:ident)),* } => { $( - impl Sealed for Simd<$ty, LANES> + impl Sealed for Simd<$ty, N> where - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { } - impl SimdUint for Simd<$ty, LANES> + impl SimdUint for Simd<$ty, N> where - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { type Scalar = $ty; - type Cast = Simd; + type Cast = Simd; #[inline] fn cast(self) -> Self::Cast { @@ -95,6 +120,12 @@ fn cast(self) -> Self::Cast { unsafe { intrinsics::simd_as(self) } } + #[inline] + fn wrapping_neg(self) -> Self { + use crate::simd::num::SimdInt; + (-self.cast::<$signed>()).cast() + } + #[inline] fn saturating_add(self, second: Self) -> Self { // Safety: `self` is a vector @@ -148,9 +179,43 @@ fn reduce_xor(self) -> Self::Scalar { // Safety: `self` is an integer vector unsafe { intrinsics::simd_reduce_xor(self) } } + + #[inline] + fn swap_bytes(self) -> Self { + // Safety: `self` is an integer vector + unsafe { intrinsics::simd_bswap(self) } + } + + #[inline] + fn reverse_bits(self) -> Self { + // Safety: `self` is an integer vector + unsafe { intrinsics::simd_bitreverse(self) } + } + + #[inline] + fn leading_zeros(self) -> Self { + // Safety: `self` is an integer vector + unsafe { intrinsics::simd_ctlz(self) } + } + + #[inline] + fn trailing_zeros(self) -> Self { + // Safety: `self` is an integer vector + unsafe { intrinsics::simd_cttz(self) } + } + + #[inline] + fn leading_ones(self) -> Self { + (!self).leading_zeros() + } + + #[inline] + fn trailing_ones(self) -> Self { + (!self).trailing_zeros() + } } )* } } -impl_trait! { u8, u16, u32, u64, usize } +impl_trait! { u8 (i8), u16 (i16), u32 (i32), u64 (i64), usize (isize) } diff --git a/library/portable-simd/crates/core_simd/src/simd/prelude.rs b/library/portable-simd/crates/core_simd/src/simd/prelude.rs index e8fdc932d49..4b7c744c013 100644 --- a/library/portable-simd/crates/core_simd/src/simd/prelude.rs +++ b/library/portable-simd/crates/core_simd/src/simd/prelude.rs @@ -7,8 +7,10 @@ #[doc(no_inline)] pub use super::{ - simd_swizzle, Mask, Simd, SimdConstPtr, SimdFloat, SimdInt, SimdMutPtr, SimdOrd, SimdPartialEq, - SimdPartialOrd, SimdUint, + cmp::{SimdOrd, SimdPartialEq, SimdPartialOrd}, + num::{SimdFloat, SimdInt, SimdUint}, + ptr::{SimdConstPtr, SimdMutPtr}, + simd_swizzle, Mask, Simd, }; #[rustfmt::skip] diff --git a/library/portable-simd/crates/core_simd/src/simd/ptr.rs b/library/portable-simd/crates/core_simd/src/simd/ptr.rs new file mode 100644 index 00000000000..3f8e6669118 --- /dev/null +++ b/library/portable-simd/crates/core_simd/src/simd/ptr.rs @@ -0,0 +1,11 @@ +//! Traits for vectors of pointers. + +mod const_ptr; +mod mut_ptr; + +mod sealed { + pub trait Sealed {} +} + +pub use const_ptr::*; +pub use mut_ptr::*; diff --git a/library/portable-simd/crates/core_simd/src/elements/const_ptr.rs b/library/portable-simd/crates/core_simd/src/simd/ptr/const_ptr.rs similarity index 81% rename from library/portable-simd/crates/core_simd/src/elements/const_ptr.rs rename to library/portable-simd/crates/core_simd/src/simd/ptr/const_ptr.rs index f215f9a61d0..97fe3fb600d 100644 --- a/library/portable-simd/crates/core_simd/src/elements/const_ptr.rs +++ b/library/portable-simd/crates/core_simd/src/simd/ptr/const_ptr.rs @@ -1,15 +1,17 @@ use super::sealed::Sealed; -use crate::simd::{intrinsics, LaneCount, Mask, Simd, SimdPartialEq, SimdUint, SupportedLaneCount}; +use crate::simd::{ + cmp::SimdPartialEq, intrinsics, num::SimdUint, LaneCount, Mask, Simd, SupportedLaneCount, +}; /// Operations on SIMD vectors of constant pointers. pub trait SimdConstPtr: Copy + Sealed { - /// Vector of `usize` with the same number of lanes. + /// Vector of `usize` with the same number of elements. type Usize; - /// Vector of `isize` with the same number of lanes. + /// Vector of `isize` with the same number of elements. type Isize; - /// Vector of const pointers with the same number of lanes. + /// Vector of const pointers with the same number of elements. type CastPtr; /// Vector of mutable pointers to the same type. @@ -18,17 +20,17 @@ pub trait SimdConstPtr: Copy + Sealed { /// Mask type used for manipulating this SIMD vector type. type Mask; - /// Returns `true` for each lane that is null. + /// Returns `true` for each element that is null. fn is_null(self) -> Self::Mask; /// Casts to a pointer of another type. /// - /// Equivalent to calling [`pointer::cast`] on each lane. + /// Equivalent to calling [`pointer::cast`] on each element. fn cast(self) -> Self::CastPtr; /// Changes constness without changing the type. /// - /// Equivalent to calling [`pointer::cast_mut`] on each lane. + /// Equivalent to calling [`pointer::cast_mut`] on each element. fn cast_mut(self) -> Self::MutPtr; /// Gets the "address" portion of the pointer. @@ -39,7 +41,7 @@ pub trait SimdConstPtr: Copy + Sealed { /// This method semantically discards *provenance* and /// *address-space* information. To properly restore that information, use [`Self::with_addr`]. /// - /// Equivalent to calling [`pointer::addr`] on each lane. + /// Equivalent to calling [`pointer::addr`] on each element. fn addr(self) -> Self::Usize; /// Creates a new pointer with the given address. @@ -47,7 +49,7 @@ pub trait SimdConstPtr: Copy + Sealed { /// This performs the same operation as a cast, but copies the *address-space* and /// *provenance* of `self` to the new pointer. /// - /// Equivalent to calling [`pointer::with_addr`] on each lane. + /// Equivalent to calling [`pointer::with_addr`] on each element. fn with_addr(self, addr: Self::Usize) -> Self; /// Gets the "address" portion of the pointer, and "exposes" the provenance part for future use @@ -56,39 +58,36 @@ pub trait SimdConstPtr: Copy + Sealed { /// Convert an address back to a pointer, picking up a previously "exposed" provenance. /// - /// Equivalent to calling [`core::ptr::from_exposed_addr`] on each lane. + /// Equivalent to calling [`core::ptr::from_exposed_addr`] on each element. fn from_exposed_addr(addr: Self::Usize) -> Self; /// Calculates the offset from a pointer using wrapping arithmetic. /// - /// Equivalent to calling [`pointer::wrapping_offset`] on each lane. + /// Equivalent to calling [`pointer::wrapping_offset`] on each element. fn wrapping_offset(self, offset: Self::Isize) -> Self; /// Calculates the offset from a pointer using wrapping arithmetic. /// - /// Equivalent to calling [`pointer::wrapping_add`] on each lane. + /// Equivalent to calling [`pointer::wrapping_add`] on each element. fn wrapping_add(self, count: Self::Usize) -> Self; /// Calculates the offset from a pointer using wrapping arithmetic. /// - /// Equivalent to calling [`pointer::wrapping_sub`] on each lane. + /// Equivalent to calling [`pointer::wrapping_sub`] on each element. fn wrapping_sub(self, count: Self::Usize) -> Self; } -impl Sealed for Simd<*const T, LANES> where - LaneCount: SupportedLaneCount -{ -} +impl Sealed for Simd<*const T, N> where LaneCount: SupportedLaneCount {} -impl SimdConstPtr for Simd<*const T, LANES> +impl SimdConstPtr for Simd<*const T, N> where - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { - type Usize = Simd; - type Isize = Simd; - type CastPtr = Simd<*const U, LANES>; - type MutPtr = Simd<*mut T, LANES>; - type Mask = Mask; + type Usize = Simd; + type Isize = Simd; + type CastPtr = Simd<*const U, N>; + type MutPtr = Simd<*mut T, N>; + type Mask = Mask; #[inline] fn is_null(self) -> Self::Mask { diff --git a/library/portable-simd/crates/core_simd/src/elements/mut_ptr.rs b/library/portable-simd/crates/core_simd/src/simd/ptr/mut_ptr.rs similarity index 81% rename from library/portable-simd/crates/core_simd/src/elements/mut_ptr.rs rename to library/portable-simd/crates/core_simd/src/simd/ptr/mut_ptr.rs index 4bdc6a14ce4..e35633d0433 100644 --- a/library/portable-simd/crates/core_simd/src/elements/mut_ptr.rs +++ b/library/portable-simd/crates/core_simd/src/simd/ptr/mut_ptr.rs @@ -1,15 +1,17 @@ use super::sealed::Sealed; -use crate::simd::{intrinsics, LaneCount, Mask, Simd, SimdPartialEq, SimdUint, SupportedLaneCount}; +use crate::simd::{ + cmp::SimdPartialEq, intrinsics, num::SimdUint, LaneCount, Mask, Simd, SupportedLaneCount, +}; /// Operations on SIMD vectors of mutable pointers. pub trait SimdMutPtr: Copy + Sealed { - /// Vector of `usize` with the same number of lanes. + /// Vector of `usize` with the same number of elements. type Usize; - /// Vector of `isize` with the same number of lanes. + /// Vector of `isize` with the same number of elements. type Isize; - /// Vector of const pointers with the same number of lanes. + /// Vector of const pointers with the same number of elements. type CastPtr; /// Vector of constant pointers to the same type. @@ -18,17 +20,17 @@ pub trait SimdMutPtr: Copy + Sealed { /// Mask type used for manipulating this SIMD vector type. type Mask; - /// Returns `true` for each lane that is null. + /// Returns `true` for each element that is null. fn is_null(self) -> Self::Mask; /// Casts to a pointer of another type. /// - /// Equivalent to calling [`pointer::cast`] on each lane. + /// Equivalent to calling [`pointer::cast`] on each element. fn cast(self) -> Self::CastPtr; /// Changes constness without changing the type. /// - /// Equivalent to calling [`pointer::cast_const`] on each lane. + /// Equivalent to calling [`pointer::cast_const`] on each element. fn cast_const(self) -> Self::ConstPtr; /// Gets the "address" portion of the pointer. @@ -36,7 +38,7 @@ pub trait SimdMutPtr: Copy + Sealed { /// This method discards pointer semantic metadata, so the result cannot be /// directly cast into a valid pointer. /// - /// Equivalent to calling [`pointer::addr`] on each lane. + /// Equivalent to calling [`pointer::addr`] on each element. fn addr(self) -> Self::Usize; /// Creates a new pointer with the given address. @@ -44,7 +46,7 @@ pub trait SimdMutPtr: Copy + Sealed { /// This performs the same operation as a cast, but copies the *address-space* and /// *provenance* of `self` to the new pointer. /// - /// Equivalent to calling [`pointer::with_addr`] on each lane. + /// Equivalent to calling [`pointer::with_addr`] on each element. fn with_addr(self, addr: Self::Usize) -> Self; /// Gets the "address" portion of the pointer, and "exposes" the provenance part for future use @@ -53,37 +55,36 @@ pub trait SimdMutPtr: Copy + Sealed { /// Convert an address back to a pointer, picking up a previously "exposed" provenance. /// - /// Equivalent to calling [`core::ptr::from_exposed_addr_mut`] on each lane. + /// Equivalent to calling [`core::ptr::from_exposed_addr_mut`] on each element. fn from_exposed_addr(addr: Self::Usize) -> Self; /// Calculates the offset from a pointer using wrapping arithmetic. /// - /// Equivalent to calling [`pointer::wrapping_offset`] on each lane. + /// Equivalent to calling [`pointer::wrapping_offset`] on each element. fn wrapping_offset(self, offset: Self::Isize) -> Self; /// Calculates the offset from a pointer using wrapping arithmetic. /// - /// Equivalent to calling [`pointer::wrapping_add`] on each lane. + /// Equivalent to calling [`pointer::wrapping_add`] on each element. fn wrapping_add(self, count: Self::Usize) -> Self; /// Calculates the offset from a pointer using wrapping arithmetic. /// - /// Equivalent to calling [`pointer::wrapping_sub`] on each lane. + /// Equivalent to calling [`pointer::wrapping_sub`] on each element. fn wrapping_sub(self, count: Self::Usize) -> Self; } -impl Sealed for Simd<*mut T, LANES> where LaneCount: SupportedLaneCount -{} +impl Sealed for Simd<*mut T, N> where LaneCount: SupportedLaneCount {} -impl SimdMutPtr for Simd<*mut T, LANES> +impl SimdMutPtr for Simd<*mut T, N> where - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { - type Usize = Simd; - type Isize = Simd; - type CastPtr = Simd<*mut U, LANES>; - type ConstPtr = Simd<*const T, LANES>; - type Mask = Mask; + type Usize = Simd; + type Isize = Simd; + type CastPtr = Simd<*mut U, N>; + type ConstPtr = Simd<*const T, N>; + type Mask = Mask; #[inline] fn is_null(self) -> Self::Mask { diff --git a/library/portable-simd/crates/core_simd/src/swizzle.rs b/library/portable-simd/crates/core_simd/src/swizzle.rs index 68f20516cf5..ec8548d5574 100644 --- a/library/portable-simd/crates/core_simd/src/swizzle.rs +++ b/library/portable-simd/crates/core_simd/src/swizzle.rs @@ -1,17 +1,15 @@ use crate::simd::intrinsics; -use crate::simd::{LaneCount, Simd, SimdElement, SupportedLaneCount}; +use crate::simd::{LaneCount, Mask, MaskElement, Simd, SimdElement, SupportedLaneCount}; -/// Constructs a new SIMD vector by copying elements from selected lanes in other vectors. +/// Constructs a new SIMD vector by copying elements from selected elements in other vectors. /// -/// When swizzling one vector, lanes are selected by a `const` array of `usize`, -/// like [`Swizzle`]. +/// When swizzling one vector, elements are selected like [`Swizzle::swizzle`]. /// -/// When swizzling two vectors, lanes are selected by a `const` array of [`Which`], -/// like [`Swizzle2`]. +/// When swizzling two vectors, elements are selected like [`Swizzle::concat_swizzle`]. /// /// # Examples /// -/// With a single SIMD vector, the const array specifies lane indices in that vector: +/// With a single SIMD vector, the const array specifies element indices in that vector: /// ``` /// # #![feature(portable_simd)] /// # use core::simd::{u32x2, u32x4, simd_swizzle}; @@ -21,25 +19,27 @@ /// let r: u32x4 = simd_swizzle!(v, [3, 0, 1, 2]); /// assert_eq!(r.to_array(), [13, 10, 11, 12]); /// -/// // Changing the number of lanes +/// // Changing the number of elements /// let r: u32x2 = simd_swizzle!(v, [3, 1]); /// assert_eq!(r.to_array(), [13, 11]); /// ``` /// -/// With two input SIMD vectors, the const array uses `Which` to specify the source of each index: +/// With two input SIMD vectors, the const array specifies element indices in the concatenation of +/// those vectors: /// ``` /// # #![feature(portable_simd)] -/// # use core::simd::{u32x2, u32x4, simd_swizzle, Which}; -/// use Which::{First, Second}; +/// # #[cfg(feature = "as_crate")] use core_simd::simd; +/// # #[cfg(not(feature = "as_crate"))] use core::simd; +/// # use simd::{u32x2, u32x4, simd_swizzle}; /// let a = u32x4::from_array([0, 1, 2, 3]); /// let b = u32x4::from_array([4, 5, 6, 7]); /// /// // Keeping the same size -/// let r: u32x4 = simd_swizzle!(a, b, [First(0), First(1), Second(2), Second(3)]); +/// let r: u32x4 = simd_swizzle!(a, b, [0, 1, 6, 7]); /// assert_eq!(r.to_array(), [0, 1, 6, 7]); /// -/// // Changing the number of lanes -/// let r: u32x2 = simd_swizzle!(a, b, [First(0), Second(0)]); +/// // Changing the number of elements +/// let r: u32x2 = simd_swizzle!(a, b, [0, 4]); /// assert_eq!(r.to_array(), [0, 4]); /// ``` #[allow(unused_macros)] @@ -50,7 +50,7 @@ { use $crate::simd::Swizzle; struct Impl; - impl Swizzle for Impl { + impl Swizzle<{$index.len()}> for Impl { const INDEX: [usize; {$index.len()}] = $index; } Impl::swizzle($vector) @@ -60,204 +60,194 @@ impl Swizzle for Impl { $first:expr, $second:expr, $index:expr $(,)? ) => { { - use $crate::simd::{Which, Swizzle2}; + use $crate::simd::Swizzle; struct Impl; - impl Swizzle2 for Impl { - const INDEX: [Which; {$index.len()}] = $index; + impl Swizzle<{$index.len()}> for Impl { + const INDEX: [usize; {$index.len()}] = $index; } - Impl::swizzle2($first, $second) + Impl::concat_swizzle($first, $second) } } } -/// Specifies a lane index into one of two SIMD vectors. -/// -/// This is an input type for [Swizzle2] and helper macros like [simd_swizzle]. -#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub enum Which { - /// Index of a lane in the first input SIMD vector. - First(usize), - /// Index of a lane in the second input SIMD vector. - Second(usize), -} - /// Create a vector from the elements of another vector. -pub trait Swizzle { - /// Map from the lanes of the input vector to the output vector. - const INDEX: [usize; OUTPUT_LANES]; +pub trait Swizzle { + /// Map from the elements of the input vector to the output vector. + const INDEX: [usize; N]; - /// Create a new vector from the lanes of `vector`. + /// Create a new vector from the elements of `vector`. /// /// Lane `i` of the output is `vector[Self::INDEX[i]]`. #[inline] #[must_use = "method returns a new vector and does not mutate the original inputs"] - fn swizzle(vector: Simd) -> Simd + fn swizzle(vector: Simd) -> Simd where T: SimdElement, - LaneCount: SupportedLaneCount, - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { - // Safety: `vector` is a vector, and `INDEX_IMPL` is a const array of u32. - unsafe { intrinsics::simd_shuffle(vector, vector, Self::INDEX_IMPL) } + // Safety: `vector` is a vector, and the index is a const array of u32. + unsafe { + intrinsics::simd_shuffle( + vector, + vector, + const { + let mut output = [0; N]; + let mut i = 0; + while i < N { + let index = Self::INDEX[i]; + assert!(index as u32 as usize == index); + assert!( + index < M, + "source element index exceeds input vector length" + ); + output[i] = index as u32; + i += 1; + } + output + }, + ) + } } -} -/// Create a vector from the elements of two other vectors. -pub trait Swizzle2 { - /// Map from the lanes of the input vectors to the output vector - const INDEX: [Which; OUTPUT_LANES]; - - /// Create a new vector from the lanes of `first` and `second`. + /// Create a new vector from the elements of `first` and `second`. /// - /// Lane `i` is `first[j]` when `Self::INDEX[i]` is `First(j)`, or `second[j]` when it is - /// `Second(j)`. + /// Lane `i` of the output is `concat[Self::INDEX[i]]`, where `concat` is the concatenation of + /// `first` and `second`. #[inline] #[must_use = "method returns a new vector and does not mutate the original inputs"] - fn swizzle2( - first: Simd, - second: Simd, - ) -> Simd + fn concat_swizzle(first: Simd, second: Simd) -> Simd where T: SimdElement, - LaneCount: SupportedLaneCount, - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { - // Safety: `first` and `second` are vectors, and `INDEX_IMPL` is a const array of u32. - unsafe { intrinsics::simd_shuffle(first, second, Self::INDEX_IMPL) } + // Safety: `first` and `second` are vectors, and the index is a const array of u32. + unsafe { + intrinsics::simd_shuffle( + first, + second, + const { + let mut output = [0; N]; + let mut i = 0; + while i < N { + let index = Self::INDEX[i]; + assert!(index as u32 as usize == index); + assert!( + index < 2 * M, + "source element index exceeds input vector length" + ); + output[i] = index as u32; + i += 1; + } + output + }, + ) + } + } + + /// Create a new mask from the elements of `mask`. + /// + /// Element `i` of the output is `concat[Self::INDEX[i]]`, where `concat` is the concatenation of + /// `first` and `second`. + #[inline] + #[must_use = "method returns a new mask and does not mutate the original inputs"] + fn swizzle_mask(mask: Mask) -> Mask + where + T: MaskElement, + LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, + { + // SAFETY: all elements of this mask come from another mask + unsafe { Mask::from_int_unchecked(Self::swizzle(mask.to_int())) } + } + + /// Create a new mask from the elements of `first` and `second`. + /// + /// Element `i` of the output is `concat[Self::INDEX[i]]`, where `concat` is the concatenation of + /// `first` and `second`. + #[inline] + #[must_use = "method returns a new mask and does not mutate the original inputs"] + fn concat_swizzle_mask(first: Mask, second: Mask) -> Mask + where + T: MaskElement, + LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, + { + // SAFETY: all elements of this mask come from another mask + unsafe { Mask::from_int_unchecked(Self::concat_swizzle(first.to_int(), second.to_int())) } } } -/// The `simd_shuffle` intrinsic expects `u32`, so do error checking and conversion here. -/// This trait hides `INDEX_IMPL` from the public API. -trait SwizzleImpl { - const INDEX_IMPL: [u32; OUTPUT_LANES]; -} - -impl SwizzleImpl - for T -where - T: Swizzle + ?Sized, -{ - const INDEX_IMPL: [u32; OUTPUT_LANES] = { - let mut output = [0; OUTPUT_LANES]; - let mut i = 0; - while i < OUTPUT_LANES { - let index = Self::INDEX[i]; - assert!(index as u32 as usize == index); - assert!(index < INPUT_LANES, "source lane exceeds input lane count",); - output[i] = index as u32; - i += 1; - } - output - }; -} - -/// The `simd_shuffle` intrinsic expects `u32`, so do error checking and conversion here. -/// This trait hides `INDEX_IMPL` from the public API. -trait Swizzle2Impl { - const INDEX_IMPL: [u32; OUTPUT_LANES]; -} - -impl Swizzle2Impl - for T -where - T: Swizzle2 + ?Sized, -{ - const INDEX_IMPL: [u32; OUTPUT_LANES] = { - let mut output = [0; OUTPUT_LANES]; - let mut i = 0; - while i < OUTPUT_LANES { - let (offset, index) = match Self::INDEX[i] { - Which::First(index) => (false, index), - Which::Second(index) => (true, index), - }; - assert!(index < INPUT_LANES, "source lane exceeds input lane count",); - - // lanes are indexed by the first vector, then second vector - let index = if offset { index + INPUT_LANES } else { index }; - assert!(index as u32 as usize == index); - output[i] = index as u32; - i += 1; - } - output - }; -} - -impl Simd +impl Simd where T: SimdElement, - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, { - /// Reverse the order of the lanes in the vector. + /// Reverse the order of the elements in the vector. #[inline] #[must_use = "method returns a new vector and does not mutate the original inputs"] pub fn reverse(self) -> Self { - const fn reverse_index() -> [usize; LANES] { - let mut index = [0; LANES]; - let mut i = 0; - while i < LANES { - index[i] = LANES - i - 1; - i += 1; - } - index - } - struct Reverse; - impl Swizzle for Reverse { - const INDEX: [usize; LANES] = reverse_index::(); + impl Swizzle for Reverse { + const INDEX: [usize; N] = const { + let mut index = [0; N]; + let mut i = 0; + while i < N { + index[i] = N - i - 1; + i += 1; + } + index + }; } Reverse::swizzle(self) } /// Rotates the vector such that the first `OFFSET` elements of the slice move to the end - /// while the last `LANES - OFFSET` elements move to the front. After calling `rotate_lanes_left`, - /// the element previously in lane `OFFSET` will become the first element in the slice. + /// while the last `self.len() - OFFSET` elements move to the front. After calling `rotate_elements_left`, + /// the element previously at index `OFFSET` will become the first element in the slice. #[inline] #[must_use = "method returns a new vector and does not mutate the original inputs"] - pub fn rotate_lanes_left(self) -> Self { - const fn rotate_index() -> [usize; LANES] { - let offset = OFFSET % LANES; - let mut index = [0; LANES]; - let mut i = 0; - while i < LANES { - index[i] = (i + offset) % LANES; - i += 1; - } - index - } - + pub fn rotate_elements_left(self) -> Self { struct Rotate; - impl Swizzle for Rotate { - const INDEX: [usize; LANES] = rotate_index::(); + impl Swizzle for Rotate { + const INDEX: [usize; N] = const { + let offset = OFFSET % N; + let mut index = [0; N]; + let mut i = 0; + while i < N { + index[i] = (i + offset) % N; + i += 1; + } + index + }; } Rotate::::swizzle(self) } - /// Rotates the vector such that the first `LANES - OFFSET` elements of the vector move to - /// the end while the last `OFFSET` elements move to the front. After calling `rotate_lanes_right`, - /// the element previously at index `LANES - OFFSET` will become the first element in the slice. + /// Rotates the vector such that the first `self.len() - OFFSET` elements of the vector move to + /// the end while the last `OFFSET` elements move to the front. After calling `rotate_elements_right`, + /// the element previously at index `self.len() - OFFSET` will become the first element in the slice. #[inline] #[must_use = "method returns a new vector and does not mutate the original inputs"] - pub fn rotate_lanes_right(self) -> Self { - const fn rotate_index() -> [usize; LANES] { - let offset = LANES - OFFSET % LANES; - let mut index = [0; LANES]; - let mut i = 0; - while i < LANES { - index[i] = (i + offset) % LANES; - i += 1; - } - index - } - + pub fn rotate_elements_right(self) -> Self { struct Rotate; - impl Swizzle for Rotate { - const INDEX: [usize; LANES] = rotate_index::(); + impl Swizzle for Rotate { + const INDEX: [usize; N] = const { + let offset = N - OFFSET % N; + let mut index = [0; N]; + let mut i = 0; + while i < N { + index[i] = (i + offset) % N; + i += 1; + } + index + }; } Rotate::::swizzle(self) @@ -265,7 +255,7 @@ impl Swizzle for Rotate Swizzle for Rotate (Self, Self) { - const fn interleave(high: bool) -> [Which; LANES] { - let mut idx = [Which::First(0); LANES]; + const fn interleave(high: bool) -> [usize; N] { + let mut idx = [0; N]; let mut i = 0; - while i < LANES { - // Treat the source as a concatenated vector - let dst_index = if high { i + LANES } else { i }; - let src_index = dst_index / 2 + (dst_index % 2) * LANES; - idx[i] = if src_index < LANES { - Which::First(src_index) - } else { - Which::Second(src_index % LANES) - }; + while i < N { + let dst_index = if high { i + N } else { i }; + let src_index = dst_index / 2 + (dst_index % 2) * N; + idx[i] = src_index; i += 1; } idx @@ -302,24 +287,27 @@ pub fn interleave(self, other: Self) -> (Self, Self) { struct Lo; struct Hi; - impl Swizzle2 for Lo { - const INDEX: [Which; LANES] = interleave::(false); + impl Swizzle for Lo { + const INDEX: [usize; N] = interleave::(false); } - impl Swizzle2 for Hi { - const INDEX: [Which; LANES] = interleave::(true); + impl Swizzle for Hi { + const INDEX: [usize; N] = interleave::(true); } - (Lo::swizzle2(self, other), Hi::swizzle2(self, other)) + ( + Lo::concat_swizzle(self, other), + Hi::concat_swizzle(self, other), + ) } /// Deinterleave two vectors. /// - /// The first result takes every other lane of `self` and then `other`, starting with - /// the first lane. + /// The first result takes every other element of `self` and then `other`, starting with + /// the first element. /// - /// The second result takes every other lane of `self` and then `other`, starting with - /// the second lane. + /// The second result takes every other element of `self` and then `other`, starting with + /// the second element. /// /// The reverse of this operation is [`Simd::interleave`]. /// @@ -335,17 +323,11 @@ impl Swizzle2 for Hi { #[inline] #[must_use = "method returns a new vector and does not mutate the original inputs"] pub fn deinterleave(self, other: Self) -> (Self, Self) { - const fn deinterleave(second: bool) -> [Which; LANES] { - let mut idx = [Which::First(0); LANES]; + const fn deinterleave(second: bool) -> [usize; N] { + let mut idx = [0; N]; let mut i = 0; - while i < LANES { - // Treat the source as a concatenated vector - let src_index = i * 2 + second as usize; - idx[i] = if src_index < LANES { - Which::First(src_index) - } else { - Which::Second(src_index % LANES) - }; + while i < N { + idx[i] = i * 2 + second as usize; i += 1; } idx @@ -354,14 +336,52 @@ pub fn deinterleave(self, other: Self) -> (Self, Self) { struct Even; struct Odd; - impl Swizzle2 for Even { - const INDEX: [Which; LANES] = deinterleave::(false); + impl Swizzle for Even { + const INDEX: [usize; N] = deinterleave::(false); } - impl Swizzle2 for Odd { - const INDEX: [Which; LANES] = deinterleave::(true); + impl Swizzle for Odd { + const INDEX: [usize; N] = deinterleave::(true); } - (Even::swizzle2(self, other), Odd::swizzle2(self, other)) + ( + Even::concat_swizzle(self, other), + Odd::concat_swizzle(self, other), + ) + } + + /// Resize a vector. + /// + /// If `M` > `N`, extends the length of a vector, setting the new elements to `value`. + /// If `M` < `N`, truncates the vector to the first `M` elements. + /// + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::u32x4; + /// let x = u32x4::from_array([0, 1, 2, 3]); + /// assert_eq!(x.resize::<8>(9).to_array(), [0, 1, 2, 3, 9, 9, 9, 9]); + /// assert_eq!(x.resize::<2>(9).to_array(), [0, 1]); + /// ``` + #[inline] + #[must_use = "method returns a new vector and does not mutate the original inputs"] + pub fn resize(self, value: T) -> Simd + where + LaneCount: SupportedLaneCount, + { + struct Resize; + impl Swizzle for Resize { + const INDEX: [usize; M] = const { + let mut index = [0; M]; + let mut i = 0; + while i < M { + index[i] = if i < N { i } else { N }; + i += 1; + } + index + }; + } + Resize::::concat_swizzle(self, Simd::splat(value)) } } diff --git a/library/portable-simd/crates/core_simd/src/swizzle_dyn.rs b/library/portable-simd/crates/core_simd/src/swizzle_dyn.rs index ce621792534..bd8a38e350d 100644 --- a/library/portable-simd/crates/core_simd/src/swizzle_dyn.rs +++ b/library/portable-simd/crates/core_simd/src/swizzle_dyn.rs @@ -86,7 +86,7 @@ pub fn swizzle_dyn(self, idxs: Simd) -> Self { #[inline] #[allow(clippy::let_and_return)] unsafe fn avx2_pshufb(bytes: Simd, idxs: Simd) -> Simd { - use crate::simd::SimdPartialOrd; + use crate::simd::cmp::SimdPartialOrd; #[cfg(target_arch = "x86")] use core::arch::x86; #[cfg(target_arch = "x86_64")] @@ -149,7 +149,7 @@ fn zeroing_idxs(idxs: Simd) -> Simd // On x86, make sure the top bit is set. #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] let idxs = { - use crate::simd::SimdPartialOrd; + use crate::simd::cmp::SimdPartialOrd; idxs.simd_lt(Simd::splat(N as u8)) .select(idxs, Simd::splat(u8::MAX)) }; diff --git a/library/portable-simd/crates/core_simd/src/to_bytes.rs b/library/portable-simd/crates/core_simd/src/to_bytes.rs index b36b1a347b2..222526c4ab3 100644 --- a/library/portable-simd/crates/core_simd/src/to_bytes.rs +++ b/library/portable-simd/crates/core_simd/src/to_bytes.rs @@ -1,24 +1,125 @@ +use crate::simd::{ + num::{SimdFloat, SimdInt, SimdUint}, + LaneCount, Simd, SimdElement, SupportedLaneCount, +}; + +mod sealed { + use super::*; + pub trait Sealed {} + impl Sealed for Simd where LaneCount: SupportedLaneCount {} +} +use sealed::Sealed; + +/// Convert SIMD vectors to vectors of bytes +pub trait ToBytes: Sealed { + /// This type, reinterpreted as bytes. + type Bytes: Copy + + Unpin + + Send + + Sync + + AsRef<[u8]> + + AsMut<[u8]> + + SimdUint + + 'static; + + /// Return the memory representation of this integer as a byte array in native byte + /// order. + fn to_ne_bytes(self) -> Self::Bytes; + + /// Return the memory representation of this integer as a byte array in big-endian + /// (network) byte order. + fn to_be_bytes(self) -> Self::Bytes; + + /// Return the memory representation of this integer as a byte array in little-endian + /// byte order. + fn to_le_bytes(self) -> Self::Bytes; + + /// Create a native endian integer value from its memory representation as a byte array + /// in native endianness. + fn from_ne_bytes(bytes: Self::Bytes) -> Self; + + /// Create an integer value from its representation as a byte array in big endian. + fn from_be_bytes(bytes: Self::Bytes) -> Self; + + /// Create an integer value from its representation as a byte array in little endian. + fn from_le_bytes(bytes: Self::Bytes) -> Self; +} + +macro_rules! swap_bytes { + { f32, $x:expr } => { Simd::from_bits($x.to_bits().swap_bytes()) }; + { f64, $x:expr } => { Simd::from_bits($x.to_bits().swap_bytes()) }; + { $ty:ty, $x:expr } => { $x.swap_bytes() } +} + macro_rules! impl_to_bytes { - { $ty:ty, $size:literal } => { - impl crate::simd::Simd<$ty, LANES> - where - crate::simd::LaneCount: crate::simd::SupportedLaneCount, - crate::simd::LaneCount<{{ $size * LANES }}>: crate::simd::SupportedLaneCount, - { - /// Return the memory representation of this integer as a byte array in native byte - /// order. - pub fn to_ne_bytes(self) -> crate::simd::Simd { + { $ty:tt, 1 } => { impl_to_bytes! { $ty, 1 * [1, 2, 4, 8, 16, 32, 64] } }; + { $ty:tt, 2 } => { impl_to_bytes! { $ty, 2 * [1, 2, 4, 8, 16, 32] } }; + { $ty:tt, 4 } => { impl_to_bytes! { $ty, 4 * [1, 2, 4, 8, 16] } }; + { $ty:tt, 8 } => { impl_to_bytes! { $ty, 8 * [1, 2, 4, 8] } }; + { $ty:tt, 16 } => { impl_to_bytes! { $ty, 16 * [1, 2, 4] } }; + { $ty:tt, 32 } => { impl_to_bytes! { $ty, 32 * [1, 2] } }; + { $ty:tt, 64 } => { impl_to_bytes! { $ty, 64 * [1] } }; + + { $ty:tt, $size:literal * [$($elems:literal),*] } => { + $( + impl ToBytes for Simd<$ty, $elems> { + type Bytes = Simd; + + #[inline] + fn to_ne_bytes(self) -> Self::Bytes { // Safety: transmuting between vectors is safe - unsafe { core::mem::transmute_copy(&self) } + unsafe { + #![allow(clippy::useless_transmute)] + core::mem::transmute(self) + } } - /// Create a native endian integer value from its memory representation as a byte array - /// in native endianness. - pub fn from_ne_bytes(bytes: crate::simd::Simd) -> Self { + #[inline] + fn to_be_bytes(mut self) -> Self::Bytes { + if !cfg!(target_endian = "big") { + self = swap_bytes!($ty, self); + } + self.to_ne_bytes() + } + + #[inline] + fn to_le_bytes(mut self) -> Self::Bytes { + if !cfg!(target_endian = "little") { + self = swap_bytes!($ty, self); + } + self.to_ne_bytes() + } + + #[inline] + fn from_ne_bytes(bytes: Self::Bytes) -> Self { // Safety: transmuting between vectors is safe - unsafe { core::mem::transmute_copy(&bytes) } + unsafe { + #![allow(clippy::useless_transmute)] + core::mem::transmute(bytes) + } + } + + #[inline] + fn from_be_bytes(bytes: Self::Bytes) -> Self { + let ret = Self::from_ne_bytes(bytes); + if cfg!(target_endian = "big") { + ret + } else { + swap_bytes!($ty, ret) + } + } + + #[inline] + fn from_le_bytes(bytes: Self::Bytes) -> Self { + let ret = Self::from_ne_bytes(bytes); + if cfg!(target_endian = "little") { + ret + } else { + swap_bytes!($ty, ret) + } } } + )* } } @@ -39,3 +140,6 @@ pub fn from_ne_bytes(bytes: crate::simd::Simd) -> Self impl_to_bytes! { isize, 4 } #[cfg(target_pointer_width = "64")] impl_to_bytes! { isize, 8 } + +impl_to_bytes! { f32, 4 } +impl_to_bytes! { f64, 8 } diff --git a/library/portable-simd/crates/core_simd/src/vector.rs b/library/portable-simd/crates/core_simd/src/vector.rs index 9aa7bacfce9..105c06741c5 100644 --- a/library/portable-simd/crates/core_simd/src/vector.rs +++ b/library/portable-simd/crates/core_simd/src/vector.rs @@ -1,6 +1,8 @@ use crate::simd::{ - intrinsics, LaneCount, Mask, MaskElement, SimdConstPtr, SimdMutPtr, SimdPartialOrd, - SupportedLaneCount, Swizzle, + cmp::SimdPartialOrd, + intrinsics, + ptr::{SimdConstPtr, SimdMutPtr}, + LaneCount, Mask, MaskElement, SupportedLaneCount, Swizzle, }; use core::convert::{TryFrom, TryInto}; @@ -110,7 +112,7 @@ impl Simd T: SimdElement, { /// Number of elements in this vector. - pub const LANES: usize = N; + pub const LEN: usize = N; /// Returns the number of elements in this SIMD vector. /// @@ -118,13 +120,16 @@ impl Simd /// /// ``` /// # #![feature(portable_simd)] - /// # use core::simd::u32x4; + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::u32x4; /// let v = u32x4::splat(0); - /// assert_eq!(v.lanes(), 4); + /// assert_eq!(v.len(), 4); /// ``` #[inline] - pub const fn lanes(&self) -> usize { - Self::LANES + #[allow(clippy::len_without_is_empty)] + pub const fn len(&self) -> usize { + Self::LEN } /// Constructs a new SIMD vector with all elements set to the given value. @@ -133,7 +138,9 @@ pub const fn lanes(&self) -> usize { /// /// ``` /// # #![feature(portable_simd)] - /// # use core::simd::u32x4; + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::u32x4; /// let v = u32x4::splat(8); /// assert_eq!(v.as_array(), &[8, 8, 8, 8]); /// ``` @@ -142,10 +149,10 @@ pub fn splat(value: T) -> Self { // This is preferred over `[value; N]`, since it's explicitly a splat: // https://github.com/rust-lang/rust/issues/97804 struct Splat; - impl Swizzle<1, N> for Splat { + impl Swizzle for Splat { const INDEX: [usize; N] = [0; N]; } - Splat::swizzle(Simd::::from([value])) + Splat::swizzle::(Simd::::from([value])) } /// Returns an array reference containing the entire SIMD vector. @@ -271,7 +278,7 @@ impl Swizzle<1, N> for Splat { #[track_caller] pub const fn from_slice(slice: &[T]) -> Self { assert!( - slice.len() >= Self::LANES, + slice.len() >= Self::LEN, "slice length must be at least the number of elements" ); // SAFETY: We just checked that the slice contains @@ -301,7 +308,7 @@ pub const fn from_slice(slice: &[T]) -> Self { #[track_caller] pub fn copy_to_slice(self, slice: &mut [T]) { assert!( - slice.len() >= Self::LANES, + slice.len() >= Self::LEN, "slice length must be at least the number of elements" ); // SAFETY: We just checked that the slice contains @@ -394,7 +401,7 @@ pub fn gather_select( /// # #![feature(portable_simd)] /// # #[cfg(feature = "as_crate")] use core_simd::simd; /// # #[cfg(not(feature = "as_crate"))] use core::simd; - /// # use simd::{Simd, SimdPartialOrd, Mask}; + /// # use simd::{Simd, cmp::SimdPartialOrd, Mask}; /// let vec: Vec = vec![10, 11, 12, 13, 14, 15, 16, 17, 18]; /// let idxs = Simd::from_array([9, 3, 0, 5]); // Includes an out-of-bounds index /// let alt = Simd::from_array([-5, -4, -3, -2]); @@ -434,7 +441,7 @@ pub unsafe fn gather_select_unchecked( /// # #![feature(portable_simd)] /// # #[cfg(feature = "as_crate")] use core_simd::simd; /// # #[cfg(not(feature = "as_crate"))] use core::simd; - /// # use simd::{Simd, SimdConstPtr}; + /// # use simd::prelude::*; /// let values = [6, 2, 4, 9]; /// let offsets = Simd::from_array([1, 0, 0, 3]); /// let source = Simd::splat(values.as_ptr()).wrapping_add(offsets); @@ -467,7 +474,7 @@ pub unsafe fn gather_ptr(source: Simd<*const T, N>) -> Self /// # #![feature(portable_simd)] /// # #[cfg(feature = "as_crate")] use core_simd::simd; /// # #[cfg(not(feature = "as_crate"))] use core::simd; - /// # use simd::{Mask, Simd, SimdConstPtr}; + /// # use simd::prelude::*; /// let values = [6, 2, 4, 9]; /// let enable = Mask::from_array([true, true, false, true]); /// let offsets = Simd::from_array([1, 0, 0, 3]); @@ -550,7 +557,7 @@ pub fn scatter_select(self, slice: &mut [T], enable: Mask, idxs: Simd< /// # #![feature(portable_simd)] /// # #[cfg(feature = "as_crate")] use core_simd::simd; /// # #[cfg(not(feature = "as_crate"))] use core::simd; - /// # use simd::{Simd, SimdPartialOrd, Mask}; + /// # use simd::{Simd, cmp::SimdPartialOrd, Mask}; /// let mut vec: Vec = vec![10, 11, 12, 13, 14, 15, 16, 17, 18]; /// let idxs = Simd::from_array([9, 3, 0, 0]); /// let vals = Simd::from_array([-27, 82, -41, 124]); @@ -604,7 +611,7 @@ pub unsafe fn scatter_select_unchecked( /// # #![feature(portable_simd)] /// # #[cfg(feature = "as_crate")] use core_simd::simd; /// # #[cfg(not(feature = "as_crate"))] use core::simd; - /// # use simd::{Simd, SimdMutPtr}; + /// # use simd::{Simd, ptr::SimdMutPtr}; /// let mut values = [0; 4]; /// let offset = Simd::from_array([3, 2, 1, 0]); /// let ptrs = Simd::splat(values.as_mut_ptr()).wrapping_add(offset); @@ -631,7 +638,7 @@ pub unsafe fn scatter_ptr(self, dest: Simd<*mut T, N>) { /// # #![feature(portable_simd)] /// # #[cfg(feature = "as_crate")] use core_simd::simd; /// # #[cfg(not(feature = "as_crate"))] use core::simd; - /// # use simd::{Mask, Simd, SimdMutPtr}; + /// # use simd::{Mask, Simd, ptr::SimdMutPtr}; /// let mut values = [0; 4]; /// let offset = Simd::from_array([3, 2, 1, 0]); /// let ptrs = Simd::splat(values.as_mut_ptr()).wrapping_add(offset); diff --git a/library/portable-simd/crates/core_simd/src/vendor.rs b/library/portable-simd/crates/core_simd/src/vendor.rs index 9fb70218c95..6223bedb4e1 100644 --- a/library/portable-simd/crates/core_simd/src/vendor.rs +++ b/library/portable-simd/crates/core_simd/src/vendor.rs @@ -21,7 +21,7 @@ fn from(value: $from) -> $to { #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] mod x86; -#[cfg(any(target_arch = "wasm32"))] +#[cfg(target_arch = "wasm32")] mod wasm32; #[cfg(any(target_arch = "aarch64", target_arch = "arm",))] diff --git a/library/portable-simd/crates/core_simd/src/vendor/x86.rs b/library/portable-simd/crates/core_simd/src/vendor/x86.rs index 0dd47015ed2..66aaf90eef5 100644 --- a/library/portable-simd/crates/core_simd/src/vendor/x86.rs +++ b/library/portable-simd/crates/core_simd/src/vendor/x86.rs @@ -1,6 +1,6 @@ use crate::simd::*; -#[cfg(any(target_arch = "x86"))] +#[cfg(target_arch = "x86")] use core::arch::x86::*; #[cfg(target_arch = "x86_64")] diff --git a/library/portable-simd/crates/core_simd/tests/cast.rs b/library/portable-simd/crates/core_simd/tests/cast.rs index 00545936ea2..185e1945faa 100644 --- a/library/portable-simd/crates/core_simd/tests/cast.rs +++ b/library/portable-simd/crates/core_simd/tests/cast.rs @@ -3,7 +3,7 @@ macro_rules! cast_types { ($start:ident, $($target:ident),*) => { mod $start { #[allow(unused)] - use core_simd::simd::{Simd, SimdInt, SimdUint, SimdFloat}; + use core_simd::simd::prelude::*; type Vector = Simd<$start, N>; $( mod $target { diff --git a/library/portable-simd/crates/core_simd/tests/masks.rs b/library/portable-simd/crates/core_simd/tests/masks.rs index 9f8bad1c36c..00fc2a24e27 100644 --- a/library/portable-simd/crates/core_simd/tests/masks.rs +++ b/library/portable-simd/crates/core_simd/tests/masks.rs @@ -72,7 +72,6 @@ fn roundtrip_int_conversion() { #[test] fn roundtrip_bitmask_conversion() { - use core_simd::simd::ToBitMask; let values = [ true, false, false, true, false, false, true, false, true, true, false, false, false, false, false, true, @@ -85,8 +84,6 @@ fn roundtrip_bitmask_conversion() { #[test] fn roundtrip_bitmask_conversion_short() { - use core_simd::simd::ToBitMask; - let values = [ false, false, false, true, ]; @@ -125,18 +122,17 @@ fn cast_impl() cast_impl::(); } - #[cfg(feature = "generic_const_exprs")] #[test] - fn roundtrip_bitmask_array_conversion() { - use core_simd::simd::ToBitMaskArray; + fn roundtrip_bitmask_vector_conversion() { + use core_simd::simd::ToBytes; let values = [ true, false, false, true, false, false, true, false, true, true, false, false, false, false, false, true, ]; let mask = Mask::<$type, 16>::from_array(values); - let bitmask = mask.to_bitmask_array(); - assert_eq!(bitmask, [0b01001001, 0b10000011]); - assert_eq!(Mask::<$type, 16>::from_bitmask_array(bitmask), mask); + let bitmask = mask.to_bitmask_vector(); + assert_eq!(bitmask.resize::<2>(0).to_ne_bytes()[..2], [0b01001001, 0b10000011]); + assert_eq!(Mask::<$type, 16>::from_bitmask_vector(bitmask), mask); } } } diff --git a/library/portable-simd/crates/core_simd/tests/ops_macros.rs b/library/portable-simd/crates/core_simd/tests/ops_macros.rs index 3a02f3f01e1..aa565a13752 100644 --- a/library/portable-simd/crates/core_simd/tests/ops_macros.rs +++ b/library/portable-simd/crates/core_simd/tests/ops_macros.rs @@ -6,7 +6,7 @@ macro_rules! impl_unary_op_test { { $scalar:ty, $trait:ident :: $fn:ident, $scalar_fn:expr } => { test_helpers::test_lanes! { fn $fn() { - test_helpers::test_unary_elementwise( + test_helpers::test_unary_elementwise_flush_subnormals( & as core::ops::$trait>::$fn, &$scalar_fn, &|_| true, @@ -31,7 +31,7 @@ mod $fn { test_helpers::test_lanes! { fn normal() { - test_helpers::test_binary_elementwise( + test_helpers::test_binary_elementwise_flush_subnormals( & as core::ops::$trait>::$fn, &$scalar_fn, &|_, _| true, @@ -39,7 +39,7 @@ fn normal() { } fn assign() { - test_helpers::test_binary_elementwise( + test_helpers::test_binary_elementwise_flush_subnormals( &|mut a, b| { as core::ops::$trait_assign>::$fn_assign(&mut a, b); a }, &$scalar_fn, &|_, _| true, @@ -68,6 +68,7 @@ mod $fn { test_helpers::test_lanes! { fn normal() { + #![allow(clippy::redundant_closure_call)] test_helpers::test_binary_elementwise( & as core::ops::$trait>::$fn, &$scalar_fn, @@ -76,6 +77,7 @@ fn normal() { } fn assign() { + #![allow(clippy::redundant_closure_call)] test_helpers::test_binary_elementwise( &|mut a, b| { as core::ops::$trait_assign>::$fn_assign(&mut a, b); a }, &$scalar_fn, @@ -94,11 +96,43 @@ fn assign() { macro_rules! impl_common_integer_tests { { $vector:ident, $scalar:ident } => { test_helpers::test_lanes! { + fn shr() { + use core::ops::Shr; + let shr = |x: $scalar, y: $scalar| x.wrapping_shr(y as _); + test_helpers::test_binary_elementwise( + &<$vector:: as Shr<$vector::>>::shr, + &shr, + &|_, _| true, + ); + test_helpers::test_binary_scalar_rhs_elementwise( + &<$vector:: as Shr<$scalar>>::shr, + &shr, + &|_, _| true, + ); + } + + fn shl() { + use core::ops::Shl; + let shl = |x: $scalar, y: $scalar| x.wrapping_shl(y as _); + test_helpers::test_binary_elementwise( + &<$vector:: as Shl<$vector::>>::shl, + &shl, + &|_, _| true, + ); + test_helpers::test_binary_scalar_rhs_elementwise( + &<$vector:: as Shl<$scalar>>::shl, + &shl, + &|_, _| true, + ); + } + fn reduce_sum() { test_helpers::test_1(&|x| { + use test_helpers::subnormals::{flush, flush_in}; test_helpers::prop_assert_biteq! ( $vector::::from_array(x).reduce_sum(), x.iter().copied().fold(0 as $scalar, $scalar::wrapping_add), + flush(x.iter().copied().map(flush_in).fold(0 as $scalar, $scalar::wrapping_add)), ); Ok(()) }); @@ -106,9 +140,11 @@ fn reduce_sum() { fn reduce_product() { test_helpers::test_1(&|x| { + use test_helpers::subnormals::{flush, flush_in}; test_helpers::prop_assert_biteq! ( $vector::::from_array(x).reduce_product(), x.iter().copied().fold(1 as $scalar, $scalar::wrapping_mul), + flush(x.iter().copied().map(flush_in).fold(1 as $scalar, $scalar::wrapping_mul)), ); Ok(()) }); @@ -163,6 +199,54 @@ fn reduce_min() { Ok(()) }); } + + fn swap_bytes() { + test_helpers::test_unary_elementwise( + &$vector::::swap_bytes, + &$scalar::swap_bytes, + &|_| true, + ) + } + + fn reverse_bits() { + test_helpers::test_unary_elementwise( + &$vector::::reverse_bits, + &$scalar::reverse_bits, + &|_| true, + ) + } + + fn leading_zeros() { + test_helpers::test_unary_elementwise( + &$vector::::leading_zeros, + &|x| x.leading_zeros() as _, + &|_| true, + ) + } + + fn trailing_zeros() { + test_helpers::test_unary_elementwise( + &$vector::::trailing_zeros, + &|x| x.trailing_zeros() as _, + &|_| true, + ) + } + + fn leading_ones() { + test_helpers::test_unary_elementwise( + &$vector::::leading_ones, + &|x| x.leading_ones() as _, + &|_| true, + ) + } + + fn trailing_ones() { + test_helpers::test_unary_elementwise( + &$vector::::trailing_ones, + &|x| x.trailing_ones() as _, + &|_| true, + ) + } } } } @@ -172,7 +256,7 @@ fn reduce_min() { macro_rules! impl_signed_tests { { $scalar:tt } => { mod $scalar { - use core_simd::simd::SimdInt; + use core_simd::simd::num::SimdInt; type Vector = core_simd::simd::Simd; type Scalar = $scalar; @@ -224,7 +308,7 @@ fn rem_min_may_overflow() { } fn simd_min() { - use core_simd::simd::SimdOrd; + use core_simd::simd::cmp::SimdOrd; let a = Vector::::splat(Scalar::MIN); let b = Vector::::splat(0); assert_eq!(a.simd_min(b), a); @@ -234,7 +318,7 @@ fn simd_min() { } fn simd_max() { - use core_simd::simd::SimdOrd; + use core_simd::simd::cmp::SimdOrd; let a = Vector::::splat(Scalar::MIN); let b = Vector::::splat(0); assert_eq!(a.simd_max(b), b); @@ -244,7 +328,7 @@ fn simd_max() { } fn simd_clamp() { - use core_simd::simd::SimdOrd; + use core_simd::simd::cmp::SimdOrd; let min = Vector::::splat(Scalar::MIN); let max = Vector::::splat(Scalar::MAX); let zero = Vector::::splat(0); @@ -313,7 +397,7 @@ fn rem_neg_one_no_panic() { macro_rules! impl_unsigned_tests { { $scalar:tt } => { mod $scalar { - use core_simd::simd::SimdUint; + use core_simd::simd::num::SimdUint; type Vector = core_simd::simd::Simd; type Scalar = $scalar; @@ -327,6 +411,16 @@ fn rem_zero_panic() { } } + test_helpers::test_lanes! { + fn wrapping_neg() { + test_helpers::test_unary_elementwise( + &Vector::::wrapping_neg, + &Scalar::wrapping_neg, + &|_| true, + ); + } + } + impl_binary_op_test!(Scalar, Add::add, AddAssign::add_assign, Scalar::wrapping_add); impl_binary_op_test!(Scalar, Sub::sub, SubAssign::sub_assign, Scalar::wrapping_sub); impl_binary_op_test!(Scalar, Mul::mul, MulAssign::mul_assign, Scalar::wrapping_mul); @@ -348,7 +442,7 @@ fn rem_zero_panic() { macro_rules! impl_float_tests { { $scalar:tt, $int_scalar:tt } => { mod $scalar { - use core_simd::simd::SimdFloat; + use core_simd::simd::num::SimdFloat; type Vector = core_simd::simd::Simd; type Scalar = $scalar; @@ -433,7 +527,7 @@ fn recip() { } fn to_degrees() { - test_helpers::test_unary_elementwise( + test_helpers::test_unary_elementwise_flush_subnormals( &Vector::::to_degrees, &Scalar::to_degrees, &|_| true, @@ -441,7 +535,7 @@ fn to_degrees() { } fn to_radians() { - test_helpers::test_unary_elementwise( + test_helpers::test_unary_elementwise_flush_subnormals( &Vector::::to_radians, &Scalar::to_radians, &|_| true, @@ -511,7 +605,12 @@ fn simd_max() { } fn simd_clamp() { + if cfg!(all(target_arch = "powerpc64", target_feature = "vsx")) { + // https://gitlab.com/qemu-project/qemu/-/issues/1780 + return; + } test_helpers::test_3(&|value: [Scalar; LANES], mut min: [Scalar; LANES], mut max: [Scalar; LANES]| { + use test_helpers::subnormals::flush_in; for (min, max) in min.iter_mut().zip(max.iter_mut()) { if max < min { core::mem::swap(min, max); @@ -528,8 +627,20 @@ fn simd_clamp() { for i in 0..LANES { result_scalar[i] = value[i].clamp(min[i], max[i]); } + let mut result_scalar_flush = [Scalar::default(); LANES]; + for i in 0..LANES { + // Comparisons flush-to-zero, but return value selection is _not_ flushed. + let mut value = value[i]; + if flush_in(value) < flush_in(min[i]) { + value = min[i]; + } + if flush_in(value) > flush_in(max[i]) { + value = max[i]; + } + result_scalar_flush[i] = value + } let result_vector = Vector::from_array(value).simd_clamp(min.into(), max.into()).to_array(); - test_helpers::prop_assert_biteq!(result_scalar, result_vector); + test_helpers::prop_assert_biteq!(result_vector, result_scalar, result_scalar_flush); Ok(()) }) } diff --git a/library/portable-simd/crates/core_simd/tests/pointers.rs b/library/portable-simd/crates/core_simd/tests/pointers.rs index 0ae8f83b8b9..a90ff928ced 100644 --- a/library/portable-simd/crates/core_simd/tests/pointers.rs +++ b/library/portable-simd/crates/core_simd/tests/pointers.rs @@ -1,6 +1,9 @@ #![feature(portable_simd, strict_provenance)] -use core_simd::simd::{Simd, SimdConstPtr, SimdMutPtr}; +use core_simd::simd::{ + ptr::{SimdConstPtr, SimdMutPtr}, + Simd, +}; macro_rules! common_tests { { $constness:ident } => { diff --git a/library/portable-simd/crates/core_simd/tests/round.rs b/library/portable-simd/crates/core_simd/tests/round.rs index aacf7bd3bcc..847766ec41e 100644 --- a/library/portable-simd/crates/core_simd/tests/round.rs +++ b/library/portable-simd/crates/core_simd/tests/round.rs @@ -43,7 +43,7 @@ fn trunc() { } fn fract() { - test_helpers::test_unary_elementwise( + test_helpers::test_unary_elementwise_flush_subnormals( &Vector::::fract, &Scalar::fract, &|_| true, @@ -53,7 +53,7 @@ fn fract() { test_helpers::test_lanes! { fn to_int_unchecked() { - use core_simd::simd::SimdFloat; + use core_simd::simd::num::SimdFloat; // The maximum integer that can be represented by the equivalently sized float has // all of the mantissa digits set to 1, pushed up to the MSB. const ALL_MANTISSA_BITS: IntScalar = ((1 << ::MANTISSA_DIGITS) - 1); diff --git a/library/portable-simd/crates/core_simd/tests/swizzle.rs b/library/portable-simd/crates/core_simd/tests/swizzle.rs index 8cd7c33e823..522d71439b7 100644 --- a/library/portable-simd/crates/core_simd/tests/swizzle.rs +++ b/library/portable-simd/crates/core_simd/tests/swizzle.rs @@ -11,10 +11,10 @@ #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] fn swizzle() { struct Index; - impl Swizzle<4, 4> for Index { + impl Swizzle<4> for Index { const INDEX: [usize; 4] = [2, 1, 3, 0]; } - impl Swizzle<4, 2> for Index { + impl Swizzle<2> for Index { const INDEX: [usize; 2] = [1, 1]; } @@ -34,18 +34,18 @@ fn reverse() { #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] fn rotate() { let a = Simd::from_array([1, 2, 3, 4]); - assert_eq!(a.rotate_lanes_left::<0>().to_array(), [1, 2, 3, 4]); - assert_eq!(a.rotate_lanes_left::<1>().to_array(), [2, 3, 4, 1]); - assert_eq!(a.rotate_lanes_left::<2>().to_array(), [3, 4, 1, 2]); - assert_eq!(a.rotate_lanes_left::<3>().to_array(), [4, 1, 2, 3]); - assert_eq!(a.rotate_lanes_left::<4>().to_array(), [1, 2, 3, 4]); - assert_eq!(a.rotate_lanes_left::<5>().to_array(), [2, 3, 4, 1]); - assert_eq!(a.rotate_lanes_right::<0>().to_array(), [1, 2, 3, 4]); - assert_eq!(a.rotate_lanes_right::<1>().to_array(), [4, 1, 2, 3]); - assert_eq!(a.rotate_lanes_right::<2>().to_array(), [3, 4, 1, 2]); - assert_eq!(a.rotate_lanes_right::<3>().to_array(), [2, 3, 4, 1]); - assert_eq!(a.rotate_lanes_right::<4>().to_array(), [1, 2, 3, 4]); - assert_eq!(a.rotate_lanes_right::<5>().to_array(), [4, 1, 2, 3]); + assert_eq!(a.rotate_elements_left::<0>().to_array(), [1, 2, 3, 4]); + assert_eq!(a.rotate_elements_left::<1>().to_array(), [2, 3, 4, 1]); + assert_eq!(a.rotate_elements_left::<2>().to_array(), [3, 4, 1, 2]); + assert_eq!(a.rotate_elements_left::<3>().to_array(), [4, 1, 2, 3]); + assert_eq!(a.rotate_elements_left::<4>().to_array(), [1, 2, 3, 4]); + assert_eq!(a.rotate_elements_left::<5>().to_array(), [2, 3, 4, 1]); + assert_eq!(a.rotate_elements_right::<0>().to_array(), [1, 2, 3, 4]); + assert_eq!(a.rotate_elements_right::<1>().to_array(), [4, 1, 2, 3]); + assert_eq!(a.rotate_elements_right::<2>().to_array(), [3, 4, 1, 2]); + assert_eq!(a.rotate_elements_right::<3>().to_array(), [2, 3, 4, 1]); + assert_eq!(a.rotate_elements_right::<4>().to_array(), [1, 2, 3, 4]); + assert_eq!(a.rotate_elements_right::<5>().to_array(), [4, 1, 2, 3]); } #[test] diff --git a/library/portable-simd/crates/core_simd/tests/swizzle_dyn.rs b/library/portable-simd/crates/core_simd/tests/swizzle_dyn.rs index 646cd5f3383..f21a937f01c 100644 --- a/library/portable-simd/crates/core_simd/tests/swizzle_dyn.rs +++ b/library/portable-simd/crates/core_simd/tests/swizzle_dyn.rs @@ -1,6 +1,5 @@ #![feature(portable_simd)] use core::{fmt, ops::RangeInclusive}; -use proptest; use test_helpers::{self, biteq, make_runner, prop_assert_biteq}; fn swizzle_dyn_scalar_ver(values: [u8; N], idxs: [u8; N]) -> [u8; N] { diff --git a/library/portable-simd/crates/core_simd/tests/to_bytes.rs b/library/portable-simd/crates/core_simd/tests/to_bytes.rs index be0ee4349c5..66a7981cdc3 100644 --- a/library/portable-simd/crates/core_simd/tests/to_bytes.rs +++ b/library/portable-simd/crates/core_simd/tests/to_bytes.rs @@ -1,14 +1,20 @@ -#![feature(portable_simd, generic_const_exprs, adt_const_params)] -#![allow(incomplete_features)] -#![cfg(feature = "generic_const_exprs")] +#![feature(portable_simd)] -use core_simd::simd::Simd; +use core_simd::simd::{Simd, ToBytes}; #[test] fn byte_convert() { let int = Simd::::from_array([0xdeadbeef, 0x8badf00d]); - let bytes = int.to_ne_bytes(); - assert_eq!(int[0].to_ne_bytes(), bytes[..4]); - assert_eq!(int[1].to_ne_bytes(), bytes[4..]); - assert_eq!(Simd::::from_ne_bytes(bytes), int); + let ne_bytes = int.to_ne_bytes(); + let be_bytes = int.to_be_bytes(); + let le_bytes = int.to_le_bytes(); + assert_eq!(int[0].to_ne_bytes(), ne_bytes[..4]); + assert_eq!(int[1].to_ne_bytes(), ne_bytes[4..]); + assert_eq!(int[0].to_be_bytes(), be_bytes[..4]); + assert_eq!(int[1].to_be_bytes(), be_bytes[4..]); + assert_eq!(int[0].to_le_bytes(), le_bytes[..4]); + assert_eq!(int[1].to_le_bytes(), le_bytes[4..]); + assert_eq!(Simd::::from_ne_bytes(ne_bytes), int); + assert_eq!(Simd::::from_be_bytes(be_bytes), int); + assert_eq!(Simd::::from_le_bytes(le_bytes), int); } diff --git a/library/portable-simd/crates/std_float/src/lib.rs b/library/portable-simd/crates/std_float/src/lib.rs index 4ac60b10c92..1fef17242ca 100644 --- a/library/portable-simd/crates/std_float/src/lib.rs +++ b/library/portable-simd/crates/std_float/src/lib.rs @@ -1,5 +1,10 @@ #![cfg_attr(feature = "as_crate", no_std)] // We are std! -#![cfg_attr(feature = "as_crate", feature(platform_intrinsics), feature(portable_simd))] +#![cfg_attr( + feature = "as_crate", + feature(platform_intrinsics), + feature(portable_simd), + allow(internal_features) +)] #[cfg(not(feature = "as_crate"))] use core::simd; #[cfg(feature = "as_crate")] @@ -144,7 +149,7 @@ fn fract(self) -> Self { #[cfg(test)] mod tests { use super::*; - use simd::*; + use simd::prelude::*; #[test] fn everything_works() { diff --git a/library/portable-simd/crates/test_helpers/Cargo.toml b/library/portable-simd/crates/test_helpers/Cargo.toml index 1d2bc8b519a..23dae7c9338 100644 --- a/library/portable-simd/crates/test_helpers/Cargo.toml +++ b/library/portable-simd/crates/test_helpers/Cargo.toml @@ -4,10 +4,8 @@ version = "0.1.0" edition = "2021" publish = false -[dependencies.proptest] -version = "0.10" -default-features = false -features = ["alloc"] +[dependencies] +proptest = { version = "0.10", default-features = false, features = ["alloc"] } [features] all_lane_counts = [] diff --git a/library/portable-simd/crates/test_helpers/src/biteq.rs b/library/portable-simd/crates/test_helpers/src/biteq.rs index 7d91260d838..cbc20cda0d6 100644 --- a/library/portable-simd/crates/test_helpers/src/biteq.rs +++ b/library/portable-simd/crates/test_helpers/src/biteq.rs @@ -113,6 +113,27 @@ fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { } } +#[doc(hidden)] +pub struct BitEqEitherWrapper<'a, T>(pub &'a T, pub &'a T); + +impl PartialEq> for BitEqWrapper<'_, T> { + fn eq(&self, other: &BitEqEitherWrapper<'_, T>) -> bool { + self.0.biteq(other.0) || self.0.biteq(other.1) + } +} + +impl core::fmt::Debug for BitEqEitherWrapper<'_, T> { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + if self.0.biteq(self.1) { + self.0.fmt(f) + } else { + self.0.fmt(f)?; + write!(f, " or ")?; + self.1.fmt(f) + } + } +} + #[macro_export] macro_rules! prop_assert_biteq { { $a:expr, $b:expr $(,)? } => { @@ -122,5 +143,14 @@ macro_rules! prop_assert_biteq { let b = $b; proptest::prop_assert_eq!(BitEqWrapper(&a), BitEqWrapper(&b)); } - } + }; + { $a:expr, $b:expr, $c:expr $(,)? } => { + { + use $crate::biteq::{BitEqWrapper, BitEqEitherWrapper}; + let a = $a; + let b = $b; + let c = $c; + proptest::prop_assert_eq!(BitEqWrapper(&a), BitEqEitherWrapper(&b, &c)); + } + }; } diff --git a/library/portable-simd/crates/test_helpers/src/lib.rs b/library/portable-simd/crates/test_helpers/src/lib.rs index b26cdc311a2..b80c745aaf2 100644 --- a/library/portable-simd/crates/test_helpers/src/lib.rs +++ b/library/portable-simd/crates/test_helpers/src/lib.rs @@ -1,3 +1,5 @@ +#![feature(stdsimd, powerpc_target_feature)] + pub mod array; #[cfg(target_arch = "wasm32")] @@ -6,6 +8,9 @@ #[macro_use] pub mod biteq; +pub mod subnormals; +use subnormals::FlushSubnormals; + /// Specifies the default strategy for testing a type. /// /// This strategy should be what "makes sense" to test. @@ -151,7 +156,6 @@ pub fn test_3< } /// Test a unary vector function against a unary scalar function, applied elementwise. -#[inline(never)] pub fn test_unary_elementwise( fv: &dyn Fn(Vector) -> VectorResult, fs: &dyn Fn(Scalar) -> ScalarResult, @@ -177,6 +181,48 @@ pub fn test_unary_elementwise( + fv: &dyn Fn(Vector) -> VectorResult, + fs: &dyn Fn(Scalar) -> ScalarResult, + check: &dyn Fn([Scalar; LANES]) -> bool, +) where + Scalar: Copy + core::fmt::Debug + DefaultStrategy + FlushSubnormals, + ScalarResult: Copy + biteq::BitEq + core::fmt::Debug + DefaultStrategy + FlushSubnormals, + Vector: Into<[Scalar; LANES]> + From<[Scalar; LANES]> + Copy, + VectorResult: Into<[ScalarResult; LANES]> + From<[ScalarResult; LANES]> + Copy, +{ + let flush = |x: Scalar| subnormals::flush(fs(subnormals::flush_in(x))); + test_1(&|x: [Scalar; LANES]| { + proptest::prop_assume!(check(x)); + let result_v: [ScalarResult; LANES] = fv(x.into()).into(); + let result_s: [ScalarResult; LANES] = x + .iter() + .copied() + .map(fs) + .collect::>() + .try_into() + .unwrap(); + let result_sf: [ScalarResult; LANES] = x + .iter() + .copied() + .map(flush) + .collect::>() + .try_into() + .unwrap(); + crate::prop_assert_biteq!(result_v, result_s, result_sf); + Ok(()) + }); +} + /// Test a unary vector function against a unary scalar function, applied elementwise. #[inline(never)] pub fn test_unary_mask_elementwise( @@ -204,7 +250,6 @@ pub fn test_unary_mask_elementwise( } /// Test a binary vector function against a binary scalar function, applied elementwise. -#[inline(never)] pub fn test_binary_elementwise< Scalar1, Scalar2, @@ -241,6 +286,85 @@ pub fn test_binary_elementwise< }); } +/// Test a binary vector function against a binary scalar function, applied elementwise. +/// +/// Where subnormals are flushed, use approximate equality. +pub fn test_binary_elementwise_flush_subnormals< + Scalar1, + Scalar2, + ScalarResult, + Vector1, + Vector2, + VectorResult, + const LANES: usize, +>( + fv: &dyn Fn(Vector1, Vector2) -> VectorResult, + fs: &dyn Fn(Scalar1, Scalar2) -> ScalarResult, + check: &dyn Fn([Scalar1; LANES], [Scalar2; LANES]) -> bool, +) where + Scalar1: Copy + core::fmt::Debug + DefaultStrategy + FlushSubnormals, + Scalar2: Copy + core::fmt::Debug + DefaultStrategy + FlushSubnormals, + ScalarResult: Copy + biteq::BitEq + core::fmt::Debug + DefaultStrategy + FlushSubnormals, + Vector1: Into<[Scalar1; LANES]> + From<[Scalar1; LANES]> + Copy, + Vector2: Into<[Scalar2; LANES]> + From<[Scalar2; LANES]> + Copy, + VectorResult: Into<[ScalarResult; LANES]> + From<[ScalarResult; LANES]> + Copy, +{ + let flush = |x: Scalar1, y: Scalar2| { + subnormals::flush(fs(subnormals::flush_in(x), subnormals::flush_in(y))) + }; + test_2(&|x: [Scalar1; LANES], y: [Scalar2; LANES]| { + proptest::prop_assume!(check(x, y)); + let result_v: [ScalarResult; LANES] = fv(x.into(), y.into()).into(); + let result_s: [ScalarResult; LANES] = x + .iter() + .copied() + .zip(y.iter().copied()) + .map(|(x, y)| fs(x, y)) + .collect::>() + .try_into() + .unwrap(); + let result_sf: [ScalarResult; LANES] = x + .iter() + .copied() + .zip(y.iter().copied()) + .map(|(x, y)| flush(x, y)) + .collect::>() + .try_into() + .unwrap(); + crate::prop_assert_biteq!(result_v, result_s, result_sf); + Ok(()) + }); +} + +/// Test a unary vector function against a unary scalar function, applied elementwise. +#[inline(never)] +pub fn test_binary_mask_elementwise( + fv: &dyn Fn(Vector1, Vector2) -> Mask, + fs: &dyn Fn(Scalar1, Scalar2) -> bool, + check: &dyn Fn([Scalar1; LANES], [Scalar2; LANES]) -> bool, +) where + Scalar1: Copy + core::fmt::Debug + DefaultStrategy, + Scalar2: Copy + core::fmt::Debug + DefaultStrategy, + Vector1: Into<[Scalar1; LANES]> + From<[Scalar1; LANES]> + Copy, + Vector2: Into<[Scalar2; LANES]> + From<[Scalar2; LANES]> + Copy, + Mask: Into<[bool; LANES]> + From<[bool; LANES]> + Copy, +{ + test_2(&|x: [Scalar1; LANES], y: [Scalar2; LANES]| { + proptest::prop_assume!(check(x, y)); + let result_v: [bool; LANES] = fv(x.into(), y.into()).into(); + let result_s: [bool; LANES] = x + .iter() + .copied() + .zip(y.iter().copied()) + .map(|(x, y)| fs(x, y)) + .collect::>() + .try_into() + .unwrap(); + crate::prop_assert_biteq!(result_v, result_s); + Ok(()) + }); +} + /// Test a binary vector-scalar function against a binary scalar function, applied elementwise. #[inline(never)] pub fn test_binary_scalar_rhs_elementwise< diff --git a/library/portable-simd/crates/test_helpers/src/subnormals.rs b/library/portable-simd/crates/test_helpers/src/subnormals.rs new file mode 100644 index 00000000000..ec0f1fb24b9 --- /dev/null +++ b/library/portable-simd/crates/test_helpers/src/subnormals.rs @@ -0,0 +1,91 @@ +pub trait FlushSubnormals: Sized { + fn flush(self) -> Self { + self + } +} + +impl FlushSubnormals for *const T {} +impl FlushSubnormals for *mut T {} + +macro_rules! impl_float { + { $($ty:ty),* } => { + $( + impl FlushSubnormals for $ty { + fn flush(self) -> Self { + let is_f32 = core::mem::size_of::() == 4; + let ppc_flush = is_f32 && cfg!(all( + any(target_arch = "powerpc", all(target_arch = "powerpc64", target_endian = "big")), + target_feature = "altivec", + not(target_feature = "vsx"), + )); + let arm_flush = is_f32 && cfg!(all(target_arch = "arm", target_feature = "neon")); + let flush = ppc_flush || arm_flush; + if flush && self.is_subnormal() { + <$ty>::copysign(0., self) + } else { + self + } + } + } + )* + } +} + +macro_rules! impl_else { + { $($ty:ty),* } => { + $( + impl FlushSubnormals for $ty {} + )* + } +} + +impl_float! { f32, f64 } +impl_else! { i8, i16, i32, i64, isize, u8, u16, u32, u64, usize } + +/// AltiVec should flush subnormal inputs to zero, but QEMU seems to only flush outputs. +/// https://gitlab.com/qemu-project/qemu/-/issues/1779 +#[cfg(all( + any(target_arch = "powerpc", target_arch = "powerpc64"), + target_feature = "altivec" +))] +fn in_buggy_qemu() -> bool { + use std::sync::OnceLock; + static BUGGY: OnceLock = OnceLock::new(); + + fn add(x: f32, y: f32) -> f32 { + #[cfg(target_arch = "powerpc")] + use core::arch::powerpc::*; + #[cfg(target_arch = "powerpc64")] + use core::arch::powerpc64::*; + + let array: [f32; 4] = + unsafe { core::mem::transmute(vec_add(vec_splats(x), vec_splats(y))) }; + array[0] + } + + *BUGGY.get_or_init(|| add(-1.0857398e-38, 0.).is_sign_negative()) +} + +#[cfg(all( + any(target_arch = "powerpc", target_arch = "powerpc64"), + target_feature = "altivec" +))] +pub fn flush_in(x: T) -> T { + if in_buggy_qemu() { + x + } else { + x.flush() + } +} + +#[cfg(not(all( + any(target_arch = "powerpc", target_arch = "powerpc64"), + target_feature = "altivec" +)))] +pub fn flush_in(x: T) -> T { + x.flush() +} + +pub fn flush(x: T) -> T { + x.flush() +}