From 681b1cd4b384f0deed32b44ae922bd5ceb1f3b78 Mon Sep 17 00:00:00 2001 From: Axel Gneiting Date: Thu, 27 Aug 2026 11:17:42 -0700 Subject: [PATCH 1/8] Add portable SIMD compress and expand operations ^ Conflicts: ^ fearless_simd_gen/src/mk_wasm.rs --- fearless_simd/src/generated/avx2.rs | 912 ++++++++++++++++++ fearless_simd/src/generated/avx512.rs | 215 +++++ fearless_simd/src/generated/fallback.rs | 141 +++ fearless_simd/src/generated/neon.rs | 388 ++++++++ fearless_simd/src/generated/simd_trait.rs | 54 ++ fearless_simd/src/generated/sse2.rs | 141 +++ fearless_simd/src/generated/sse4_2.rs | 815 ++++++++++++++++ fearless_simd/src/generated/wasm.rs | 352 +++++++ fearless_simd/src/support.rs | 197 ++++ fearless_simd_gen/src/generic.rs | 312 ++++++ fearless_simd_gen/src/mk_fallback.rs | 3 +- fearless_simd_gen/src/mk_neon.rs | 71 +- fearless_simd_gen/src/mk_wasm.rs | 17 +- fearless_simd_gen/src/mk_x86.rs | 671 ++++++++++++- fearless_simd_gen/src/ops.rs | 109 ++- .../tests/harness/ops/compress.rs | 62 ++ .../tests/harness/ops/expand.rs | 51 + fearless_simd_tests/tests/harness/ops/mod.rs | 2 + 18 files changed, 4501 insertions(+), 12 deletions(-) create mode 100644 fearless_simd_tests/tests/harness/ops/compress.rs create mode 100644 fearless_simd_tests/tests/harness/ops/expand.rs diff --git a/fearless_simd/src/generated/avx2.rs b/fearless_simd/src/generated/avx2.rs index 336edc192..e01c2d6ac 100644 --- a/fearless_simd/src/generated/avx2.rs +++ b/fearless_simd/src/generated/avx2.rs @@ -1845,6 +1845,121 @@ impl Simd for Avx2 { kernel(self, a, b, c) } #[inline(always)] + fn compress_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: u8x16, mask: mask8x16) -> u8x16 { + let mask_bits = token.to_bitmask_mask8x16(mask) as u16; + let low_mask = (usize::from(mask_bits)) & 0xff; + let high_mask = (usize::from(mask_bits)) >> 8; + let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask]; + let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask]; + let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + 0x0808_0808_0808_0808) + | (high_control & 0x8080_8080_8080_8080); + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let low_count = low_mask.count_ones() as usize; + let compacted = _mm_shuffle_epi8(values.into(), control); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count]; + let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let compressed = _mm_shuffle_epi8(compacted, splice); + compressed.simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_u8x16( + self, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + let mask_bits = token.to_bitmask_mask8x16(mask) as u16; + let low_mask = (usize::from(mask_bits)) & 0xff; + let high_mask = (usize::from(mask_bits)) >> 8; + let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask]; + let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask]; + let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + 0x0808_0808_0808_0808) + | (high_control & 0x8080_8080_8080_8080); + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let low_count = low_mask.count_ones() as usize; + let compacted = _mm_shuffle_epi8(values.into(), control); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count]; + let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let compressed = _mm_shuffle_epi8(compacted, splice); + let count = mask_bits.count_ones() as usize; + let prefix = unsafe { + _mm_load_si128( + core::ptr::from_ref(&crate::support::COMPACT_PREFIX_MASKS[count]) + .cast::<__m128i>(), + ) + }; + _mm_blendv_epi8(merge.into(), compressed, prefix).simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: u8x16, mask: mask8x16) -> u8x16 { + let mask_bits = token.to_bitmask_mask8x16(mask) as u16; + let low_mask = (usize::from(mask_bits)) & 0xff; + let high_mask = (usize::from(mask_bits)) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) + | (high_control & 0x8080_8080_8080_8080); + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let expanded = _mm_shuffle_epi8(values.into(), control); + expanded.simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_u8x16( + self, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + let mask_bits = token.to_bitmask_mask8x16(mask) as u16; + let low_mask = (usize::from(mask_bits)) & 0xff; + let high_mask = (usize::from(mask_bits)) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) + | (high_control & 0x8080_8080_8080_8080); + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let expanded = _mm_shuffle_epi8(values.into(), control); + _mm_blendv_epi8(merge.into(), expanded, mask.into()).simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn combine_u8x16(self, a: u8x16, b: u8x16) -> u8x32 { crate::kernel!( #[inline(always)] @@ -8287,6 +8402,263 @@ impl Simd for Avx2 { kernel(self, a, b, c) } #[inline(always)] + fn compress_u8x32(self, values: u8x32, mask: mask8x32) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: u8x32, mask: mask8x32) -> u8x32 { + let mask_bits = token.to_bitmask_mask8x32(mask); + let block_bits_0 = (mask_bits & 0xffff) as usize; + let low_mask_0 = (block_bits_0) & 0xff; + let high_mask_0 = (block_bits_0) >> 8; + let low_control_0 = crate::support::COMPRESS_8_CONTROLS[low_mask_0]; + let high_control_0 = crate::support::COMPRESS_8_CONTROLS[high_mask_0]; + let high_control_0 = ((high_control_0 & 0x7f7f_7f7f_7f7f_7f7f) + + 0x0808_0808_0808_0808) + | (high_control_0 & 0x8080_8080_8080_8080); + let control_0 = + _mm_set_epi64x(high_control_0.cast_signed(), low_control_0.cast_signed()); + let low_count_0 = low_mask_0.count_ones() as usize; + let block_bits_1 = (mask_bits >> 16 & 0xffff) as usize; + let low_mask_1 = (block_bits_1) & 0xff; + let high_mask_1 = (block_bits_1) >> 8; + let low_control_1 = crate::support::COMPRESS_8_CONTROLS[low_mask_1]; + let high_control_1 = crate::support::COMPRESS_8_CONTROLS[high_mask_1]; + let high_control_1 = ((high_control_1 & 0x7f7f_7f7f_7f7f_7f7f) + + 0x0808_0808_0808_0808) + | (high_control_1 & 0x8080_8080_8080_8080); + let control_1 = + _mm_set_epi64x(high_control_1.cast_signed(), low_control_1.cast_signed()); + let low_count_1 = low_mask_1.count_ones() as usize; + let control = _mm256_set_m128i(control_1, control_0); + let compacted = _mm256_shuffle_epi8(values.val.0, control); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count_0]; + let splice_0 = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count_1]; + let splice_1 = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let splice = _mm256_set_m128i(splice_1, splice_0); + let compressed = _mm256_shuffle_epi8(compacted, splice); + let splice = unsafe { + _mm256_load_si256( + core::ptr::from_ref( + &crate::support::COMPACT_16_SPLICE_CONTROLS + [block_bits_0.count_ones() as usize], + ) + .cast::<__m256i>(), + ) + }; + let compressed_low = _mm256_permute2x128_si256::<0x80>(compressed, compressed); + let compressed_high = _mm256_permute2x128_si256::<0x11>(compressed, compressed); + let compressed_0 = + _mm256_or_si256(compressed_low, _mm256_shuffle_epi8(compressed_high, splice)); + let result_0 = compressed_0; + result_0.simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_u8x32( + self, + values: u8x32, + mask: mask8x32, + merge: u8x32, + ) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: u8x32, + mask: mask8x32, + merge: u8x32, + ) -> u8x32 { + let mask_bits = token.to_bitmask_mask8x32(mask); + let block_bits_0 = (mask_bits & 0xffff) as usize; + let low_mask_0 = (block_bits_0) & 0xff; + let high_mask_0 = (block_bits_0) >> 8; + let low_control_0 = crate::support::COMPRESS_8_CONTROLS[low_mask_0]; + let high_control_0 = crate::support::COMPRESS_8_CONTROLS[high_mask_0]; + let high_control_0 = ((high_control_0 & 0x7f7f_7f7f_7f7f_7f7f) + + 0x0808_0808_0808_0808) + | (high_control_0 & 0x8080_8080_8080_8080); + let control_0 = + _mm_set_epi64x(high_control_0.cast_signed(), low_control_0.cast_signed()); + let low_count_0 = low_mask_0.count_ones() as usize; + let block_bits_1 = (mask_bits >> 16 & 0xffff) as usize; + let low_mask_1 = (block_bits_1) & 0xff; + let high_mask_1 = (block_bits_1) >> 8; + let low_control_1 = crate::support::COMPRESS_8_CONTROLS[low_mask_1]; + let high_control_1 = crate::support::COMPRESS_8_CONTROLS[high_mask_1]; + let high_control_1 = ((high_control_1 & 0x7f7f_7f7f_7f7f_7f7f) + + 0x0808_0808_0808_0808) + | (high_control_1 & 0x8080_8080_8080_8080); + let control_1 = + _mm_set_epi64x(high_control_1.cast_signed(), low_control_1.cast_signed()); + let low_count_1 = low_mask_1.count_ones() as usize; + let control = _mm256_set_m128i(control_1, control_0); + let compacted = _mm256_shuffle_epi8(values.val.0, control); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count_0]; + let splice_0 = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count_1]; + let splice_1 = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let splice = _mm256_set_m128i(splice_1, splice_0); + let compressed = _mm256_shuffle_epi8(compacted, splice); + let splice = unsafe { + _mm256_load_si256( + core::ptr::from_ref( + &crate::support::COMPACT_16_SPLICE_CONTROLS + [block_bits_0.count_ones() as usize], + ) + .cast::<__m256i>(), + ) + }; + let compressed_low = _mm256_permute2x128_si256::<0x80>(compressed, compressed); + let compressed_high = _mm256_permute2x128_si256::<0x11>(compressed, compressed); + let compressed_0 = + _mm256_or_si256(compressed_low, _mm256_shuffle_epi8(compressed_high, splice)); + let output_lane = (block_bits_0.count_ones() + block_bits_1.count_ones()) as usize; + let prefix = unsafe { + _mm256_load_si256( + core::ptr::from_ref( + &crate::support::COMPACT_PREFIX_MASKS[output_lane.min(32)], + ) + .cast::<__m256i>(), + ) + }; + let result_0 = _mm256_blendv_epi8(merge.val.0, compressed_0, prefix); + result_0.simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_u8x32(self, values: u8x32, mask: mask8x32) -> u8x32 { + #[inline] + #[target_feature(enable = "fxsr,sse4.2,cmpxchg16b,popcnt")] + unsafe fn kernel(token: Avx2, values: u8x32, mask: mask8x32) -> u8x32 { + let mask_bits = _mm_movemask_epi8(unsafe { + _mm_load_si128(core::ptr::from_ref(&mask.val.0).cast::<__m128i>().add(0)) + }) as u64 + | (_mm_movemask_epi8(unsafe { + _mm_load_si128(core::ptr::from_ref(&mask.val.0).cast::<__m128i>().add(1)) + }) as u64) + << 16; + let input_values = <[u8; 32]>::from(values); + let input = input_values.as_ptr(); + let mut output = [0u8; 32]; + let mut input_lane = 0usize; + let block_bits = (mask_bits & 0xffff) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) + | (high_control & 0x8080_8080_8080_8080); + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = unsafe { _mm_loadu_si128(input.add(input_lane).cast::<__m128i>()) }; + let expanded = _mm_shuffle_epi8(packed, control); + let result = expanded; + unsafe { + _mm_storeu_si128(output.as_mut_ptr().add(0).cast::<__m128i>(), result); + } + input_lane += block_bits.count_ones() as usize; + let block_bits = (mask_bits >> 16 & 0xffff) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) + | (high_control & 0x8080_8080_8080_8080); + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = unsafe { _mm_loadu_si128(input.add(input_lane).cast::<__m128i>()) }; + let expanded = _mm_shuffle_epi8(packed, control); + let result = expanded; + unsafe { + _mm_storeu_si128(output.as_mut_ptr().add(16).cast::<__m128i>(), result); + } + u8x32::simd_from(token, output) + } + unsafe { kernel(self, values, mask) } + } + #[inline(always)] + fn expand_merge_u8x32( + self, + values: u8x32, + mask: mask8x32, + merge: u8x32, + ) -> u8x32 { + #[inline] + #[target_feature(enable = "fxsr,sse4.2,cmpxchg16b,popcnt")] + unsafe fn kernel( + token: Avx2, + values: u8x32, + mask: mask8x32, + merge: u8x32, + ) -> u8x32 { + let mask_bits = _mm_movemask_epi8(unsafe { + _mm_load_si128(core::ptr::from_ref(&mask.val.0).cast::<__m128i>().add(0)) + }) as u64 + | (_mm_movemask_epi8(unsafe { + _mm_load_si128(core::ptr::from_ref(&mask.val.0).cast::<__m128i>().add(1)) + }) as u64) + << 16; + let input_values = <[u8; 32]>::from(values); + let input = input_values.as_ptr(); + let mut output = [0u8; 32]; + let mut input_lane = 0usize; + let block_bits = (mask_bits & 0xffff) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) + | (high_control & 0x8080_8080_8080_8080); + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = unsafe { _mm_loadu_si128(input.add(input_lane).cast::<__m128i>()) }; + let expanded = _mm_shuffle_epi8(packed, control); + let result = unsafe { + _mm_blendv_epi8( + _mm_load_si128(core::ptr::from_ref(&merge.val.0).cast::<__m128i>().add(0)), + expanded, + _mm_load_si128(core::ptr::from_ref(&mask.val.0).cast::<__m128i>().add(0)), + ) + }; + unsafe { + _mm_storeu_si128(output.as_mut_ptr().add(0).cast::<__m128i>(), result); + } + input_lane += block_bits.count_ones() as usize; + let block_bits = (mask_bits >> 16 & 0xffff) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) + | (high_control & 0x8080_8080_8080_8080); + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = unsafe { _mm_loadu_si128(input.add(input_lane).cast::<__m128i>()) }; + let expanded = _mm_shuffle_epi8(packed, control); + let result = unsafe { + _mm_blendv_epi8( + _mm_load_si128(core::ptr::from_ref(&merge.val.0).cast::<__m128i>().add(1)), + expanded, + _mm_load_si128(core::ptr::from_ref(&mask.val.0).cast::<__m128i>().add(1)), + ) + }; + unsafe { + _mm_storeu_si128(output.as_mut_ptr().add(16).cast::<__m128i>(), result); + } + u8x32::simd_from(token, output) + } + unsafe { kernel(self, values, mask, merge) } + } + #[inline(always)] fn combine_u8x32(self, a: u8x32, b: u8x32) -> u8x64 { u8x64 { val: crate::support::Aligned512([a.val.0, b.val.0]), @@ -13008,6 +13380,546 @@ impl Simd for Avx2 { kernel(self, a, b, indices) } #[inline(always)] + fn compress_u8x64(self, values: u8x64, mask: mask8x64) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: u8x64, mask: mask8x64) -> u8x64 { + let mask_bits = token.to_bitmask_mask8x64(mask); + if mask_bits == 0 || mask_bits == u64::MAX { + return if mask_bits == 0 { + u8x64::splat(token, 0) + } else { + values + }; + } + if mask_bits.is_power_of_two() { + let selected = values.as_array()[mask_bits.trailing_zeros() as usize]; + return { + let first = _mm256_set_epi64x(0, 0, 0, i64::from(selected)); + u8x64 { + val: crate::support::Aligned512([first, _mm256_setzero_si256()]), + simd: token, + } + }; + } + let block_bits_0 = (mask_bits & 0xffff) as usize; + let low_mask_0 = (block_bits_0) & 0xff; + let high_mask_0 = (block_bits_0) >> 8; + let low_control_0 = crate::support::COMPRESS_8_CONTROLS[low_mask_0]; + let high_control_0 = crate::support::COMPRESS_8_CONTROLS[high_mask_0]; + let high_control_0 = ((high_control_0 & 0x7f7f_7f7f_7f7f_7f7f) + + 0x0808_0808_0808_0808) + | (high_control_0 & 0x8080_8080_8080_8080); + let control_0 = + _mm_set_epi64x(high_control_0.cast_signed(), low_control_0.cast_signed()); + let low_count_0 = low_mask_0.count_ones() as usize; + let block_bits_1 = (mask_bits >> 16 & 0xffff) as usize; + let low_mask_1 = (block_bits_1) & 0xff; + let high_mask_1 = (block_bits_1) >> 8; + let low_control_1 = crate::support::COMPRESS_8_CONTROLS[low_mask_1]; + let high_control_1 = crate::support::COMPRESS_8_CONTROLS[high_mask_1]; + let high_control_1 = ((high_control_1 & 0x7f7f_7f7f_7f7f_7f7f) + + 0x0808_0808_0808_0808) + | (high_control_1 & 0x8080_8080_8080_8080); + let control_1 = + _mm_set_epi64x(high_control_1.cast_signed(), low_control_1.cast_signed()); + let low_count_1 = low_mask_1.count_ones() as usize; + let control = _mm256_set_m128i(control_1, control_0); + let compacted = _mm256_shuffle_epi8(values.val.0[0], control); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count_0]; + let splice_0 = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count_1]; + let splice_1 = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let splice = _mm256_set_m128i(splice_1, splice_0); + let compressed = _mm256_shuffle_epi8(compacted, splice); + let splice = unsafe { + _mm256_load_si256( + core::ptr::from_ref( + &crate::support::COMPACT_16_SPLICE_CONTROLS + [block_bits_0.count_ones() as usize], + ) + .cast::<__m256i>(), + ) + }; + let compressed_low = _mm256_permute2x128_si256::<0x80>(compressed, compressed); + let compressed_high = _mm256_permute2x128_si256::<0x11>(compressed, compressed); + let compressed_0 = + _mm256_or_si256(compressed_low, _mm256_shuffle_epi8(compressed_high, splice)); + let block_bits_2 = (mask_bits >> 32 & 0xffff) as usize; + let low_mask_2 = (block_bits_2) & 0xff; + let high_mask_2 = (block_bits_2) >> 8; + let low_control_2 = crate::support::COMPRESS_8_CONTROLS[low_mask_2]; + let high_control_2 = crate::support::COMPRESS_8_CONTROLS[high_mask_2]; + let high_control_2 = ((high_control_2 & 0x7f7f_7f7f_7f7f_7f7f) + + 0x0808_0808_0808_0808) + | (high_control_2 & 0x8080_8080_8080_8080); + let control_2 = + _mm_set_epi64x(high_control_2.cast_signed(), low_control_2.cast_signed()); + let low_count_2 = low_mask_2.count_ones() as usize; + let block_bits_3 = (mask_bits >> 48 & 0xffff) as usize; + let low_mask_3 = (block_bits_3) & 0xff; + let high_mask_3 = (block_bits_3) >> 8; + let low_control_3 = crate::support::COMPRESS_8_CONTROLS[low_mask_3]; + let high_control_3 = crate::support::COMPRESS_8_CONTROLS[high_mask_3]; + let high_control_3 = ((high_control_3 & 0x7f7f_7f7f_7f7f_7f7f) + + 0x0808_0808_0808_0808) + | (high_control_3 & 0x8080_8080_8080_8080); + let control_3 = + _mm_set_epi64x(high_control_3.cast_signed(), low_control_3.cast_signed()); + let low_count_3 = low_mask_3.count_ones() as usize; + let control = _mm256_set_m128i(control_3, control_2); + let compacted = _mm256_shuffle_epi8(values.val.0[1], control); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count_2]; + let splice_2 = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count_3]; + let splice_3 = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let splice = _mm256_set_m128i(splice_3, splice_2); + let compressed = _mm256_shuffle_epi8(compacted, splice); + let splice = unsafe { + _mm256_load_si256( + core::ptr::from_ref( + &crate::support::COMPACT_16_SPLICE_CONTROLS + [block_bits_2.count_ones() as usize], + ) + .cast::<__m256i>(), + ) + }; + let compressed_low = _mm256_permute2x128_si256::<0x80>(compressed, compressed); + let compressed_high = _mm256_permute2x128_si256::<0x11>(compressed, compressed); + let compressed_1 = + _mm256_or_si256(compressed_low, _mm256_shuffle_epi8(compressed_high, splice)); + let first_count = (block_bits_0.count_ones() + block_bits_1.count_ones()) as usize; + let controls = &crate::support::COMPACT_32_STITCH_CONTROLS[first_count]; + let left_same = unsafe { + _mm256_load_si256(core::ptr::from_ref(&controls.left_same).cast::<__m256i>()) + }; + let left_cross = unsafe { + _mm256_load_si256(core::ptr::from_ref(&controls.left_cross).cast::<__m256i>()) + }; + let right_same = unsafe { + _mm256_load_si256(core::ptr::from_ref(&controls.right_same).cast::<__m256i>()) + }; + let right_cross = unsafe { + _mm256_load_si256(core::ptr::from_ref(&controls.right_cross).cast::<__m256i>()) + }; + let low_to_high = _mm256_permute2x128_si256::<0x08>(compressed_1, compressed_1); + let shifted_left = _mm256_or_si256( + _mm256_shuffle_epi8(compressed_1, left_same), + _mm256_shuffle_epi8(low_to_high, left_cross), + ); + let compressed_0 = _mm256_or_si256(compressed_0, shifted_left); + let high_to_low = _mm256_permute2x128_si256::<0x81>(compressed_1, compressed_1); + let compressed_1 = _mm256_or_si256( + _mm256_shuffle_epi8(compressed_1, right_same), + _mm256_shuffle_epi8(high_to_low, right_cross), + ); + let result_0 = compressed_0; + let result_1 = compressed_1; + u8x64 { + val: crate::support::Aligned512([result_0, result_1]), + simd: token, + } + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_u8x64( + self, + values: u8x64, + mask: mask8x64, + merge: u8x64, + ) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: u8x64, + mask: mask8x64, + merge: u8x64, + ) -> u8x64 { + let mask_bits = token.to_bitmask_mask8x64(mask); + if mask_bits == 0 || mask_bits == u64::MAX { + return if mask_bits == 0 { merge } else { values }; + } + if mask_bits.is_power_of_two() { + let selected = values.as_array()[mask_bits.trailing_zeros() as usize]; + return { + let low = _mm_insert_epi8::<0>( + _mm256_castsi256_si128(merge.val.0[0]), + i32::from(selected), + ); + let first = _mm256_inserti128_si256::<0>(merge.val.0[0], low); + u8x64 { + val: crate::support::Aligned512([first, merge.val.0[1]]), + simd: token, + } + }; + } + let block_bits_0 = (mask_bits & 0xffff) as usize; + let low_mask_0 = (block_bits_0) & 0xff; + let high_mask_0 = (block_bits_0) >> 8; + let low_control_0 = crate::support::COMPRESS_8_CONTROLS[low_mask_0]; + let high_control_0 = crate::support::COMPRESS_8_CONTROLS[high_mask_0]; + let high_control_0 = ((high_control_0 & 0x7f7f_7f7f_7f7f_7f7f) + + 0x0808_0808_0808_0808) + | (high_control_0 & 0x8080_8080_8080_8080); + let control_0 = + _mm_set_epi64x(high_control_0.cast_signed(), low_control_0.cast_signed()); + let low_count_0 = low_mask_0.count_ones() as usize; + let block_bits_1 = (mask_bits >> 16 & 0xffff) as usize; + let low_mask_1 = (block_bits_1) & 0xff; + let high_mask_1 = (block_bits_1) >> 8; + let low_control_1 = crate::support::COMPRESS_8_CONTROLS[low_mask_1]; + let high_control_1 = crate::support::COMPRESS_8_CONTROLS[high_mask_1]; + let high_control_1 = ((high_control_1 & 0x7f7f_7f7f_7f7f_7f7f) + + 0x0808_0808_0808_0808) + | (high_control_1 & 0x8080_8080_8080_8080); + let control_1 = + _mm_set_epi64x(high_control_1.cast_signed(), low_control_1.cast_signed()); + let low_count_1 = low_mask_1.count_ones() as usize; + let control = _mm256_set_m128i(control_1, control_0); + let compacted = _mm256_shuffle_epi8(values.val.0[0], control); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count_0]; + let splice_0 = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count_1]; + let splice_1 = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let splice = _mm256_set_m128i(splice_1, splice_0); + let compressed = _mm256_shuffle_epi8(compacted, splice); + let splice = unsafe { + _mm256_load_si256( + core::ptr::from_ref( + &crate::support::COMPACT_16_SPLICE_CONTROLS + [block_bits_0.count_ones() as usize], + ) + .cast::<__m256i>(), + ) + }; + let compressed_low = _mm256_permute2x128_si256::<0x80>(compressed, compressed); + let compressed_high = _mm256_permute2x128_si256::<0x11>(compressed, compressed); + let compressed_0 = + _mm256_or_si256(compressed_low, _mm256_shuffle_epi8(compressed_high, splice)); + let block_bits_2 = (mask_bits >> 32 & 0xffff) as usize; + let low_mask_2 = (block_bits_2) & 0xff; + let high_mask_2 = (block_bits_2) >> 8; + let low_control_2 = crate::support::COMPRESS_8_CONTROLS[low_mask_2]; + let high_control_2 = crate::support::COMPRESS_8_CONTROLS[high_mask_2]; + let high_control_2 = ((high_control_2 & 0x7f7f_7f7f_7f7f_7f7f) + + 0x0808_0808_0808_0808) + | (high_control_2 & 0x8080_8080_8080_8080); + let control_2 = + _mm_set_epi64x(high_control_2.cast_signed(), low_control_2.cast_signed()); + let low_count_2 = low_mask_2.count_ones() as usize; + let block_bits_3 = (mask_bits >> 48 & 0xffff) as usize; + let low_mask_3 = (block_bits_3) & 0xff; + let high_mask_3 = (block_bits_3) >> 8; + let low_control_3 = crate::support::COMPRESS_8_CONTROLS[low_mask_3]; + let high_control_3 = crate::support::COMPRESS_8_CONTROLS[high_mask_3]; + let high_control_3 = ((high_control_3 & 0x7f7f_7f7f_7f7f_7f7f) + + 0x0808_0808_0808_0808) + | (high_control_3 & 0x8080_8080_8080_8080); + let control_3 = + _mm_set_epi64x(high_control_3.cast_signed(), low_control_3.cast_signed()); + let low_count_3 = low_mask_3.count_ones() as usize; + let control = _mm256_set_m128i(control_3, control_2); + let compacted = _mm256_shuffle_epi8(values.val.0[1], control); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count_2]; + let splice_2 = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count_3]; + let splice_3 = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let splice = _mm256_set_m128i(splice_3, splice_2); + let compressed = _mm256_shuffle_epi8(compacted, splice); + let splice = unsafe { + _mm256_load_si256( + core::ptr::from_ref( + &crate::support::COMPACT_16_SPLICE_CONTROLS + [block_bits_2.count_ones() as usize], + ) + .cast::<__m256i>(), + ) + }; + let compressed_low = _mm256_permute2x128_si256::<0x80>(compressed, compressed); + let compressed_high = _mm256_permute2x128_si256::<0x11>(compressed, compressed); + let compressed_1 = + _mm256_or_si256(compressed_low, _mm256_shuffle_epi8(compressed_high, splice)); + let first_count = (block_bits_0.count_ones() + block_bits_1.count_ones()) as usize; + let controls = &crate::support::COMPACT_32_STITCH_CONTROLS[first_count]; + let left_same = unsafe { + _mm256_load_si256(core::ptr::from_ref(&controls.left_same).cast::<__m256i>()) + }; + let left_cross = unsafe { + _mm256_load_si256(core::ptr::from_ref(&controls.left_cross).cast::<__m256i>()) + }; + let right_same = unsafe { + _mm256_load_si256(core::ptr::from_ref(&controls.right_same).cast::<__m256i>()) + }; + let right_cross = unsafe { + _mm256_load_si256(core::ptr::from_ref(&controls.right_cross).cast::<__m256i>()) + }; + let low_to_high = _mm256_permute2x128_si256::<0x08>(compressed_1, compressed_1); + let shifted_left = _mm256_or_si256( + _mm256_shuffle_epi8(compressed_1, left_same), + _mm256_shuffle_epi8(low_to_high, left_cross), + ); + let compressed_0 = _mm256_or_si256(compressed_0, shifted_left); + let high_to_low = _mm256_permute2x128_si256::<0x81>(compressed_1, compressed_1); + let compressed_1 = _mm256_or_si256( + _mm256_shuffle_epi8(compressed_1, right_same), + _mm256_shuffle_epi8(high_to_low, right_cross), + ); + let output_lane = mask_bits.count_ones() as usize; + let prefix = unsafe { + _mm256_load_si256( + core::ptr::from_ref( + &crate::support::COMPACT_PREFIX_MASKS[output_lane.min(32)], + ) + .cast::<__m256i>(), + ) + }; + let result_0 = _mm256_blendv_epi8(merge.val.0[0], compressed_0, prefix); + let prefix = unsafe { + _mm256_load_si256( + core::ptr::from_ref( + &crate::support::COMPACT_PREFIX_MASKS + [output_lane.saturating_sub(32).min(32)], + ) + .cast::<__m256i>(), + ) + }; + let result_1 = _mm256_blendv_epi8(merge.val.0[1], compressed_1, prefix); + u8x64 { + val: crate::support::Aligned512([result_0, result_1]), + simd: token, + } + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_u8x64(self, values: u8x64, mask: mask8x64) -> u8x64 { + #[inline] + #[target_feature(enable = "fxsr,sse4.2,cmpxchg16b,popcnt")] + unsafe fn kernel(token: Avx2, values: u8x64, mask: mask8x64) -> u8x64 { + let mask_bits = _mm_movemask_epi8(unsafe { + _mm_load_si128(core::ptr::from_ref(&mask.val.0).cast::<__m128i>().add(0)) + }) as u64 + | (_mm_movemask_epi8(unsafe { + _mm_load_si128(core::ptr::from_ref(&mask.val.0).cast::<__m128i>().add(1)) + }) as u64) + << 16 + | (_mm_movemask_epi8(unsafe { + _mm_load_si128(core::ptr::from_ref(&mask.val.0).cast::<__m128i>().add(2)) + }) as u64) + << 32 + | (_mm_movemask_epi8(unsafe { + _mm_load_si128(core::ptr::from_ref(&mask.val.0).cast::<__m128i>().add(3)) + }) as u64) + << 48; + let input_values = <[u8; 64]>::from(values); + let input = input_values.as_ptr(); + let mut output = [0u8; 64]; + let mut input_lane = 0usize; + let block_bits = (mask_bits & 0xffff) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) + | (high_control & 0x8080_8080_8080_8080); + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = unsafe { _mm_loadu_si128(input.add(input_lane).cast::<__m128i>()) }; + let expanded = _mm_shuffle_epi8(packed, control); + let result = expanded; + unsafe { + _mm_storeu_si128(output.as_mut_ptr().add(0).cast::<__m128i>(), result); + } + input_lane += block_bits.count_ones() as usize; + let block_bits = (mask_bits >> 16 & 0xffff) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) + | (high_control & 0x8080_8080_8080_8080); + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = unsafe { _mm_loadu_si128(input.add(input_lane).cast::<__m128i>()) }; + let expanded = _mm_shuffle_epi8(packed, control); + let result = expanded; + unsafe { + _mm_storeu_si128(output.as_mut_ptr().add(16).cast::<__m128i>(), result); + } + input_lane += block_bits.count_ones() as usize; + let block_bits = (mask_bits >> 32 & 0xffff) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) + | (high_control & 0x8080_8080_8080_8080); + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = unsafe { _mm_loadu_si128(input.add(input_lane).cast::<__m128i>()) }; + let expanded = _mm_shuffle_epi8(packed, control); + let result = expanded; + unsafe { + _mm_storeu_si128(output.as_mut_ptr().add(32).cast::<__m128i>(), result); + } + input_lane += block_bits.count_ones() as usize; + let block_bits = (mask_bits >> 48 & 0xffff) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) + | (high_control & 0x8080_8080_8080_8080); + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = unsafe { _mm_loadu_si128(input.add(input_lane).cast::<__m128i>()) }; + let expanded = _mm_shuffle_epi8(packed, control); + let result = expanded; + unsafe { + _mm_storeu_si128(output.as_mut_ptr().add(48).cast::<__m128i>(), result); + } + u8x64::simd_from(token, output) + } + unsafe { kernel(self, values, mask) } + } + #[inline(always)] + fn expand_merge_u8x64( + self, + values: u8x64, + mask: mask8x64, + merge: u8x64, + ) -> u8x64 { + #[inline] + #[target_feature(enable = "fxsr,sse4.2,cmpxchg16b,popcnt")] + unsafe fn kernel( + token: Avx2, + values: u8x64, + mask: mask8x64, + merge: u8x64, + ) -> u8x64 { + let mask_bits = _mm_movemask_epi8(unsafe { + _mm_load_si128(core::ptr::from_ref(&mask.val.0).cast::<__m128i>().add(0)) + }) as u64 + | (_mm_movemask_epi8(unsafe { + _mm_load_si128(core::ptr::from_ref(&mask.val.0).cast::<__m128i>().add(1)) + }) as u64) + << 16 + | (_mm_movemask_epi8(unsafe { + _mm_load_si128(core::ptr::from_ref(&mask.val.0).cast::<__m128i>().add(2)) + }) as u64) + << 32 + | (_mm_movemask_epi8(unsafe { + _mm_load_si128(core::ptr::from_ref(&mask.val.0).cast::<__m128i>().add(3)) + }) as u64) + << 48; + let input_values = <[u8; 64]>::from(values); + let input = input_values.as_ptr(); + let mut output = [0u8; 64]; + let mut input_lane = 0usize; + let block_bits = (mask_bits & 0xffff) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) + | (high_control & 0x8080_8080_8080_8080); + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = unsafe { _mm_loadu_si128(input.add(input_lane).cast::<__m128i>()) }; + let expanded = _mm_shuffle_epi8(packed, control); + let result = unsafe { + _mm_blendv_epi8( + _mm_load_si128(core::ptr::from_ref(&merge.val.0).cast::<__m128i>().add(0)), + expanded, + _mm_load_si128(core::ptr::from_ref(&mask.val.0).cast::<__m128i>().add(0)), + ) + }; + unsafe { + _mm_storeu_si128(output.as_mut_ptr().add(0).cast::<__m128i>(), result); + } + input_lane += block_bits.count_ones() as usize; + let block_bits = (mask_bits >> 16 & 0xffff) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) + | (high_control & 0x8080_8080_8080_8080); + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = unsafe { _mm_loadu_si128(input.add(input_lane).cast::<__m128i>()) }; + let expanded = _mm_shuffle_epi8(packed, control); + let result = unsafe { + _mm_blendv_epi8( + _mm_load_si128(core::ptr::from_ref(&merge.val.0).cast::<__m128i>().add(1)), + expanded, + _mm_load_si128(core::ptr::from_ref(&mask.val.0).cast::<__m128i>().add(1)), + ) + }; + unsafe { + _mm_storeu_si128(output.as_mut_ptr().add(16).cast::<__m128i>(), result); + } + input_lane += block_bits.count_ones() as usize; + let block_bits = (mask_bits >> 32 & 0xffff) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) + | (high_control & 0x8080_8080_8080_8080); + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = unsafe { _mm_loadu_si128(input.add(input_lane).cast::<__m128i>()) }; + let expanded = _mm_shuffle_epi8(packed, control); + let result = unsafe { + _mm_blendv_epi8( + _mm_load_si128(core::ptr::from_ref(&merge.val.0).cast::<__m128i>().add(2)), + expanded, + _mm_load_si128(core::ptr::from_ref(&mask.val.0).cast::<__m128i>().add(2)), + ) + }; + unsafe { + _mm_storeu_si128(output.as_mut_ptr().add(32).cast::<__m128i>(), result); + } + input_lane += block_bits.count_ones() as usize; + let block_bits = (mask_bits >> 48 & 0xffff) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) + | (high_control & 0x8080_8080_8080_8080); + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = unsafe { _mm_loadu_si128(input.add(input_lane).cast::<__m128i>()) }; + let expanded = _mm_shuffle_epi8(packed, control); + let result = unsafe { + _mm_blendv_epi8( + _mm_load_si128(core::ptr::from_ref(&merge.val.0).cast::<__m128i>().add(3)), + expanded, + _mm_load_si128(core::ptr::from_ref(&mask.val.0).cast::<__m128i>().add(3)), + ) + }; + unsafe { + _mm_storeu_si128(output.as_mut_ptr().add(48).cast::<__m128i>(), result); + } + u8x64::simd_from(token, output) + } + unsafe { kernel(self, values, mask, merge) } + } + #[inline(always)] fn split_u8x64(self, a: u8x64) -> (u8x32, u8x32) { ( u8x32 { diff --git a/fearless_simd/src/generated/avx512.rs b/fearless_simd/src/generated/avx512.rs index 8092b7ec2..a7cdcf9d6 100644 --- a/fearless_simd/src/generated/avx512.rs +++ b/fearless_simd/src/generated/avx512.rs @@ -2135,6 +2135,77 @@ impl Simd for Avx512 { kernel(self, a, b, c) } #[inline(always)] + fn compress_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u8x16, + mask: mask8x16, + ) -> u8x16 { + _mm_maskz_compress_epi8(u64::from((mask).val) as u16, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_u8x16( + self, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + _mm_mask_compress_epi8(merge.into(), u64::from((mask).val) as u16, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u8x16, + mask: mask8x16, + ) -> u8x16 { + _mm_maskz_expand_epi8(u64::from((mask).val) as u16, values.into()).simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_u8x16( + self, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + _mm_mask_expand_epi8(merge.into(), u64::from((mask).val) as u16, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn combine_u8x16(self, a: u8x16, b: u8x16) -> u8x32 { crate::kernel!( #[inline(always)] @@ -8184,6 +8255,78 @@ impl Simd for Avx512 { kernel(self, a, b, c) } #[inline(always)] + fn compress_u8x32(self, values: u8x32, mask: mask8x32) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u8x32, + mask: mask8x32, + ) -> u8x32 { + _mm256_maskz_compress_epi8(u64::from((mask).val) as u32, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_u8x32( + self, + values: u8x32, + mask: mask8x32, + merge: u8x32, + ) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u8x32, + mask: mask8x32, + merge: u8x32, + ) -> u8x32 { + _mm256_mask_compress_epi8(merge.into(), u64::from((mask).val) as u32, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_u8x32(self, values: u8x32, mask: mask8x32) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u8x32, + mask: mask8x32, + ) -> u8x32 { + _mm256_maskz_expand_epi8(u64::from((mask).val) as u32, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_u8x32( + self, + values: u8x32, + mask: mask8x32, + merge: u8x32, + ) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u8x32, + mask: mask8x32, + merge: u8x32, + ) -> u8x32 { + _mm256_mask_expand_epi8(merge.into(), u64::from((mask).val) as u32, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn combine_u8x32(self, a: u8x32, b: u8x32) -> u8x64 { crate::kernel!( #[inline(always)] @@ -14143,6 +14286,78 @@ impl Simd for Avx512 { kernel(self, a, b, c) } #[inline(always)] + fn compress_u8x64(self, values: u8x64, mask: mask8x64) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u8x64, + mask: mask8x64, + ) -> u8x64 { + _mm512_maskz_compress_epi8(u64::from((mask).val) as u64, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_u8x64( + self, + values: u8x64, + mask: mask8x64, + merge: u8x64, + ) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u8x64, + mask: mask8x64, + merge: u8x64, + ) -> u8x64 { + _mm512_mask_compress_epi8(merge.into(), u64::from((mask).val) as u64, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_u8x64(self, values: u8x64, mask: mask8x64) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u8x64, + mask: mask8x64, + ) -> u8x64 { + _mm512_maskz_expand_epi8(u64::from((mask).val) as u64, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_u8x64( + self, + values: u8x64, + mask: mask8x64, + merge: u8x64, + ) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u8x64, + mask: mask8x64, + merge: u8x64, + ) -> u8x64 { + _mm512_mask_expand_epi8(merge.into(), u64::from((mask).val) as u64, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn split_u8x64(self, a: u8x64) -> (u8x32, u8x32) { crate::kernel!( #[inline(always)] diff --git a/fearless_simd/src/generated/fallback.rs b/fearless_simd/src/generated/fallback.rs index 2aec237c3..b26da3e8f 100644 --- a/fearless_simd/src/generated/fallback.rs +++ b/fearless_simd/src/generated/fallback.rs @@ -2318,6 +2318,53 @@ impl Simd for Fallback { .simd_into(self) } #[inline(always)] + fn compress_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16 { + self.compress_merge_u8x16(values, mask, u8x16::splat(self, 0)) + } + #[inline(always)] + fn compress_merge_u8x16( + self, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + let mask = self.to_bitmask_mask8x16(mask); + let mut merge_padded = [0u8; 16 + 1]; + merge_padded[..16].copy_from_slice(&*merge); + let mut compacted = merge_padded; + let mut output_lane = 0; + for input_lane in 0..16 { + compacted[output_lane] = values[input_lane]; + output_lane += ((mask >> input_lane) & 1) as usize; + } + compacted[output_lane] = merge_padded[output_lane]; + let mut result = merge; + result.copy_from_slice(&compacted[..16]); + result + } + #[inline(always)] + fn expand_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16 { + self.expand_merge_u8x16(values, mask, u8x16::splat(self, 0)) + } + #[inline(always)] + fn expand_merge_u8x16( + self, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + let mask = self.to_bitmask_mask8x16(mask); + let mut result = merge; + let mut input_lane = 0; + for output_lane in 0..16 { + let bit = ((mask >> output_lane) & 1) as u8; + let keep = 0u8.wrapping_sub(bit); + result[output_lane] = (values[input_lane] & keep) | (result[output_lane] & !keep); + input_lane += usize::from(bit); + } + result + } + #[inline(always)] fn combine_u8x16(self, a: u8x16, b: u8x16) -> u8x32 { let mut result = [0; 32usize]; result[0..16usize].copy_from_slice(&a.val.0); @@ -6749,6 +6796,53 @@ impl Simd for Fallback { Bytes::from_bytes(result) } #[inline(always)] + fn compress_u8x32(self, values: u8x32, mask: mask8x32) -> u8x32 { + self.compress_merge_u8x32(values, mask, u8x32::splat(self, 0)) + } + #[inline(always)] + fn compress_merge_u8x32( + self, + values: u8x32, + mask: mask8x32, + merge: u8x32, + ) -> u8x32 { + let mask = self.to_bitmask_mask8x32(mask); + let mut merge_padded = [0u8; 32 + 1]; + merge_padded[..32].copy_from_slice(&*merge); + let mut compacted = merge_padded; + let mut output_lane = 0; + for input_lane in 0..32 { + compacted[output_lane] = values[input_lane]; + output_lane += ((mask >> input_lane) & 1) as usize; + } + compacted[output_lane] = merge_padded[output_lane]; + let mut result = merge; + result.copy_from_slice(&compacted[..32]); + result + } + #[inline(always)] + fn expand_u8x32(self, values: u8x32, mask: mask8x32) -> u8x32 { + self.expand_merge_u8x32(values, mask, u8x32::splat(self, 0)) + } + #[inline(always)] + fn expand_merge_u8x32( + self, + values: u8x32, + mask: mask8x32, + merge: u8x32, + ) -> u8x32 { + let mask = self.to_bitmask_mask8x32(mask); + let mut result = merge; + let mut input_lane = 0; + for output_lane in 0..32 { + let bit = ((mask >> output_lane) & 1) as u8; + let keep = 0u8.wrapping_sub(bit); + result[output_lane] = (values[input_lane] & keep) | (result[output_lane] & !keep); + input_lane += usize::from(bit); + } + result + } + #[inline(always)] fn combine_u8x32(self, a: u8x32, b: u8x32) -> u8x64 { let mut result = [0; 64usize]; result[0..32usize].copy_from_slice(&a.val.0); @@ -7310,6 +7404,53 @@ impl Simd for Fallback { Bytes::from_bytes(result) } #[inline(always)] + fn compress_u8x64(self, values: u8x64, mask: mask8x64) -> u8x64 { + self.compress_merge_u8x64(values, mask, u8x64::splat(self, 0)) + } + #[inline(always)] + fn compress_merge_u8x64( + self, + values: u8x64, + mask: mask8x64, + merge: u8x64, + ) -> u8x64 { + let mask = self.to_bitmask_mask8x64(mask); + let mut merge_padded = [0u8; 64 + 1]; + merge_padded[..64].copy_from_slice(&*merge); + let mut compacted = merge_padded; + let mut output_lane = 0; + for input_lane in 0..64 { + compacted[output_lane] = values[input_lane]; + output_lane += ((mask >> input_lane) & 1) as usize; + } + compacted[output_lane] = merge_padded[output_lane]; + let mut result = merge; + result.copy_from_slice(&compacted[..64]); + result + } + #[inline(always)] + fn expand_u8x64(self, values: u8x64, mask: mask8x64) -> u8x64 { + self.expand_merge_u8x64(values, mask, u8x64::splat(self, 0)) + } + #[inline(always)] + fn expand_merge_u8x64( + self, + values: u8x64, + mask: mask8x64, + merge: u8x64, + ) -> u8x64 { + let mask = self.to_bitmask_mask8x64(mask); + let mut result = merge; + let mut input_lane = 0; + for output_lane in 0..64 { + let bit = ((mask >> output_lane) & 1) as u8; + let keep = 0u8.wrapping_sub(bit); + result[output_lane] = (values[input_lane] & keep) | (result[output_lane] & !keep); + input_lane += usize::from(bit); + } + result + } + #[inline(always)] fn split_u8x64(self, a: u8x64) -> (u8x32, u8x32) { let mut b0 = [0; 32usize]; let mut b1 = [0; 32usize]; diff --git a/fearless_simd/src/generated/neon.rs b/fearless_simd/src/generated/neon.rs index e964bc515..ef02ec3fd 100644 --- a/fearless_simd/src/generated/neon.rs +++ b/fearless_simd/src/generated/neon.rs @@ -1434,6 +1434,130 @@ impl Simd for Neon { kernel(self, a, b, c) } #[inline(always)] + fn compress_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16 { + let mask_bits = self.to_bitmask_mask8x16(mask); + let mut control = [u8::MAX; 16]; + let mut output_lane = 0; + for block in 0..16 / 8 { + let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; + let packed = crate::support::COMPRESS_8_CONTROLS[block_mask]; + let base = (block * 8) as u64 * 0x0101_0101_0101_0101; + let adjusted = + ((packed & 0x7f7f_7f7f_7f7f_7f7f) + base) | (packed & 0x8080_8080_8080_8080); + let adjusted = adjusted.to_le_bytes(); + let write_len = core::cmp::min(8, 16 - output_lane); + control[output_lane..output_lane + write_len].copy_from_slice(&adjusted[..write_len]); + output_lane += crate::support::COMPACT_8_COUNTS[block_mask] as usize; + } + let control = u8x16::simd_from(self, control); + self.swizzle_dyn_precise_u8x16(values, control) + } + #[inline(always)] + fn compress_merge_u8x16( + self, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + let mask_bits = self.to_bitmask_mask8x16(mask); + let mut control = [u8::MAX; 16]; + let mut output_lane = 0; + for block in 0..16 / 8 { + let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; + let packed = crate::support::COMPRESS_8_CONTROLS[block_mask]; + let base = (block * 8) as u64 * 0x0101_0101_0101_0101; + let adjusted = + ((packed & 0x7f7f_7f7f_7f7f_7f7f) + base) | (packed & 0x8080_8080_8080_8080); + let adjusted = adjusted.to_le_bytes(); + let write_len = core::cmp::min(8, 16 - output_lane); + control[output_lane..output_lane + write_len].copy_from_slice(&adjusted[..write_len]); + output_lane += crate::support::COMPACT_8_COUNTS[block_mask] as usize; + } + let control = u8x16::simd_from(self, control); + { + crate::kernel!( + #[inline(always)] + fn merge_swizzle( + token: Neon, + values: u8x16, + control: u8x16, + merge: u8x16, + ) -> u8x16 { + let values: uint8x16_t = values.into(); + let control: uint8x16_t = control.into(); + let merge: uint8x16_t = merge.into(); + let result = vqtbx1q_u8(merge, values, control); + u8x16 { + val: crate::support::Aligned128(result), + simd: token, + } + } + ); + merge_swizzle(self, values, control, merge) + } + } + #[inline(always)] + fn expand_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16 { + let mask_bits = self.to_bitmask_mask8x16(mask); + let mut control = [u8::MAX; 16]; + let mut input_lane = 0; + for block in 0..16 / 8 { + let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; + let packed = crate::support::EXPAND_8_CONTROLS[block_mask]; + let base = input_lane as u64 * 0x0101_0101_0101_0101; + let adjusted = + ((packed & 0x7f7f_7f7f_7f7f_7f7f) + base) | (packed & 0x8080_8080_8080_8080); + let output_lane = block * 8; + control[output_lane..output_lane + 8].copy_from_slice(&adjusted.to_le_bytes()); + input_lane += crate::support::COMPACT_8_COUNTS[block_mask] as usize; + } + let control = u8x16::simd_from(self, control); + self.swizzle_dyn_precise_u8x16(values, control) + } + #[inline(always)] + fn expand_merge_u8x16( + self, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + let mask_bits = self.to_bitmask_mask8x16(mask); + let mut control = [u8::MAX; 16]; + let mut input_lane = 0; + for block in 0..16 / 8 { + let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; + let packed = crate::support::EXPAND_8_CONTROLS[block_mask]; + let base = input_lane as u64 * 0x0101_0101_0101_0101; + let adjusted = + ((packed & 0x7f7f_7f7f_7f7f_7f7f) + base) | (packed & 0x8080_8080_8080_8080); + let output_lane = block * 8; + control[output_lane..output_lane + 8].copy_from_slice(&adjusted.to_le_bytes()); + input_lane += crate::support::COMPACT_8_COUNTS[block_mask] as usize; + } + let control = u8x16::simd_from(self, control); + { + crate::kernel!( + #[inline(always)] + fn merge_swizzle( + token: Neon, + values: u8x16, + control: u8x16, + merge: u8x16, + ) -> u8x16 { + let values: uint8x16_t = values.into(); + let control: uint8x16_t = control.into(); + let merge: uint8x16_t = merge.into(); + let result = vqtbx1q_u8(merge, values, control); + u8x16 { + val: crate::support::Aligned128(result), + simd: token, + } + } + ); + merge_swizzle(self, values, control, merge) + } + } + #[inline(always)] fn combine_u8x16(self, a: u8x16, b: u8x16) -> u8x32 { u8x32 { val: crate::support::Aligned256(uint8x16x2_t(a.val.0, b.val.0)), @@ -5672,6 +5796,136 @@ impl Simd for Neon { kernel(self, a, b, indices) } #[inline(always)] + fn compress_u8x32(self, values: u8x32, mask: mask8x32) -> u8x32 { + let mask_bits = self.to_bitmask_mask8x32(mask); + let mut control = [u8::MAX; 32]; + let mut output_lane = 0; + for block in 0..32 / 8 { + let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; + let packed = crate::support::COMPRESS_8_CONTROLS[block_mask]; + let base = (block * 8) as u64 * 0x0101_0101_0101_0101; + let adjusted = + ((packed & 0x7f7f_7f7f_7f7f_7f7f) + base) | (packed & 0x8080_8080_8080_8080); + let adjusted = adjusted.to_le_bytes(); + let write_len = core::cmp::min(8, 32 - output_lane); + control[output_lane..output_lane + write_len].copy_from_slice(&adjusted[..write_len]); + output_lane += crate::support::COMPACT_8_COUNTS[block_mask] as usize; + } + let control = u8x32::simd_from(self, control); + self.swizzle_dyn_precise_u8x32(values, control) + } + #[inline(always)] + fn compress_merge_u8x32( + self, + values: u8x32, + mask: mask8x32, + merge: u8x32, + ) -> u8x32 { + let mask_bits = self.to_bitmask_mask8x32(mask); + let mut control = [u8::MAX; 32]; + let mut output_lane = 0; + for block in 0..32 / 8 { + let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; + let packed = crate::support::COMPRESS_8_CONTROLS[block_mask]; + let base = (block * 8) as u64 * 0x0101_0101_0101_0101; + let adjusted = + ((packed & 0x7f7f_7f7f_7f7f_7f7f) + base) | (packed & 0x8080_8080_8080_8080); + let adjusted = adjusted.to_le_bytes(); + let write_len = core::cmp::min(8, 32 - output_lane); + control[output_lane..output_lane + write_len].copy_from_slice(&adjusted[..write_len]); + output_lane += crate::support::COMPACT_8_COUNTS[block_mask] as usize; + } + let control = u8x32::simd_from(self, control); + { + crate::kernel!( + #[inline(always)] + fn merge_swizzle( + token: Neon, + values: u8x32, + control: u8x32, + merge: u8x32, + ) -> u8x32 { + let values: uint8x16x2_t = values.into(); + let control: uint8x16x2_t = control.into(); + let merge: uint8x16x2_t = merge.into(); + let result = uint8x16x2_t( + vqtbx2q_u8(merge.0, values, control.0), + vqtbx2q_u8(merge.1, values, control.1), + ); + u8x32 { + val: crate::support::Aligned256(result), + simd: token, + } + } + ); + merge_swizzle(self, values, control, merge) + } + } + #[inline(always)] + fn expand_u8x32(self, values: u8x32, mask: mask8x32) -> u8x32 { + let mask_bits = self.to_bitmask_mask8x32(mask); + let mut control = [u8::MAX; 32]; + let mut input_lane = 0; + for block in 0..32 / 8 { + let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; + let packed = crate::support::EXPAND_8_CONTROLS[block_mask]; + let base = input_lane as u64 * 0x0101_0101_0101_0101; + let adjusted = + ((packed & 0x7f7f_7f7f_7f7f_7f7f) + base) | (packed & 0x8080_8080_8080_8080); + let output_lane = block * 8; + control[output_lane..output_lane + 8].copy_from_slice(&adjusted.to_le_bytes()); + input_lane += crate::support::COMPACT_8_COUNTS[block_mask] as usize; + } + let control = u8x32::simd_from(self, control); + self.swizzle_dyn_precise_u8x32(values, control) + } + #[inline(always)] + fn expand_merge_u8x32( + self, + values: u8x32, + mask: mask8x32, + merge: u8x32, + ) -> u8x32 { + let mask_bits = self.to_bitmask_mask8x32(mask); + let mut control = [u8::MAX; 32]; + let mut input_lane = 0; + for block in 0..32 / 8 { + let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; + let packed = crate::support::EXPAND_8_CONTROLS[block_mask]; + let base = input_lane as u64 * 0x0101_0101_0101_0101; + let adjusted = + ((packed & 0x7f7f_7f7f_7f7f_7f7f) + base) | (packed & 0x8080_8080_8080_8080); + let output_lane = block * 8; + control[output_lane..output_lane + 8].copy_from_slice(&adjusted.to_le_bytes()); + input_lane += crate::support::COMPACT_8_COUNTS[block_mask] as usize; + } + let control = u8x32::simd_from(self, control); + { + crate::kernel!( + #[inline(always)] + fn merge_swizzle( + token: Neon, + values: u8x32, + control: u8x32, + merge: u8x32, + ) -> u8x32 { + let values: uint8x16x2_t = values.into(); + let control: uint8x16x2_t = control.into(); + let merge: uint8x16x2_t = merge.into(); + let result = uint8x16x2_t( + vqtbx2q_u8(merge.0, values, control.0), + vqtbx2q_u8(merge.1, values, control.1), + ); + u8x32 { + val: crate::support::Aligned256(result), + simd: token, + } + } + ); + merge_swizzle(self, values, control, merge) + } + } + #[inline(always)] fn combine_u8x32(self, a: u8x32, b: u8x32) -> u8x64 { u8x64 { val: crate::support::Aligned512(uint8x16x4_t( @@ -6748,6 +7002,140 @@ impl Simd for Neon { kernel(self, a, b, indices) } #[inline(always)] + fn compress_u8x64(self, values: u8x64, mask: mask8x64) -> u8x64 { + let mask_bits = self.to_bitmask_mask8x64(mask); + let mut control = [u8::MAX; 64]; + let mut output_lane = 0; + for block in 0..64 / 8 { + let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; + let packed = crate::support::COMPRESS_8_CONTROLS[block_mask]; + let base = (block * 8) as u64 * 0x0101_0101_0101_0101; + let adjusted = + ((packed & 0x7f7f_7f7f_7f7f_7f7f) + base) | (packed & 0x8080_8080_8080_8080); + let adjusted = adjusted.to_le_bytes(); + let write_len = core::cmp::min(8, 64 - output_lane); + control[output_lane..output_lane + write_len].copy_from_slice(&adjusted[..write_len]); + output_lane += crate::support::COMPACT_8_COUNTS[block_mask] as usize; + } + let control = u8x64::simd_from(self, control); + self.swizzle_dyn_precise_u8x64(values, control) + } + #[inline(always)] + fn compress_merge_u8x64( + self, + values: u8x64, + mask: mask8x64, + merge: u8x64, + ) -> u8x64 { + let mask_bits = self.to_bitmask_mask8x64(mask); + let mut control = [u8::MAX; 64]; + let mut output_lane = 0; + for block in 0..64 / 8 { + let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; + let packed = crate::support::COMPRESS_8_CONTROLS[block_mask]; + let base = (block * 8) as u64 * 0x0101_0101_0101_0101; + let adjusted = + ((packed & 0x7f7f_7f7f_7f7f_7f7f) + base) | (packed & 0x8080_8080_8080_8080); + let adjusted = adjusted.to_le_bytes(); + let write_len = core::cmp::min(8, 64 - output_lane); + control[output_lane..output_lane + write_len].copy_from_slice(&adjusted[..write_len]); + output_lane += crate::support::COMPACT_8_COUNTS[block_mask] as usize; + } + let control = u8x64::simd_from(self, control); + { + crate::kernel!( + #[inline(always)] + fn merge_swizzle( + token: Neon, + values: u8x64, + control: u8x64, + merge: u8x64, + ) -> u8x64 { + let values: uint8x16x4_t = values.into(); + let control: uint8x16x4_t = control.into(); + let merge: uint8x16x4_t = merge.into(); + let result = uint8x16x4_t( + vqtbx4q_u8(merge.0, values, control.0), + vqtbx4q_u8(merge.1, values, control.1), + vqtbx4q_u8(merge.2, values, control.2), + vqtbx4q_u8(merge.3, values, control.3), + ); + u8x64 { + val: crate::support::Aligned512(result), + simd: token, + } + } + ); + merge_swizzle(self, values, control, merge) + } + } + #[inline(always)] + fn expand_u8x64(self, values: u8x64, mask: mask8x64) -> u8x64 { + let mask_bits = self.to_bitmask_mask8x64(mask); + let mut control = [u8::MAX; 64]; + let mut input_lane = 0; + for block in 0..64 / 8 { + let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; + let packed = crate::support::EXPAND_8_CONTROLS[block_mask]; + let base = input_lane as u64 * 0x0101_0101_0101_0101; + let adjusted = + ((packed & 0x7f7f_7f7f_7f7f_7f7f) + base) | (packed & 0x8080_8080_8080_8080); + let output_lane = block * 8; + control[output_lane..output_lane + 8].copy_from_slice(&adjusted.to_le_bytes()); + input_lane += crate::support::COMPACT_8_COUNTS[block_mask] as usize; + } + let control = u8x64::simd_from(self, control); + self.swizzle_dyn_precise_u8x64(values, control) + } + #[inline(always)] + fn expand_merge_u8x64( + self, + values: u8x64, + mask: mask8x64, + merge: u8x64, + ) -> u8x64 { + let mask_bits = self.to_bitmask_mask8x64(mask); + let mut control = [u8::MAX; 64]; + let mut input_lane = 0; + for block in 0..64 / 8 { + let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; + let packed = crate::support::EXPAND_8_CONTROLS[block_mask]; + let base = input_lane as u64 * 0x0101_0101_0101_0101; + let adjusted = + ((packed & 0x7f7f_7f7f_7f7f_7f7f) + base) | (packed & 0x8080_8080_8080_8080); + let output_lane = block * 8; + control[output_lane..output_lane + 8].copy_from_slice(&adjusted.to_le_bytes()); + input_lane += crate::support::COMPACT_8_COUNTS[block_mask] as usize; + } + let control = u8x64::simd_from(self, control); + { + crate::kernel!( + #[inline(always)] + fn merge_swizzle( + token: Neon, + values: u8x64, + control: u8x64, + merge: u8x64, + ) -> u8x64 { + let values: uint8x16x4_t = values.into(); + let control: uint8x16x4_t = control.into(); + let merge: uint8x16x4_t = merge.into(); + let result = uint8x16x4_t( + vqtbx4q_u8(merge.0, values, control.0), + vqtbx4q_u8(merge.1, values, control.1), + vqtbx4q_u8(merge.2, values, control.2), + vqtbx4q_u8(merge.3, values, control.3), + ); + u8x64 { + val: crate::support::Aligned512(result), + simd: token, + } + } + ); + merge_swizzle(self, values, control, merge) + } + } + #[inline(always)] fn split_u8x64(self, a: u8x64) -> (u8x32, u8x32) { ( u8x32 { diff --git a/fearless_simd/src/generated/simd_trait.rs b/fearless_simd/src/generated/simd_trait.rs index bdf3cfef7..b7cf8949e 100644 --- a/fearless_simd/src/generated/simd_trait.rs +++ b/fearless_simd/src/generated/simd_trait.rs @@ -674,6 +674,24 @@ pub trait Simd: fn deinterleave_u8x16(self, a: u8x16, b: u8x16) -> (u8x16, u8x16); #[doc = "Select elements from b and c based on the mask operand a.\n\nThis operation's behavior is unspecified if a was constructed from signed integer lanes that are neither all-zeroes (integer value 0) nor all-ones (integer value -1). See the [`Select`] trait's documentation for more information."] fn select_u8x16(self, a: mask8x16, b: u8x16, c: u8x16) -> u8x16; + #[doc = "Compact the bytes selected by `mask` into consecutive low lanes.\n\nLanes above the number of selected bytes are zero."] + fn compress_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16; + #[doc = "Compact the bytes selected by `mask` into consecutive low lanes.\n\nLanes above the number of selected bytes retain the corresponding values from `merge`."] + fn compress_merge_u8x16( + self, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16; + #[doc = "Expand consecutive low bytes from `values` into the lanes selected by `mask`.\n\nUnselected lanes are zero."] + fn expand_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16; + #[doc = "Expand consecutive low bytes from `values` into the lanes selected by `mask`.\n\nUnselected lanes retain the corresponding values from `merge`."] + fn expand_merge_u8x16( + self, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16; #[doc = "Combine two vectors into a single vector with twice the width.\n\n`a` provides the lower elements and `b` provides the upper elements."] fn combine_u8x16(self, a: u8x16, b: u8x16) -> u8x32; #[doc = "Load four 128-bit vectors from an array with 4-way interleaving.\n\nThis is useful e.g. in image processing to turn interleaved RGBA pixels into vectors of each color component.\n\nFor example, with 32-bit lanes, memory laid out as`[r0, g0, b0, a0, r1, g1, b1, a1, r2, g2, b2, a2, r3, g3, b3, a3]` loads as`[[r0, r1, r2, r3], [g0, g1, g2, g3], [b0, b1, b2, b3], [a0, a1, a2, a3]]`."] @@ -3068,6 +3086,24 @@ pub trait Simd: let (c0, c1) = self.split_u8x32(c); self.combine_u8x16(self.select_u8x16(a0, b0, c0), self.select_u8x16(a1, b1, c1)) } + #[doc = "Compact the bytes selected by `mask` into consecutive low lanes.\n\nLanes above the number of selected bytes are zero."] + fn compress_u8x32(self, values: u8x32, mask: mask8x32) -> u8x32; + #[doc = "Compact the bytes selected by `mask` into consecutive low lanes.\n\nLanes above the number of selected bytes retain the corresponding values from `merge`."] + fn compress_merge_u8x32( + self, + values: u8x32, + mask: mask8x32, + merge: u8x32, + ) -> u8x32; + #[doc = "Expand consecutive low bytes from `values` into the lanes selected by `mask`.\n\nUnselected lanes are zero."] + fn expand_u8x32(self, values: u8x32, mask: mask8x32) -> u8x32; + #[doc = "Expand consecutive low bytes from `values` into the lanes selected by `mask`.\n\nUnselected lanes retain the corresponding values from `merge`."] + fn expand_merge_u8x32( + self, + values: u8x32, + mask: mask8x32, + merge: u8x32, + ) -> u8x32; #[doc = "Combine two vectors into a single vector with twice the width.\n\n`a` provides the lower elements and `b` provides the upper elements."] fn combine_u8x32(self, a: u8x32, b: u8x32) -> u8x64; #[doc = "Split a vector into two vectors of half the width.\n\nReturns a tuple of (lower half, upper half)."] @@ -7288,6 +7324,24 @@ pub trait Simd: let (c0, c1) = self.split_u8x64(c); self.combine_u8x32(self.select_u8x32(a0, b0, c0), self.select_u8x32(a1, b1, c1)) } + #[doc = "Compact the bytes selected by `mask` into consecutive low lanes.\n\nLanes above the number of selected bytes are zero."] + fn compress_u8x64(self, values: u8x64, mask: mask8x64) -> u8x64; + #[doc = "Compact the bytes selected by `mask` into consecutive low lanes.\n\nLanes above the number of selected bytes retain the corresponding values from `merge`."] + fn compress_merge_u8x64( + self, + values: u8x64, + mask: mask8x64, + merge: u8x64, + ) -> u8x64; + #[doc = "Expand consecutive low bytes from `values` into the lanes selected by `mask`.\n\nUnselected lanes are zero."] + fn expand_u8x64(self, values: u8x64, mask: mask8x64) -> u8x64; + #[doc = "Expand consecutive low bytes from `values` into the lanes selected by `mask`.\n\nUnselected lanes retain the corresponding values from `merge`."] + fn expand_merge_u8x64( + self, + values: u8x64, + mask: mask8x64, + merge: u8x64, + ) -> u8x64; #[doc = "Split a vector into two vectors of half the width.\n\nReturns a tuple of (lower half, upper half)."] fn split_u8x64(self, a: u8x64) -> (u8x32, u8x32); #[doc = "Widen every lane into two same-width vectors.\n\nThe first result contains the widened lower lanes and the second contains the widened upper lanes."] diff --git a/fearless_simd/src/generated/sse2.rs b/fearless_simd/src/generated/sse2.rs index 1231fdb59..d747af81c 100644 --- a/fearless_simd/src/generated/sse2.rs +++ b/fearless_simd/src/generated/sse2.rs @@ -2096,6 +2096,53 @@ impl Simd for Sse2 { kernel(self, a, b, c) } #[inline(always)] + fn compress_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16 { + self.compress_merge_u8x16(values, mask, u8x16::splat(self, 0)) + } + #[inline(always)] + fn compress_merge_u8x16( + self, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + let mask = self.to_bitmask_mask8x16(mask); + let mut merge_padded = [0u8; 16 + 1]; + merge_padded[..16].copy_from_slice(&*merge); + let mut compacted = merge_padded; + let mut output_lane = 0; + for input_lane in 0..16 { + compacted[output_lane] = values[input_lane]; + output_lane += ((mask >> input_lane) & 1) as usize; + } + compacted[output_lane] = merge_padded[output_lane]; + let mut result = merge; + result.copy_from_slice(&compacted[..16]); + result + } + #[inline(always)] + fn expand_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16 { + self.expand_merge_u8x16(values, mask, u8x16::splat(self, 0)) + } + #[inline(always)] + fn expand_merge_u8x16( + self, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + let mask = self.to_bitmask_mask8x16(mask); + let mut result = merge; + let mut input_lane = 0; + for output_lane in 0..16 { + let bit = ((mask >> output_lane) & 1) as u8; + let keep = 0u8.wrapping_sub(bit); + result[output_lane] = (values[input_lane] & keep) | (result[output_lane] & !keep); + input_lane += usize::from(bit); + } + result + } + #[inline(always)] fn combine_u8x16(self, a: u8x16, b: u8x16) -> u8x32 { u8x32 { val: crate::support::Aligned256([a.val.0, b.val.0]), @@ -7137,6 +7184,53 @@ impl Simd for Sse2 { Bytes::from_bytes(result) } #[inline(always)] + fn compress_u8x32(self, values: u8x32, mask: mask8x32) -> u8x32 { + self.compress_merge_u8x32(values, mask, u8x32::splat(self, 0)) + } + #[inline(always)] + fn compress_merge_u8x32( + self, + values: u8x32, + mask: mask8x32, + merge: u8x32, + ) -> u8x32 { + let mask = self.to_bitmask_mask8x32(mask); + let mut merge_padded = [0u8; 32 + 1]; + merge_padded[..32].copy_from_slice(&*merge); + let mut compacted = merge_padded; + let mut output_lane = 0; + for input_lane in 0..32 { + compacted[output_lane] = values[input_lane]; + output_lane += ((mask >> input_lane) & 1) as usize; + } + compacted[output_lane] = merge_padded[output_lane]; + let mut result = merge; + result.copy_from_slice(&compacted[..32]); + result + } + #[inline(always)] + fn expand_u8x32(self, values: u8x32, mask: mask8x32) -> u8x32 { + self.expand_merge_u8x32(values, mask, u8x32::splat(self, 0)) + } + #[inline(always)] + fn expand_merge_u8x32( + self, + values: u8x32, + mask: mask8x32, + merge: u8x32, + ) -> u8x32 { + let mask = self.to_bitmask_mask8x32(mask); + let mut result = merge; + let mut input_lane = 0; + for output_lane in 0..32 { + let bit = ((mask >> output_lane) & 1) as u8; + let keep = 0u8.wrapping_sub(bit); + result[output_lane] = (values[input_lane] & keep) | (result[output_lane] & !keep); + input_lane += usize::from(bit); + } + result + } + #[inline(always)] fn combine_u8x32(self, a: u8x32, b: u8x32) -> u8x64 { u8x64 { val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), @@ -7871,6 +7965,53 @@ impl Simd for Sse2 { Bytes::from_bytes(result) } #[inline(always)] + fn compress_u8x64(self, values: u8x64, mask: mask8x64) -> u8x64 { + self.compress_merge_u8x64(values, mask, u8x64::splat(self, 0)) + } + #[inline(always)] + fn compress_merge_u8x64( + self, + values: u8x64, + mask: mask8x64, + merge: u8x64, + ) -> u8x64 { + let mask = self.to_bitmask_mask8x64(mask); + let mut merge_padded = [0u8; 64 + 1]; + merge_padded[..64].copy_from_slice(&*merge); + let mut compacted = merge_padded; + let mut output_lane = 0; + for input_lane in 0..64 { + compacted[output_lane] = values[input_lane]; + output_lane += ((mask >> input_lane) & 1) as usize; + } + compacted[output_lane] = merge_padded[output_lane]; + let mut result = merge; + result.copy_from_slice(&compacted[..64]); + result + } + #[inline(always)] + fn expand_u8x64(self, values: u8x64, mask: mask8x64) -> u8x64 { + self.expand_merge_u8x64(values, mask, u8x64::splat(self, 0)) + } + #[inline(always)] + fn expand_merge_u8x64( + self, + values: u8x64, + mask: mask8x64, + merge: u8x64, + ) -> u8x64 { + let mask = self.to_bitmask_mask8x64(mask); + let mut result = merge; + let mut input_lane = 0; + for output_lane in 0..64 { + let bit = ((mask >> output_lane) & 1) as u8; + let keep = 0u8.wrapping_sub(bit); + result[output_lane] = (values[input_lane] & keep) | (result[output_lane] & !keep); + input_lane += usize::from(bit); + } + result + } + #[inline(always)] fn split_u8x64(self, a: u8x64) -> (u8x32, u8x32) { ( u8x32 { diff --git a/fearless_simd/src/generated/sse4_2.rs b/fearless_simd/src/generated/sse4_2.rs index 159b686df..8bca0c15f 100644 --- a/fearless_simd/src/generated/sse4_2.rs +++ b/fearless_simd/src/generated/sse4_2.rs @@ -1843,6 +1843,129 @@ impl Simd for Sse4_2 { kernel(self, a, b, c) } #[inline(always)] + fn compress_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Sse4_2, + values: u8x16, + mask: mask8x16, + ) -> u8x16 { + let mask_bits = token.to_bitmask_mask8x16(mask) as u16; + let low_mask = (usize::from(mask_bits)) & 0xff; + let high_mask = (usize::from(mask_bits)) >> 8; + let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask]; + let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask]; + let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + 0x0808_0808_0808_0808) + | (high_control & 0x8080_8080_8080_8080); + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let low_count = low_mask.count_ones() as usize; + let compacted = _mm_shuffle_epi8(values.into(), control); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count]; + let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let compressed = _mm_shuffle_epi8(compacted, splice); + compressed.simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_u8x16( + self, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Sse4_2, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + let mask_bits = token.to_bitmask_mask8x16(mask) as u16; + let low_mask = (usize::from(mask_bits)) & 0xff; + let high_mask = (usize::from(mask_bits)) >> 8; + let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask]; + let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask]; + let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + 0x0808_0808_0808_0808) + | (high_control & 0x8080_8080_8080_8080); + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let low_count = low_mask.count_ones() as usize; + let compacted = _mm_shuffle_epi8(values.into(), control); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count]; + let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let compressed = _mm_shuffle_epi8(compacted, splice); + let count = mask_bits.count_ones() as usize; + let prefix = unsafe { + _mm_load_si128( + core::ptr::from_ref(&crate::support::COMPACT_PREFIX_MASKS[count]) + .cast::<__m128i>(), + ) + }; + _mm_blendv_epi8(merge.into(), compressed, prefix).simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Sse4_2, + values: u8x16, + mask: mask8x16, + ) -> u8x16 { + let mask_bits = token.to_bitmask_mask8x16(mask) as u16; + let low_mask = (usize::from(mask_bits)) & 0xff; + let high_mask = (usize::from(mask_bits)) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) + | (high_control & 0x8080_8080_8080_8080); + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let expanded = _mm_shuffle_epi8(values.into(), control); + expanded.simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_u8x16( + self, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Sse4_2, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + let mask_bits = token.to_bitmask_mask8x16(mask) as u16; + let low_mask = (usize::from(mask_bits)) & 0xff; + let high_mask = (usize::from(mask_bits)) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) + | (high_control & 0x8080_8080_8080_8080); + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let expanded = _mm_shuffle_epi8(values.into(), control); + _mm_blendv_epi8(merge.into(), expanded, mask.into()).simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn combine_u8x16(self, a: u8x16, b: u8x16) -> u8x32 { u8x32 { val: crate::support::Aligned256([a.val.0, b.val.0]), @@ -6754,6 +6877,263 @@ impl Simd for Sse4_2 { Bytes::from_bytes(result_bytes) } #[inline(always)] + fn compress_u8x32(self, values: u8x32, mask: mask8x32) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Sse4_2, + values: u8x32, + mask: mask8x32, + ) -> u8x32 { + let mask_bits = token.to_bitmask_mask8x32(mask); + let mut output = [0u8; 32]; + let mut output_lane = 0; + let block_bits = (mask_bits & 0xffff) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask]; + let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask]; + let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + 0x0808_0808_0808_0808) + | (high_control & 0x8080_8080_8080_8080); + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let low_count = low_mask.count_ones() as usize; + let compacted = _mm_shuffle_epi8(values.val.0[0], control); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count]; + let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let compressed = _mm_shuffle_epi8(compacted, splice); + unsafe { + _mm_storeu_si128( + output.as_mut_ptr().add(output_lane).cast::<__m128i>(), + compressed, + ); + } + output_lane += block_bits.count_ones() as usize; + let block_bits = (mask_bits >> 16 & 0xffff) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask]; + let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask]; + let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + 0x0808_0808_0808_0808) + | (high_control & 0x8080_8080_8080_8080); + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let low_count = low_mask.count_ones() as usize; + let compacted = _mm_shuffle_epi8(values.val.0[1], control); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count]; + let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let compressed = _mm_shuffle_epi8(compacted, splice); + unsafe { + _mm_storeu_si128( + output.as_mut_ptr().add(output_lane).cast::<__m128i>(), + compressed, + ); + } + u8x32::simd_from(token, output) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_u8x32( + self, + values: u8x32, + mask: mask8x32, + merge: u8x32, + ) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Sse4_2, + values: u8x32, + mask: mask8x32, + merge: u8x32, + ) -> u8x32 { + let mask_bits = token.to_bitmask_mask8x32(mask); + let mut output = [0u8; 32]; + let mut output_lane = 0; + let block_bits = (mask_bits & 0xffff) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask]; + let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask]; + let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + 0x0808_0808_0808_0808) + | (high_control & 0x8080_8080_8080_8080); + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let low_count = low_mask.count_ones() as usize; + let compacted = _mm_shuffle_epi8(values.val.0[0], control); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count]; + let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let compressed = _mm_shuffle_epi8(compacted, splice); + unsafe { + _mm_storeu_si128( + output.as_mut_ptr().add(output_lane).cast::<__m128i>(), + compressed, + ); + } + output_lane += block_bits.count_ones() as usize; + let block_bits = (mask_bits >> 16 & 0xffff) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask]; + let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask]; + let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + 0x0808_0808_0808_0808) + | (high_control & 0x8080_8080_8080_8080); + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let low_count = low_mask.count_ones() as usize; + let compacted = _mm_shuffle_epi8(values.val.0[1], control); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count]; + let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let compressed = _mm_shuffle_epi8(compacted, splice); + unsafe { + _mm_storeu_si128( + output.as_mut_ptr().add(output_lane).cast::<__m128i>(), + compressed, + ); + } + output_lane += block_bits.count_ones() as usize; + let remaining = output_lane.saturating_sub(0).min(16); + let prefix = unsafe { + _mm_load_si128( + core::ptr::from_ref(&crate::support::COMPACT_PREFIX_MASKS[remaining]) + .cast::<__m128i>(), + ) + }; + let compressed = + unsafe { _mm_loadu_si128(output.as_ptr().add(0).cast::<__m128i>()) }; + let blended = _mm_blendv_epi8(merge.val.0[0], compressed, prefix); + unsafe { + _mm_storeu_si128(output.as_mut_ptr().add(0).cast::<__m128i>(), blended); + } + let remaining = output_lane.saturating_sub(16).min(16); + let prefix = unsafe { + _mm_load_si128( + core::ptr::from_ref(&crate::support::COMPACT_PREFIX_MASKS[remaining]) + .cast::<__m128i>(), + ) + }; + let compressed = + unsafe { _mm_loadu_si128(output.as_ptr().add(16).cast::<__m128i>()) }; + let blended = _mm_blendv_epi8(merge.val.0[1], compressed, prefix); + unsafe { + _mm_storeu_si128(output.as_mut_ptr().add(16).cast::<__m128i>(), blended); + } + u8x32::simd_from(token, output) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_u8x32(self, values: u8x32, mask: mask8x32) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Sse4_2, + values: u8x32, + mask: mask8x32, + ) -> u8x32 { + let mask_bits = token.to_bitmask_mask8x32(mask); + let input = <[u8; 32]>::from(values); + let mut output = [0u8; 32]; + let mut input_lane = 0; + let block_bits = (mask_bits & 0xffff) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) + | (high_control & 0x8080_8080_8080_8080); + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = + unsafe { _mm_loadu_si128(input.as_ptr().add(input_lane).cast::<__m128i>()) }; + let expanded = _mm_shuffle_epi8(packed, control); + let result = expanded; + unsafe { + _mm_storeu_si128(output.as_mut_ptr().add(0).cast::<__m128i>(), result); + } + input_lane += block_bits.count_ones() as usize; + let block_bits = (mask_bits >> 16 & 0xffff) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) + | (high_control & 0x8080_8080_8080_8080); + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = + unsafe { _mm_loadu_si128(input.as_ptr().add(input_lane).cast::<__m128i>()) }; + let expanded = _mm_shuffle_epi8(packed, control); + let result = expanded; + unsafe { + _mm_storeu_si128(output.as_mut_ptr().add(16).cast::<__m128i>(), result); + } + u8x32::simd_from(token, output) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_u8x32( + self, + values: u8x32, + mask: mask8x32, + merge: u8x32, + ) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Sse4_2, + values: u8x32, + mask: mask8x32, + merge: u8x32, + ) -> u8x32 { + let mask_bits = token.to_bitmask_mask8x32(mask); + let input = <[u8; 32]>::from(values); + let mut output = [0u8; 32]; + let mut input_lane = 0; + let block_bits = (mask_bits & 0xffff) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) + | (high_control & 0x8080_8080_8080_8080); + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = + unsafe { _mm_loadu_si128(input.as_ptr().add(input_lane).cast::<__m128i>()) }; + let expanded = _mm_shuffle_epi8(packed, control); + let result = _mm_blendv_epi8(merge.val.0[0], expanded, mask.val.0[0]); + unsafe { + _mm_storeu_si128(output.as_mut_ptr().add(0).cast::<__m128i>(), result); + } + input_lane += block_bits.count_ones() as usize; + let block_bits = (mask_bits >> 16 & 0xffff) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) + | (high_control & 0x8080_8080_8080_8080); + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = + unsafe { _mm_loadu_si128(input.as_ptr().add(input_lane).cast::<__m128i>()) }; + let expanded = _mm_shuffle_epi8(packed, control); + let result = _mm_blendv_epi8(merge.val.0[1], expanded, mask.val.0[1]); + unsafe { + _mm_storeu_si128(output.as_mut_ptr().add(16).cast::<__m128i>(), result); + } + u8x32::simd_from(token, output) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn combine_u8x32(self, a: u8x32, b: u8x32) -> u8x64 { u8x64 { val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), @@ -7477,6 +7857,441 @@ impl Simd for Sse4_2 { Bytes::from_bytes(result_bytes) } #[inline(always)] + fn compress_u8x64(self, values: u8x64, mask: mask8x64) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Sse4_2, + values: u8x64, + mask: mask8x64, + ) -> u8x64 { + let mask_bits = token.to_bitmask_mask8x64(mask); + let mut output = [0u8; 64]; + let mut output_lane = 0; + let block_bits = (mask_bits & 0xffff) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask]; + let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask]; + let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + 0x0808_0808_0808_0808) + | (high_control & 0x8080_8080_8080_8080); + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let low_count = low_mask.count_ones() as usize; + let compacted = _mm_shuffle_epi8(values.val.0[0], control); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count]; + let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let compressed = _mm_shuffle_epi8(compacted, splice); + unsafe { + _mm_storeu_si128( + output.as_mut_ptr().add(output_lane).cast::<__m128i>(), + compressed, + ); + } + output_lane += block_bits.count_ones() as usize; + let block_bits = (mask_bits >> 16 & 0xffff) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask]; + let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask]; + let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + 0x0808_0808_0808_0808) + | (high_control & 0x8080_8080_8080_8080); + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let low_count = low_mask.count_ones() as usize; + let compacted = _mm_shuffle_epi8(values.val.0[1], control); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count]; + let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let compressed = _mm_shuffle_epi8(compacted, splice); + unsafe { + _mm_storeu_si128( + output.as_mut_ptr().add(output_lane).cast::<__m128i>(), + compressed, + ); + } + output_lane += block_bits.count_ones() as usize; + let block_bits = (mask_bits >> 32 & 0xffff) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask]; + let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask]; + let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + 0x0808_0808_0808_0808) + | (high_control & 0x8080_8080_8080_8080); + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let low_count = low_mask.count_ones() as usize; + let compacted = _mm_shuffle_epi8(values.val.0[2], control); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count]; + let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let compressed = _mm_shuffle_epi8(compacted, splice); + unsafe { + _mm_storeu_si128( + output.as_mut_ptr().add(output_lane).cast::<__m128i>(), + compressed, + ); + } + output_lane += block_bits.count_ones() as usize; + let block_bits = (mask_bits >> 48 & 0xffff) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask]; + let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask]; + let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + 0x0808_0808_0808_0808) + | (high_control & 0x8080_8080_8080_8080); + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let low_count = low_mask.count_ones() as usize; + let compacted = _mm_shuffle_epi8(values.val.0[3], control); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count]; + let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let compressed = _mm_shuffle_epi8(compacted, splice); + unsafe { + _mm_storeu_si128( + output.as_mut_ptr().add(output_lane).cast::<__m128i>(), + compressed, + ); + } + u8x64::simd_from(token, output) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_u8x64( + self, + values: u8x64, + mask: mask8x64, + merge: u8x64, + ) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Sse4_2, + values: u8x64, + mask: mask8x64, + merge: u8x64, + ) -> u8x64 { + let mask_bits = token.to_bitmask_mask8x64(mask); + let mut output = [0u8; 64]; + let mut output_lane = 0; + let block_bits = (mask_bits & 0xffff) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask]; + let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask]; + let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + 0x0808_0808_0808_0808) + | (high_control & 0x8080_8080_8080_8080); + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let low_count = low_mask.count_ones() as usize; + let compacted = _mm_shuffle_epi8(values.val.0[0], control); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count]; + let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let compressed = _mm_shuffle_epi8(compacted, splice); + unsafe { + _mm_storeu_si128( + output.as_mut_ptr().add(output_lane).cast::<__m128i>(), + compressed, + ); + } + output_lane += block_bits.count_ones() as usize; + let block_bits = (mask_bits >> 16 & 0xffff) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask]; + let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask]; + let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + 0x0808_0808_0808_0808) + | (high_control & 0x8080_8080_8080_8080); + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let low_count = low_mask.count_ones() as usize; + let compacted = _mm_shuffle_epi8(values.val.0[1], control); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count]; + let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let compressed = _mm_shuffle_epi8(compacted, splice); + unsafe { + _mm_storeu_si128( + output.as_mut_ptr().add(output_lane).cast::<__m128i>(), + compressed, + ); + } + output_lane += block_bits.count_ones() as usize; + let block_bits = (mask_bits >> 32 & 0xffff) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask]; + let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask]; + let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + 0x0808_0808_0808_0808) + | (high_control & 0x8080_8080_8080_8080); + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let low_count = low_mask.count_ones() as usize; + let compacted = _mm_shuffle_epi8(values.val.0[2], control); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count]; + let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let compressed = _mm_shuffle_epi8(compacted, splice); + unsafe { + _mm_storeu_si128( + output.as_mut_ptr().add(output_lane).cast::<__m128i>(), + compressed, + ); + } + output_lane += block_bits.count_ones() as usize; + let block_bits = (mask_bits >> 48 & 0xffff) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask]; + let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask]; + let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + 0x0808_0808_0808_0808) + | (high_control & 0x8080_8080_8080_8080); + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let low_count = low_mask.count_ones() as usize; + let compacted = _mm_shuffle_epi8(values.val.0[3], control); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count]; + let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let compressed = _mm_shuffle_epi8(compacted, splice); + unsafe { + _mm_storeu_si128( + output.as_mut_ptr().add(output_lane).cast::<__m128i>(), + compressed, + ); + } + output_lane += block_bits.count_ones() as usize; + let remaining = output_lane.saturating_sub(0).min(16); + let prefix = unsafe { + _mm_load_si128( + core::ptr::from_ref(&crate::support::COMPACT_PREFIX_MASKS[remaining]) + .cast::<__m128i>(), + ) + }; + let compressed = + unsafe { _mm_loadu_si128(output.as_ptr().add(0).cast::<__m128i>()) }; + let blended = _mm_blendv_epi8(merge.val.0[0], compressed, prefix); + unsafe { + _mm_storeu_si128(output.as_mut_ptr().add(0).cast::<__m128i>(), blended); + } + let remaining = output_lane.saturating_sub(16).min(16); + let prefix = unsafe { + _mm_load_si128( + core::ptr::from_ref(&crate::support::COMPACT_PREFIX_MASKS[remaining]) + .cast::<__m128i>(), + ) + }; + let compressed = + unsafe { _mm_loadu_si128(output.as_ptr().add(16).cast::<__m128i>()) }; + let blended = _mm_blendv_epi8(merge.val.0[1], compressed, prefix); + unsafe { + _mm_storeu_si128(output.as_mut_ptr().add(16).cast::<__m128i>(), blended); + } + let remaining = output_lane.saturating_sub(32).min(16); + let prefix = unsafe { + _mm_load_si128( + core::ptr::from_ref(&crate::support::COMPACT_PREFIX_MASKS[remaining]) + .cast::<__m128i>(), + ) + }; + let compressed = + unsafe { _mm_loadu_si128(output.as_ptr().add(32).cast::<__m128i>()) }; + let blended = _mm_blendv_epi8(merge.val.0[2], compressed, prefix); + unsafe { + _mm_storeu_si128(output.as_mut_ptr().add(32).cast::<__m128i>(), blended); + } + let remaining = output_lane.saturating_sub(48).min(16); + let prefix = unsafe { + _mm_load_si128( + core::ptr::from_ref(&crate::support::COMPACT_PREFIX_MASKS[remaining]) + .cast::<__m128i>(), + ) + }; + let compressed = + unsafe { _mm_loadu_si128(output.as_ptr().add(48).cast::<__m128i>()) }; + let blended = _mm_blendv_epi8(merge.val.0[3], compressed, prefix); + unsafe { + _mm_storeu_si128(output.as_mut_ptr().add(48).cast::<__m128i>(), blended); + } + u8x64::simd_from(token, output) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_u8x64(self, values: u8x64, mask: mask8x64) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Sse4_2, + values: u8x64, + mask: mask8x64, + ) -> u8x64 { + let mask_bits = token.to_bitmask_mask8x64(mask); + let input = <[u8; 64]>::from(values); + let mut output = [0u8; 64]; + let mut input_lane = 0; + let block_bits = (mask_bits & 0xffff) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) + | (high_control & 0x8080_8080_8080_8080); + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = + unsafe { _mm_loadu_si128(input.as_ptr().add(input_lane).cast::<__m128i>()) }; + let expanded = _mm_shuffle_epi8(packed, control); + let result = expanded; + unsafe { + _mm_storeu_si128(output.as_mut_ptr().add(0).cast::<__m128i>(), result); + } + input_lane += block_bits.count_ones() as usize; + let block_bits = (mask_bits >> 16 & 0xffff) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) + | (high_control & 0x8080_8080_8080_8080); + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = + unsafe { _mm_loadu_si128(input.as_ptr().add(input_lane).cast::<__m128i>()) }; + let expanded = _mm_shuffle_epi8(packed, control); + let result = expanded; + unsafe { + _mm_storeu_si128(output.as_mut_ptr().add(16).cast::<__m128i>(), result); + } + input_lane += block_bits.count_ones() as usize; + let block_bits = (mask_bits >> 32 & 0xffff) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) + | (high_control & 0x8080_8080_8080_8080); + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = + unsafe { _mm_loadu_si128(input.as_ptr().add(input_lane).cast::<__m128i>()) }; + let expanded = _mm_shuffle_epi8(packed, control); + let result = expanded; + unsafe { + _mm_storeu_si128(output.as_mut_ptr().add(32).cast::<__m128i>(), result); + } + input_lane += block_bits.count_ones() as usize; + let block_bits = (mask_bits >> 48 & 0xffff) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) + | (high_control & 0x8080_8080_8080_8080); + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = + unsafe { _mm_loadu_si128(input.as_ptr().add(input_lane).cast::<__m128i>()) }; + let expanded = _mm_shuffle_epi8(packed, control); + let result = expanded; + unsafe { + _mm_storeu_si128(output.as_mut_ptr().add(48).cast::<__m128i>(), result); + } + u8x64::simd_from(token, output) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_u8x64( + self, + values: u8x64, + mask: mask8x64, + merge: u8x64, + ) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Sse4_2, + values: u8x64, + mask: mask8x64, + merge: u8x64, + ) -> u8x64 { + let mask_bits = token.to_bitmask_mask8x64(mask); + let input = <[u8; 64]>::from(values); + let mut output = [0u8; 64]; + let mut input_lane = 0; + let block_bits = (mask_bits & 0xffff) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) + | (high_control & 0x8080_8080_8080_8080); + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = + unsafe { _mm_loadu_si128(input.as_ptr().add(input_lane).cast::<__m128i>()) }; + let expanded = _mm_shuffle_epi8(packed, control); + let result = _mm_blendv_epi8(merge.val.0[0], expanded, mask.val.0[0]); + unsafe { + _mm_storeu_si128(output.as_mut_ptr().add(0).cast::<__m128i>(), result); + } + input_lane += block_bits.count_ones() as usize; + let block_bits = (mask_bits >> 16 & 0xffff) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) + | (high_control & 0x8080_8080_8080_8080); + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = + unsafe { _mm_loadu_si128(input.as_ptr().add(input_lane).cast::<__m128i>()) }; + let expanded = _mm_shuffle_epi8(packed, control); + let result = _mm_blendv_epi8(merge.val.0[1], expanded, mask.val.0[1]); + unsafe { + _mm_storeu_si128(output.as_mut_ptr().add(16).cast::<__m128i>(), result); + } + input_lane += block_bits.count_ones() as usize; + let block_bits = (mask_bits >> 32 & 0xffff) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) + | (high_control & 0x8080_8080_8080_8080); + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = + unsafe { _mm_loadu_si128(input.as_ptr().add(input_lane).cast::<__m128i>()) }; + let expanded = _mm_shuffle_epi8(packed, control); + let result = _mm_blendv_epi8(merge.val.0[2], expanded, mask.val.0[2]); + unsafe { + _mm_storeu_si128(output.as_mut_ptr().add(32).cast::<__m128i>(), result); + } + input_lane += block_bits.count_ones() as usize; + let block_bits = (mask_bits >> 48 & 0xffff) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) + | (high_control & 0x8080_8080_8080_8080); + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = + unsafe { _mm_loadu_si128(input.as_ptr().add(input_lane).cast::<__m128i>()) }; + let expanded = _mm_shuffle_epi8(packed, control); + let result = _mm_blendv_epi8(merge.val.0[3], expanded, mask.val.0[3]); + unsafe { + _mm_storeu_si128(output.as_mut_ptr().add(48).cast::<__m128i>(), result); + } + u8x64::simd_from(token, output) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn split_u8x64(self, a: u8x64) -> (u8x32, u8x32) { ( u8x32 { diff --git a/fearless_simd/src/generated/wasm.rs b/fearless_simd/src/generated/wasm.rs index b0c49536c..f981c553e 100644 --- a/fearless_simd/src/generated/wasm.rs +++ b/fearless_simd/src/generated/wasm.rs @@ -1195,6 +1195,98 @@ impl Simd for WasmSimd128 { } } #[inline(always)] + fn compress_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16 { + let mask_bits = self.to_bitmask_mask8x16(mask); + let mut control = [u8::MAX; 16]; + let mut output_lane = 0; + for block in 0..16 / 8 { + let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; + let packed = crate::support::COMPRESS_8_CONTROLS[block_mask]; + let base = (block * 8) as u64 * 0x0101_0101_0101_0101; + let adjusted = + ((packed & 0x7f7f_7f7f_7f7f_7f7f) + base) | (packed & 0x8080_8080_8080_8080); + let adjusted = adjusted.to_le_bytes(); + let write_len = core::cmp::min(8, 16 - output_lane); + control[output_lane..output_lane + write_len].copy_from_slice(&adjusted[..write_len]); + output_lane += block_mask.count_ones() as usize; + } + let control = u8x16::simd_from(self, control); + self.swizzle_dyn_precise_u8x16(values, control) + } + #[inline(always)] + fn compress_merge_u8x16( + self, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + let mask_bits = self.to_bitmask_mask8x16(mask); + let mut control = [u8::MAX; 16]; + let mut output_lane = 0; + for block in 0..16 / 8 { + let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; + let packed = crate::support::COMPRESS_8_CONTROLS[block_mask]; + let base = (block * 8) as u64 * 0x0101_0101_0101_0101; + let adjusted = + ((packed & 0x7f7f_7f7f_7f7f_7f7f) + base) | (packed & 0x8080_8080_8080_8080); + let adjusted = adjusted.to_le_bytes(); + let write_len = core::cmp::min(8, 16 - output_lane); + control[output_lane..output_lane + write_len].copy_from_slice(&adjusted[..write_len]); + output_lane += block_mask.count_ones() as usize; + } + let control = u8x16::simd_from(self, control); + let compressed = self.swizzle_dyn_precise_u8x16(values, control); + let prefix_bits = if output_lane == 64 { + u64::MAX + } else { + (1u64 << output_lane) - 1 + }; + let prefix_mask = self.from_bitmask_mask8x16(prefix_bits); + self.select_u8x16(prefix_mask, compressed, merge) + } + #[inline(always)] + fn expand_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16 { + let mask_bits = self.to_bitmask_mask8x16(mask); + let mut control = [u8::MAX; 16]; + let mut input_lane = 0; + for block in 0..16 / 8 { + let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; + let packed = crate::support::EXPAND_8_CONTROLS[block_mask]; + let base = input_lane as u64 * 0x0101_0101_0101_0101; + let adjusted = + ((packed & 0x7f7f_7f7f_7f7f_7f7f) + base) | (packed & 0x8080_8080_8080_8080); + let output_lane = block * 8; + control[output_lane..output_lane + 8].copy_from_slice(&adjusted.to_le_bytes()); + input_lane += block_mask.count_ones() as usize; + } + let control = u8x16::simd_from(self, control); + self.swizzle_dyn_precise_u8x16(values, control) + } + #[inline(always)] + fn expand_merge_u8x16( + self, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + let mask_bits = self.to_bitmask_mask8x16(mask); + let mut control = [u8::MAX; 16]; + let mut input_lane = 0; + for block in 0..16 / 8 { + let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; + let packed = crate::support::EXPAND_8_CONTROLS[block_mask]; + let base = input_lane as u64 * 0x0101_0101_0101_0101; + let adjusted = + ((packed & 0x7f7f_7f7f_7f7f_7f7f) + base) | (packed & 0x8080_8080_8080_8080); + let output_lane = block * 8; + control[output_lane..output_lane + 8].copy_from_slice(&adjusted.to_le_bytes()); + input_lane += block_mask.count_ones() as usize; + } + let control = u8x16::simd_from(self, control); + let expanded = self.swizzle_dyn_precise_u8x16(values, control); + self.select_u8x16(mask, expanded, merge) + } + #[inline(always)] fn combine_u8x16(self, a: u8x16, b: u8x16) -> u8x32 { u8x32 { val: crate::support::Aligned256([a.val.0, b.val.0]), @@ -4085,6 +4177,136 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn compress_u8x32(self, values: u8x32, mask: mask8x32) -> u8x32 { + let mask_bits = self.to_bitmask_mask8x32(mask); + let mut output = [0u8; 32]; + let mut output_lane = 0; + for block in 0..32 / 16 { + let block_bits = ((mask_bits >> (block * 16)) & 0xffff) as usize; + let low_mask = block_bits & 0xff; + let high_mask = block_bits >> 8; + let low_count = low_mask.count_ones() as usize; + let low = crate::support::COMPRESS_8_CONTROLS[low_mask]; + let high = crate::support::COMPRESS_8_CONTROLS[high_mask]; + let high = ((high & 0x7f7f_7f7f_7f7f_7f7f) + 0x0808_0808_0808_0808) + | (high & 0x8080_8080_8080_8080); + let mut control = [u8::MAX; 16]; + control[..8].copy_from_slice(&low.to_le_bytes()); + control[low_count..low_count + 8].copy_from_slice(&high.to_le_bytes()); + let input = u8x16::from_slice(self, &values.as_array()[block * 16..block * 16 + 16]); + let control = u8x16::simd_from(self, control); + let compacted: [u8; 16] = self.swizzle_dyn_precise_u8x16(input, control).into(); + let write_len = core::cmp::min(16, 32 - output_lane); + output[output_lane..output_lane + write_len].copy_from_slice(&compacted[..write_len]); + output_lane += low_count + high_mask.count_ones() as usize; + } + u8x32::simd_from(self, output) + } + #[inline(always)] + fn compress_merge_u8x32( + self, + values: u8x32, + mask: mask8x32, + merge: u8x32, + ) -> u8x32 { + let mask_bits = self.to_bitmask_mask8x32(mask); + let mut output = [0u8; 32]; + let mut output_lane = 0; + for block in 0..32 / 16 { + let block_bits = ((mask_bits >> (block * 16)) & 0xffff) as usize; + let low_mask = block_bits & 0xff; + let high_mask = block_bits >> 8; + let low_count = low_mask.count_ones() as usize; + let low = crate::support::COMPRESS_8_CONTROLS[low_mask]; + let high = crate::support::COMPRESS_8_CONTROLS[high_mask]; + let high = ((high & 0x7f7f_7f7f_7f7f_7f7f) + 0x0808_0808_0808_0808) + | (high & 0x8080_8080_8080_8080); + let mut control = [u8::MAX; 16]; + control[..8].copy_from_slice(&low.to_le_bytes()); + control[low_count..low_count + 8].copy_from_slice(&high.to_le_bytes()); + let input = u8x16::from_slice(self, &values.as_array()[block * 16..block * 16 + 16]); + let control = u8x16::simd_from(self, control); + let compacted: [u8; 16] = self.swizzle_dyn_precise_u8x16(input, control).into(); + let write_len = core::cmp::min(16, 32 - output_lane); + output[output_lane..output_lane + write_len].copy_from_slice(&compacted[..write_len]); + output_lane += low_count + high_mask.count_ones() as usize; + } + let compressed = u8x32::simd_from(self, output); + let prefix_bits = if output_lane == 64 { + u64::MAX + } else { + (1u64 << output_lane) - 1 + }; + let prefix_mask = self.from_bitmask_mask8x32(prefix_bits); + self.select_u8x32(prefix_mask, compressed, merge) + } + #[inline(always)] + fn expand_u8x32(self, values: u8x32, mask: mask8x32) -> u8x32 { + let mask_bits = self.to_bitmask_mask8x32(mask); + let values: [u8; 32] = values.into(); + let mut output = [0u8; 32]; + let mut input_lane = 0; + for block in 0..32 / 16 { + let block_bits = ((mask_bits >> (block * 16)) & 0xffff) as usize; + let low_mask = block_bits & 0xff; + let high_mask = block_bits >> 8; + let low_count = low_mask.count_ones() as usize; + let low = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count as u64 * 0x0101_0101_0101_0101; + let high = + ((high & 0x7f7f_7f7f_7f7f_7f7f) + high_base) | (high & 0x8080_8080_8080_8080); + let mut control = [0u8; 16]; + control[..8].copy_from_slice(&low.to_le_bytes()); + control[8..].copy_from_slice(&high.to_le_bytes()); + let mut input = [0u8; 16]; + let read_len = core::cmp::min(16, 32 - input_lane); + input[..read_len].copy_from_slice(&values[input_lane..input_lane + read_len]); + let input = u8x16::simd_from(self, input); + let control = u8x16::simd_from(self, control); + let expanded: [u8; 16] = self.swizzle_dyn_precise_u8x16(input, control).into(); + output[block * 16..block * 16 + 16].copy_from_slice(&expanded); + input_lane += low_count + high_mask.count_ones() as usize; + } + u8x32::simd_from(self, output) + } + #[inline(always)] + fn expand_merge_u8x32( + self, + values: u8x32, + mask: mask8x32, + merge: u8x32, + ) -> u8x32 { + let mask_bits = self.to_bitmask_mask8x32(mask); + let values: [u8; 32] = values.into(); + let mut output = [0u8; 32]; + let mut input_lane = 0; + for block in 0..32 / 16 { + let block_bits = ((mask_bits >> (block * 16)) & 0xffff) as usize; + let low_mask = block_bits & 0xff; + let high_mask = block_bits >> 8; + let low_count = low_mask.count_ones() as usize; + let low = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count as u64 * 0x0101_0101_0101_0101; + let high = + ((high & 0x7f7f_7f7f_7f7f_7f7f) + high_base) | (high & 0x8080_8080_8080_8080); + let mut control = [0u8; 16]; + control[..8].copy_from_slice(&low.to_le_bytes()); + control[8..].copy_from_slice(&high.to_le_bytes()); + let mut input = [0u8; 16]; + let read_len = core::cmp::min(16, 32 - input_lane); + input[..read_len].copy_from_slice(&values[input_lane..input_lane + read_len]); + let input = u8x16::simd_from(self, input); + let control = u8x16::simd_from(self, control); + let expanded: [u8; 16] = self.swizzle_dyn_precise_u8x16(input, control).into(); + output[block * 16..block * 16 + 16].copy_from_slice(&expanded); + input_lane += low_count + high_mask.count_ones() as usize; + } + let expanded = u8x32::simd_from(self, output); + self.select_u8x32(mask, expanded, merge) + } + #[inline(always)] fn combine_u8x32(self, a: u8x32, b: u8x32) -> u8x64 { u8x64 { val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), @@ -4769,6 +4991,136 @@ impl Simd for WasmSimd128 { Bytes::from_bytes(result_bytes) } #[inline(always)] + fn compress_u8x64(self, values: u8x64, mask: mask8x64) -> u8x64 { + let mask_bits = self.to_bitmask_mask8x64(mask); + let mut output = [0u8; 64]; + let mut output_lane = 0; + for block in 0..64 / 16 { + let block_bits = ((mask_bits >> (block * 16)) & 0xffff) as usize; + let low_mask = block_bits & 0xff; + let high_mask = block_bits >> 8; + let low_count = low_mask.count_ones() as usize; + let low = crate::support::COMPRESS_8_CONTROLS[low_mask]; + let high = crate::support::COMPRESS_8_CONTROLS[high_mask]; + let high = ((high & 0x7f7f_7f7f_7f7f_7f7f) + 0x0808_0808_0808_0808) + | (high & 0x8080_8080_8080_8080); + let mut control = [u8::MAX; 16]; + control[..8].copy_from_slice(&low.to_le_bytes()); + control[low_count..low_count + 8].copy_from_slice(&high.to_le_bytes()); + let input = u8x16::from_slice(self, &values.as_array()[block * 16..block * 16 + 16]); + let control = u8x16::simd_from(self, control); + let compacted: [u8; 16] = self.swizzle_dyn_precise_u8x16(input, control).into(); + let write_len = core::cmp::min(16, 64 - output_lane); + output[output_lane..output_lane + write_len].copy_from_slice(&compacted[..write_len]); + output_lane += low_count + high_mask.count_ones() as usize; + } + u8x64::simd_from(self, output) + } + #[inline(always)] + fn compress_merge_u8x64( + self, + values: u8x64, + mask: mask8x64, + merge: u8x64, + ) -> u8x64 { + let mask_bits = self.to_bitmask_mask8x64(mask); + let mut output = [0u8; 64]; + let mut output_lane = 0; + for block in 0..64 / 16 { + let block_bits = ((mask_bits >> (block * 16)) & 0xffff) as usize; + let low_mask = block_bits & 0xff; + let high_mask = block_bits >> 8; + let low_count = low_mask.count_ones() as usize; + let low = crate::support::COMPRESS_8_CONTROLS[low_mask]; + let high = crate::support::COMPRESS_8_CONTROLS[high_mask]; + let high = ((high & 0x7f7f_7f7f_7f7f_7f7f) + 0x0808_0808_0808_0808) + | (high & 0x8080_8080_8080_8080); + let mut control = [u8::MAX; 16]; + control[..8].copy_from_slice(&low.to_le_bytes()); + control[low_count..low_count + 8].copy_from_slice(&high.to_le_bytes()); + let input = u8x16::from_slice(self, &values.as_array()[block * 16..block * 16 + 16]); + let control = u8x16::simd_from(self, control); + let compacted: [u8; 16] = self.swizzle_dyn_precise_u8x16(input, control).into(); + let write_len = core::cmp::min(16, 64 - output_lane); + output[output_lane..output_lane + write_len].copy_from_slice(&compacted[..write_len]); + output_lane += low_count + high_mask.count_ones() as usize; + } + let compressed = u8x64::simd_from(self, output); + let prefix_bits = if output_lane == 64 { + u64::MAX + } else { + (1u64 << output_lane) - 1 + }; + let prefix_mask = self.from_bitmask_mask8x64(prefix_bits); + self.select_u8x64(prefix_mask, compressed, merge) + } + #[inline(always)] + fn expand_u8x64(self, values: u8x64, mask: mask8x64) -> u8x64 { + let mask_bits = self.to_bitmask_mask8x64(mask); + let values: [u8; 64] = values.into(); + let mut output = [0u8; 64]; + let mut input_lane = 0; + for block in 0..64 / 16 { + let block_bits = ((mask_bits >> (block * 16)) & 0xffff) as usize; + let low_mask = block_bits & 0xff; + let high_mask = block_bits >> 8; + let low_count = low_mask.count_ones() as usize; + let low = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count as u64 * 0x0101_0101_0101_0101; + let high = + ((high & 0x7f7f_7f7f_7f7f_7f7f) + high_base) | (high & 0x8080_8080_8080_8080); + let mut control = [0u8; 16]; + control[..8].copy_from_slice(&low.to_le_bytes()); + control[8..].copy_from_slice(&high.to_le_bytes()); + let mut input = [0u8; 16]; + let read_len = core::cmp::min(16, 64 - input_lane); + input[..read_len].copy_from_slice(&values[input_lane..input_lane + read_len]); + let input = u8x16::simd_from(self, input); + let control = u8x16::simd_from(self, control); + let expanded: [u8; 16] = self.swizzle_dyn_precise_u8x16(input, control).into(); + output[block * 16..block * 16 + 16].copy_from_slice(&expanded); + input_lane += low_count + high_mask.count_ones() as usize; + } + u8x64::simd_from(self, output) + } + #[inline(always)] + fn expand_merge_u8x64( + self, + values: u8x64, + mask: mask8x64, + merge: u8x64, + ) -> u8x64 { + let mask_bits = self.to_bitmask_mask8x64(mask); + let values: [u8; 64] = values.into(); + let mut output = [0u8; 64]; + let mut input_lane = 0; + for block in 0..64 / 16 { + let block_bits = ((mask_bits >> (block * 16)) & 0xffff) as usize; + let low_mask = block_bits & 0xff; + let high_mask = block_bits >> 8; + let low_count = low_mask.count_ones() as usize; + let low = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count as u64 * 0x0101_0101_0101_0101; + let high = + ((high & 0x7f7f_7f7f_7f7f_7f7f) + high_base) | (high & 0x8080_8080_8080_8080); + let mut control = [0u8; 16]; + control[..8].copy_from_slice(&low.to_le_bytes()); + control[8..].copy_from_slice(&high.to_le_bytes()); + let mut input = [0u8; 16]; + let read_len = core::cmp::min(16, 64 - input_lane); + input[..read_len].copy_from_slice(&values[input_lane..input_lane + read_len]); + let input = u8x16::simd_from(self, input); + let control = u8x16::simd_from(self, control); + let expanded: [u8; 16] = self.swizzle_dyn_precise_u8x16(input, control).into(); + output[block * 16..block * 16 + 16].copy_from_slice(&expanded); + input_lane += low_count + high_mask.count_ones() as usize; + } + let expanded = u8x64::simd_from(self, output); + self.select_u8x64(mask, expanded, merge) + } + #[inline(always)] fn split_u8x64(self, a: u8x64) -> (u8x32, u8x32) { ( u8x32 { diff --git a/fearless_simd/src/support.rs b/fearless_simd/src/support.rs index 2c298326f..ae7ea63da 100644 --- a/fearless_simd/src/support.rs +++ b/fearless_simd/src/support.rs @@ -63,3 +63,200 @@ pub(crate) fn cross_block_slide_blocks_at( let hi_block = if hi_idx < N { a[hi_idx] } else { b[hi_idx - N] }; [lo_block, hi_block] } + +const fn compact_control_table(expand: bool) -> [u64; 256] { + let mut table = [u64::MAX; 256]; + let mut mask = 0_usize; + while mask < table.len() { + let mut control = u64::MAX; + let mut selected = 0_usize; + let mut lane = 0_usize; + while lane < 8 { + if mask & (1 << lane) != 0 { + let output_lane = if expand { lane } else { selected }; + control &= !(0xff_u64 << (output_lane * 8)); + let index = if expand { selected } else { lane }; + control |= (index as u64) << (output_lane * 8); + selected += 1; + } + lane += 1; + } + table[mask] = control; + mask += 1; + } + table +} + +const fn compact_count_table() -> [u8; 256] { + let mut table = [0; 256]; + let mut mask = 0_usize; + while mask < table.len() { + let mut bits = mask; + let mut count = 0; + while bits != 0 { + if bits & 1 != 0 { + count += 1; + } + bits >>= 1; + } + table[mask] = count; + mask += 1; + } + table +} + +const fn compact_splice_control_table() -> [u128; 9] { + let mut table = [u128::MAX; 9]; + let mut low_count = 0; + while low_count <= 8 { + let mut control = u128::MAX; + let mut lane = 0; + while lane < low_count { + control &= !(0xff_u128 << (lane * 8)); + control |= (lane as u128) << (lane * 8); + lane += 1; + } + lane = 0; + while lane < 8 { + let output_lane = low_count + lane; + if output_lane < 16 { + control &= !(0xff_u128 << (output_lane * 8)); + control |= ((lane + 8) as u128) << (output_lane * 8); + } + lane += 1; + } + table[low_count] = control; + low_count += 1; + } + table +} + +#[allow( + clippy::cast_possible_truncation, + reason = "lane is bounded to 0..16 by the table-construction loop" +)] +const fn compact_16_splice_control_table() -> [Aligned256<[u8; 32]>; 17] { + let mut table = [Aligned256([0x80; 32]); 17]; + let mut low_count = 0; + while low_count <= 16 { + let mut lane = 0; + while lane < 16 { + let output_lane = low_count + lane; + if output_lane < 32 { + table[low_count].0[output_lane] = lane as u8; + } + lane += 1; + } + low_count += 1; + } + table +} + +const fn compact_prefix_mask_table() -> [Aligned256<[u8; 32]>; 33] { + let mut table = [Aligned256([0; 32]); 33]; + let mut count = 0; + while count <= 32 { + let mut lane = 0; + while lane < count { + table[count].0[lane] = u8::MAX; + lane += 1; + } + count += 1; + } + table +} + +#[derive(Clone, Copy)] +#[repr(C, align(32))] +pub(crate) struct Compact32StitchControls { + pub(crate) left_same: [u8; 32], + pub(crate) left_cross: [u8; 32], + pub(crate) right_same: [u8; 32], + pub(crate) right_cross: [u8; 32], +} + +/// Controls for concatenating two compacted 32-byte vectors entirely in registers. +/// +/// For a first-vector length `count`, `left_*` shifts the second vector left by `count` +/// bytes and `right_*` shifts it right by `32 - count` bytes. Separate same-lane and +/// cross-lane controls account for AVX2 `vpshufb` operating independently in each +/// 128-bit lane. +#[allow( + clippy::cast_possible_truncation, + reason = "shuffle indices are reduced modulo 16" +)] +const fn compact_32_stitch_control_table() -> [Compact32StitchControls; 33] { + const EMPTY: Compact32StitchControls = Compact32StitchControls { + left_same: [0x80; 32], + left_cross: [0x80; 32], + right_same: [0x80; 32], + right_cross: [0x80; 32], + }; + + let mut table = [EMPTY; 33]; + let mut count = 0; + while count <= 32 { + let mut lane = 0; + while lane < 32 { + if lane >= count { + let source = lane - count; + if lane / 16 == source / 16 { + table[count].left_same[lane] = (source % 16) as u8; + } else { + table[count].left_cross[lane] = (source % 16) as u8; + } + } + + let source = lane + (32 - count); + if source < 32 { + if lane / 16 == source / 16 { + table[count].right_same[lane] = (source % 16) as u8; + } else { + table[count].right_cross[lane] = (source % 16) as u8; + } + } + lane += 1; + } + count += 1; + } + table +} + +/// Eight-byte shuffle controls indexed by an eight-bit selection mask. +#[allow( + dead_code, + reason = "Used only by SIMD backends with byte-table shuffles" +)] +pub(crate) const COMPRESS_8_CONTROLS: [u64; 256] = compact_control_table(false); + +/// Eight-byte inverse-shuffle controls indexed by an eight-bit selection mask. +#[allow( + dead_code, + reason = "Used only by SIMD backends with byte-table shuffles" +)] +pub(crate) const EXPAND_8_CONTROLS: [u64; 256] = compact_control_table(true); + +/// Population counts indexed by an eight-bit selection mask. +#[allow( + dead_code, + reason = "Used only by SIMD backends with byte-table shuffles" +)] +pub(crate) const COMPACT_8_COUNTS: [u8; 256] = compact_count_table(); + +/// Shuffle controls that splice two independently compacted eight-byte halves. +#[allow(dead_code, reason = "Used only by x86 byte compression")] +pub(crate) const COMPACT_8_SPLICE_CONTROLS: [u128; 9] = compact_splice_control_table(); + +/// Shuffle controls that append a compacted 16-byte high lane after a compacted low lane. +#[allow(dead_code, reason = "Used only by AVX2 byte compression")] +pub(crate) const COMPACT_16_SPLICE_CONTROLS: [Aligned256<[u8; 32]>; 17] = + compact_16_splice_control_table(); + +/// Shuffle controls that concatenate two independently compacted 32-byte vectors. +#[allow(dead_code, reason = "Used only by AVX2 64-byte compression")] +pub(crate) const COMPACT_32_STITCH_CONTROLS: [Compact32StitchControls; 33] = + compact_32_stitch_control_table(); + +/// Byte prefix masks indexed by the number of active lanes. +#[allow(dead_code, reason = "Used only by x86 byte compression")] +pub(crate) const COMPACT_PREFIX_MASKS: [Aligned256<[u8; 32]>; 33] = compact_prefix_mask_table(); diff --git a/fearless_simd_gen/src/generic.rs b/fearless_simd_gen/src/generic.rs index 0df46a850..3c5079c47 100644 --- a/fearless_simd_gen/src/generic.rs +++ b/fearless_simd_gen/src/generic.rs @@ -353,6 +353,69 @@ pub(crate) fn generic_op(op: &Op, ty: &VecType) -> TokenStream { | OpSig::ConcatSwizzleDynPrecise => { panic!("whole-vector swizzles cannot be done via split/combine"); } + OpSig::Compress { merge: false } => { + let merge_method = generic_op_name("compress_merge", ty); + let ty = ty.rust(); + quote! { + #method_sig { + self.#merge_method(values, mask, #ty::splat(self, 0)) + } + } + } + OpSig::Compress { merge: true } => { + let len = Literal::usize_unsuffixed(ty.len); + let to_bitmask = generic_op_name("to_bitmask", &ty.mask_ty()); + quote! { + #method_sig { + // Branchless: every lane is written at the cursor, which only advances on + // selected lanes. The one slot past the selected lanes may be clobbered by an + // unselected lane, so it is restored from `merge` afterwards. + let mask = self.#to_bitmask(mask); + let mut merge_padded = [0u8; #len + 1]; + merge_padded[..#len].copy_from_slice(&*merge); + let mut compacted = merge_padded; + let mut output_lane = 0; + for input_lane in 0..#len { + compacted[output_lane] = values[input_lane]; + output_lane += ((mask >> input_lane) & 1) as usize; + } + compacted[output_lane] = merge_padded[output_lane]; + let mut result = merge; + result.copy_from_slice(&compacted[..#len]); + result + } + } + } + OpSig::Expand { merge: false } => { + let merge_method = generic_op_name("expand_merge", ty); + let ty = ty.rust(); + quote! { + #method_sig { + self.#merge_method(values, mask, #ty::splat(self, 0)) + } + } + } + OpSig::Expand { merge: true } => { + let len = Literal::usize_unsuffixed(ty.len); + let to_bitmask = generic_op_name("to_bitmask", &ty.mask_ty()); + quote! { + #method_sig { + // Branchless: the input cursor never passes the output lane, so the read is + // always in bounds and selected with a lane mask instead of a branch. + let mask = self.#to_bitmask(mask); + let mut result = merge; + let mut input_lane = 0; + for output_lane in 0..#len { + let bit = ((mask >> output_lane) & 1) as u8; + let keep = 0u8.wrapping_sub(bit); + result[output_lane] = + (values[input_lane] & keep) | (result[output_lane] & !keep); + input_lane += usize::from(bit); + } + result + } + } + } OpSig::Ternary => { quote! { #method_sig { @@ -597,6 +660,255 @@ pub(crate) fn generic_op(op: &Op, ty: &VecType) -> TokenStream { } } +pub(crate) type CompactMergeSwizzle = + fn(&VecType, &TokenStream, &TokenStream, &TokenStream) -> TokenStream; + +/// Backend capabilities used by [`composed_compact_op`]. +pub(crate) struct CompactOptions { + /// Compact wide vectors a native 128-bit block at a time and splice them through memory. + pub(crate) splice_wide_vectors: bool, + /// Use the target's scalar population-count instruction instead of the byte-count table. + pub(crate) hardware_popcount: bool, + /// Optional backend operation that combines an out-of-range-zeroing shuffle with its merge + /// operand. The arguments are `(vector type, values, control, merge)`. + pub(crate) merge_swizzle: Option, +} + +/// Implement byte compression and expansion in terms of a backend's optimized whole-vector +/// swizzle, compact mask conversion, and selection operations. +pub(crate) fn composed_compact_op(op: Op, ty: &VecType, options: CompactOptions) -> TokenStream { + assert_eq!( + ty.scalar, + ScalarType::Unsigned, + "compact operations require unsigned vectors" + ); + assert_eq!(ty.scalar_bits, 8, "compact operations require byte lanes"); + + let method_sig = op.simd_trait_method_sig(ty); + let vec = ty.rust(); + let len = Literal::usize_unsuffixed(ty.len); + let swizzle = generic_op_name("swizzle_dyn_precise", ty); + let to_bitmask = generic_op_name("to_bitmask", &ty.mask_ty()); + let from_bitmask = generic_op_name("from_bitmask", &ty.mask_ty()); + let select = generic_op_name("select", ty); + let count_8 = |mask: TokenStream| { + if options.hardware_popcount { + quote! { #mask.count_ones() as usize } + } else { + quote! { crate::support::COMPACT_8_COUNTS[#mask] as usize } + } + }; + + // Backends whose byte shuffle is confined to 128-bit blocks can avoid an expensive emulated + // whole-vector shuffle by compacting each native block and splicing through a stack buffer. + // This is the same broad strategy used by Highway on targets without native byte + // compress/expand instructions. + if options.splice_wide_vectors && ty.len > 16 { + assert!( + options.merge_swizzle.is_none(), + "block-spliced compact operations do not support a merging shuffle" + ); + let block_vec = VecType { + scalar: ScalarType::Unsigned, + scalar_bits: 8, + len: 16, + }; + let block_swizzle = generic_op_name("swizzle_dyn_precise", &block_vec); + + return match op.sig { + OpSig::Compress { merge } => { + let low_count = count_8(quote! { low_mask }); + let high_count = count_8(quote! { high_mask }); + let finish = if merge { + quote! { + let compressed = #vec::simd_from(self, output); + let prefix_bits = if output_lane == 64 { + u64::MAX + } else { + (1u64 << output_lane) - 1 + }; + let prefix_mask = self.#from_bitmask(prefix_bits); + self.#select(prefix_mask, compressed, merge) + } + } else { + quote! { #vec::simd_from(self, output) } + }; + quote! { + #method_sig { + let mask_bits = self.#to_bitmask(mask); + let mut output = [0u8; #len]; + let mut output_lane = 0; + for block in 0..#len / 16 { + let block_bits = ((mask_bits >> (block * 16)) & 0xffff) as usize; + let low_mask = block_bits & 0xff; + let high_mask = block_bits >> 8; + let low_count = #low_count; + let low = crate::support::COMPRESS_8_CONTROLS[low_mask]; + let high = crate::support::COMPRESS_8_CONTROLS[high_mask]; + let high = ((high & 0x7f7f_7f7f_7f7f_7f7f) + + 0x0808_0808_0808_0808) + | (high & 0x8080_8080_8080_8080); + let mut control = [u8::MAX; 16]; + control[..8].copy_from_slice(&low.to_le_bytes()); + control[low_count..low_count + 8] + .copy_from_slice(&high.to_le_bytes()); + let input = u8x16::from_slice( + self, + &values.as_array()[block * 16..block * 16 + 16], + ); + let control = u8x16::simd_from(self, control); + let compacted: [u8; 16] = + self.#block_swizzle(input, control).into(); + let write_len = core::cmp::min(16, #len - output_lane); + output[output_lane..output_lane + write_len] + .copy_from_slice(&compacted[..write_len]); + output_lane += low_count + #high_count; + } + #finish + } + } + } + OpSig::Expand { merge } => { + let low_count = count_8(quote! { low_mask }); + let high_count = count_8(quote! { high_mask }); + let finish = if merge { + quote! { + let expanded = #vec::simd_from(self, output); + self.#select(mask, expanded, merge) + } + } else { + quote! { #vec::simd_from(self, output) } + }; + quote! { + #method_sig { + let mask_bits = self.#to_bitmask(mask); + let values: [u8; #len] = values.into(); + let mut output = [0u8; #len]; + let mut input_lane = 0; + for block in 0..#len / 16 { + let block_bits = ((mask_bits >> (block * 16)) & 0xffff) as usize; + let low_mask = block_bits & 0xff; + let high_mask = block_bits >> 8; + let low_count = #low_count; + let low = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count as u64 * 0x0101_0101_0101_0101; + let high = ((high & 0x7f7f_7f7f_7f7f_7f7f) + high_base) + | (high & 0x8080_8080_8080_8080); + let mut control = [0u8; 16]; + control[..8].copy_from_slice(&low.to_le_bytes()); + control[8..].copy_from_slice(&high.to_le_bytes()); + let mut input = [0u8; 16]; + let read_len = core::cmp::min(16, #len - input_lane); + input[..read_len] + .copy_from_slice(&values[input_lane..input_lane + read_len]); + let input = u8x16::simd_from(self, input); + let control = u8x16::simd_from(self, control); + let expanded: [u8; 16] = + self.#block_swizzle(input, control).into(); + output[block * 16..block * 16 + 16].copy_from_slice(&expanded); + input_lane += low_count + #high_count; + } + #finish + } + } + } + _ => unreachable!("composed_compact_op only implements compact byte operations"), + }; + } + + match op.sig { + OpSig::Compress { merge } => { + let block_count = count_8(quote! { block_mask }); + let finish = if merge { + if let Some(merge_swizzle) = options.merge_swizzle { + merge_swizzle( + ty, + "e! { values }, + "e! { control }, + "e! { merge }, + ) + } else { + quote! { + let compressed = self.#swizzle(values, control); + let prefix_bits = if output_lane == 64 { + u64::MAX + } else { + (1u64 << output_lane) - 1 + }; + let prefix_mask = self.#from_bitmask(prefix_bits); + self.#select(prefix_mask, compressed, merge) + } + } + } else { + quote! { self.#swizzle(values, control) } + }; + quote! { + #method_sig { + let mask_bits = self.#to_bitmask(mask); + let mut control = [u8::MAX; #len]; + let mut output_lane = 0; + for block in 0..#len / 8 { + let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; + let packed = crate::support::COMPRESS_8_CONTROLS[block_mask]; + let base = (block * 8) as u64 * 0x0101_0101_0101_0101; + let adjusted = ((packed & 0x7f7f_7f7f_7f7f_7f7f) + base) + | (packed & 0x8080_8080_8080_8080); + let adjusted = adjusted.to_le_bytes(); + let write_len = core::cmp::min(8, #len - output_lane); + control[output_lane..output_lane + write_len] + .copy_from_slice(&adjusted[..write_len]); + output_lane += #block_count; + } + let control = #vec::simd_from(self, control); + #finish + } + } + } + OpSig::Expand { merge } => { + let block_count = count_8(quote! { block_mask }); + let finish = if merge { + if let Some(merge_swizzle) = options.merge_swizzle { + merge_swizzle( + ty, + "e! { values }, + "e! { control }, + "e! { merge }, + ) + } else { + quote! { + let expanded = self.#swizzle(values, control); + self.#select(mask, expanded, merge) + } + } + } else { + quote! { self.#swizzle(values, control) } + }; + quote! { + #method_sig { + let mask_bits = self.#to_bitmask(mask); + let mut control = [u8::MAX; #len]; + let mut input_lane = 0; + for block in 0..#len / 8 { + let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; + let packed = crate::support::EXPAND_8_CONTROLS[block_mask]; + let base = input_lane as u64 * 0x0101_0101_0101_0101; + let adjusted = ((packed & 0x7f7f_7f7f_7f7f_7f7f) + base) + | (packed & 0x8080_8080_8080_8080); + let output_lane = block * 8; + control[output_lane..output_lane + 8] + .copy_from_slice(&adjusted.to_le_bytes()); + input_lane += #block_count; + } + let control = #vec::simd_from(self, control); + #finish + } + } + } + _ => unreachable!("composed_compact_op only implements compact byte operations"), + } +} + pub(crate) fn unrolled_array(len: usize, item: impl FnMut(usize) -> TokenStream) -> TokenStream { let items = (0..len).map(item).collect::>(); quote! { [#(#items),*] } diff --git a/fearless_simd_gen/src/mk_fallback.rs b/fearless_simd_gen/src/mk_fallback.rs index 125a5e7c1..1a86c1bed 100644 --- a/fearless_simd_gen/src/mk_fallback.rs +++ b/fearless_simd_gen/src/mk_fallback.rs @@ -4,7 +4,7 @@ use crate::arch::fallback; use crate::generic::{ generic_classify, generic_mask_from_bitmask, generic_mask_set, generic_mask_to_bitmask, - generic_op_name, integer_lane_mask_rotate, integer_lane_mask_splat_arg, + generic_op, generic_op_name, integer_lane_mask_rotate, integer_lane_mask_splat_arg, }; use crate::level::Level; use crate::ops::{NarrowingMode, Op, OpSig, relaxed_narrow_method}; @@ -591,6 +591,7 @@ impl Level for Fallback { } } } + OpSig::Compress { .. } | OpSig::Expand { .. } => generic_op(&op, vec_ty), OpSig::Cvt { target_ty, scalar_bits, diff --git a/fearless_simd_gen/src/mk_neon.rs b/fearless_simd_gen/src/mk_neon.rs index a82340511..59a44c772 100644 --- a/fearless_simd_gen/src/mk_neon.rs +++ b/fearless_simd_gen/src/mk_neon.rs @@ -5,9 +5,9 @@ use proc_macro2::{Ident, Literal, Span, TokenStream}; use quote::{ToTokens as _, format_ident, quote}; use crate::generic::{ - count_zeros_method, fallback_method, generic_classify, generic_mask_set, generic_op_name, - integer_lane_mask_rotate, integer_lane_mask_splat_arg, reverse_method, - reverse_vector_mask_method, + CompactOptions, composed_compact_op, count_zeros_method, fallback_method, generic_classify, + generic_mask_set, generic_op_name, integer_lane_mask_rotate, integer_lane_mask_splat_arg, + reverse_method, reverse_vector_mask_method, }; use crate::level::Level; use crate::ops::{NarrowingMode, Op, SlideGranularity, relaxed_narrow_method}; @@ -21,6 +21,62 @@ use crate::{ pub(crate) struct Neon; impl Neon { + fn compact_merge_swizzle( + vec_ty: &VecType, + values: &TokenStream, + control: &TokenStream, + merge: &TokenStream, + ) -> TokenStream { + let vec = vec_ty.rust(); + let wrapper = vec_ty.aligned_wrapper(); + let arch_ty = Self.arch_ty(vec_ty); + + let body = match vec_ty.n_bits() { + 128 => quote! { + let result = vqtbx1q_u8(merge, values, control); + }, + 256 => quote! { + let result = uint8x16x2_t( + vqtbx2q_u8(merge.0, values, control.0), + vqtbx2q_u8(merge.1, values, control.1), + ); + }, + 512 => quote! { + let result = uint8x16x4_t( + vqtbx4q_u8(merge.0, values, control.0), + vqtbx4q_u8(merge.1, values, control.1), + vqtbx4q_u8(merge.2, values, control.2), + vqtbx4q_u8(merge.3, values, control.3), + ); + }, + _ => unreachable!(), + }; + + quote! { + { + crate::kernel!( + #[inline(always)] + fn merge_swizzle( + token: Neon, + values: #vec, + control: #vec, + merge: #vec, + ) -> #vec { + let values: #arch_ty = values.into(); + let control: #arch_ty = control.into(); + let merge: #arch_ty = merge.into(); + #body + #vec { + val: #wrapper(result), + simd: token, + } + } + ); + merge_swizzle(self, #values, #control, #merge) + } + } + } + fn handle_count_ones(&self, op: Op, vec_ty: &VecType) -> TokenStream { let input = match vec_ty.scalar { ScalarType::Unsigned if vec_ty.scalar_bits == 8 => quote! { a.into() }, @@ -794,6 +850,15 @@ impl Level for Neon { } }) } + OpSig::Compress { .. } | OpSig::Expand { .. } => composed_compact_op( + op, + vec_ty, + CompactOptions { + splice_wide_vectors: false, + hardware_popcount: false, + merge_swizzle: Some(Self::compact_merge_swizzle), + }, + ), OpSig::Cvt { target_ty, scalar_bits, diff --git a/fearless_simd_gen/src/mk_wasm.rs b/fearless_simd_gen/src/mk_wasm.rs index d4bdbbc12..2487c7f12 100644 --- a/fearless_simd_gen/src/mk_wasm.rs +++ b/fearless_simd_gen/src/mk_wasm.rs @@ -6,10 +6,10 @@ use quote::{format_ident, quote}; use crate::arch::wasm::{arch_prefix, v128_intrinsic}; use crate::generic::{ - concat_swizzle_dyn_precise_body, count_zeros_method, fallback_method, generic_block_combine, - generic_block_split, generic_classify, generic_mask_set, generic_op_name, - integer_lane_mask_rotate, integer_lane_mask_splat_arg, recursive_swizzle_dyn_precise_body, - reverse_method, reverse_vector_mask_method, + CompactOptions, composed_compact_op, concat_swizzle_dyn_precise_body, count_zeros_method, + fallback_method, generic_block_combine, generic_block_split, generic_classify, + generic_mask_set, generic_op_name, integer_lane_mask_rotate, integer_lane_mask_splat_arg, + recursive_swizzle_dyn_precise_body, reverse_method, reverse_vector_mask_method, }; use crate::level::Level; use crate::ops::{ @@ -1166,6 +1166,15 @@ impl Level for WasmSimd128 { } _ => unreachable!(), }, + OpSig::Compress { .. } | OpSig::Expand { .. } => composed_compact_op( + op, + vec_ty, + CompactOptions { + splice_wide_vectors: true, + hardware_popcount: true, + merge_swizzle: None, + }, + ), OpSig::Cvt { target_ty, scalar_bits, diff --git a/fearless_simd_gen/src/mk_x86.rs b/fearless_simd_gen/src/mk_x86.rs index 180d9cb9c..11325763f 100644 --- a/fearless_simd_gen/src/mk_x86.rs +++ b/fearless_simd_gen/src/mk_x86.rs @@ -8,7 +8,7 @@ use crate::arch::x86::{ }; use crate::generic::{ concat_swizzle_dyn_precise_body, count_zeros_method, fallback_method, generic_block_combine, - generic_block_split, generic_classify, generic_mask_from_bitmask, generic_mask_set, + generic_block_split, generic_classify, generic_mask_from_bitmask, generic_mask_set, generic_op, generic_op_name, integer_lane_mask_rotate, integer_lane_mask_splat_arg, recursive_swizzle_dyn_precise_body, reverse_method, reverse_vector_mask_method, }; @@ -328,6 +328,21 @@ impl Level for X86 { OpSig::SwizzleDynPrecise => self.handle_swizzle_dyn_precise(op, vec_ty), OpSig::ConcatSwizzleDyn => self.handle_concat_swizzle_dyn(op, vec_ty), OpSig::ConcatSwizzleDynPrecise => self.handle_concat_swizzle_dyn_precise(op, vec_ty), + OpSig::Compress { .. } | OpSig::Expand { .. } => { + if *self == Self::Avx512 { + self.handle_avx512_compact_op(op, vec_ty) + } else if matches!(*self, Self::Sse4_2 | Self::Avx2) && vec_ty.len == 16 { + self.handle_x86_compact_16(op, vec_ty) + } else if *self == Self::Avx2 && matches!(sig, OpSig::Compress { .. }) { + self.handle_avx2_shuffle_compact(op, vec_ty) + } else if *self == Self::Avx2 { + self.handle_avx2_sse_expand(op, vec_ty) + } else if *self == Self::Sse4_2 { + self.handle_x86_wide_compact(op, vec_ty) + } else { + generic_op(&op, vec_ty) + } + } OpSig::Cvt { target_ty, scalar_bits, @@ -1225,7 +1240,661 @@ fn interleaved_store_indices(len: usize, block_count: usize) -> Vec { .collect() } +fn shifted_mask_bits(shift: usize) -> TokenStream { + if shift == 0 { + quote! { mask_bits } + } else { + let shift = Literal::usize_unsuffixed(shift); + quote! { mask_bits >> #shift } + } +} + +/// Emit the `PSHUFB` control construction for one 16-byte compact block. +fn compact_16_control(block_bits: TokenStream, suffix: &str) -> (TokenStream, Ident, Ident) { + let low_mask = format_ident!("low_mask{suffix}"); + let high_mask = format_ident!("high_mask{suffix}"); + let low_control = format_ident!("low_control{suffix}"); + let high_control = format_ident!("high_control{suffix}"); + let control = format_ident!("control{suffix}"); + let low_count = format_ident!("low_count{suffix}"); + let setup = quote! { + let #low_mask = (#block_bits) & 0xff; + let #high_mask = (#block_bits) >> 8; + let #low_control = crate::support::COMPRESS_8_CONTROLS[#low_mask]; + let #high_control = crate::support::COMPRESS_8_CONTROLS[#high_mask]; + let #high_control = ((#high_control & 0x7f7f_7f7f_7f7f_7f7f) + + 0x0808_0808_0808_0808) + | (#high_control & 0x8080_8080_8080_8080); + let #control = _mm_set_epi64x(#high_control.cast_signed(), #low_control.cast_signed()); + let #low_count = #low_mask.count_ones() as usize; + }; + (setup, control, low_count) +} + +/// Emit the `PSHUFB` control construction for one 16-byte expand block. +fn expand_16_control(block_bits: TokenStream) -> TokenStream { + quote! { + let low_mask = (#block_bits) & 0xff; + let high_mask = (#block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) + | (high_control & 0x8080_8080_8080_8080); + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + } +} + impl X86 { + fn handle_x86_compact_16(&self, op: Op, vec_ty: &VecType) -> TokenStream { + assert!( + matches!(*self, Self::Sse4_2 | Self::Avx2), + "16-byte compact shuffles require SSE4.2 or AVX2" + ); + assert_eq!(vec_ty.len, 16, "this handler is only for 16-byte vectors"); + let to_bitmask = generic_op_name("to_bitmask", &vec_ty.mask_ty()); + + self.kernel_method(op, vec_ty, |token| match op.sig { + OpSig::Compress { merge } => { + let (setup, control, low_count) = + compact_16_control(quote! { usize::from(mask_bits) }, ""); + let finish = if merge { + quote! { + let count = mask_bits.count_ones() as usize; + let prefix = unsafe { + _mm_load_si128( + core::ptr::from_ref( + &crate::support::COMPACT_PREFIX_MASKS[count], + ) + .cast::<__m128i>(), + ) + }; + _mm_blendv_epi8(merge.into(), compressed, prefix).simd_into(#token) + } + } else { + quote! { compressed.simd_into(#token) } + }; + quote! { + let mask_bits = #token.#to_bitmask(mask) as u16; + #setup + let compacted = _mm_shuffle_epi8(values.into(), #control); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[#low_count]; + let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let compressed = _mm_shuffle_epi8(compacted, splice); + #finish + } + } + OpSig::Expand { merge } => { + let setup = expand_16_control(quote! { usize::from(mask_bits) }); + let finish = if merge { + quote! { + _mm_blendv_epi8(merge.into(), expanded, mask.into()).simd_into(#token) + } + } else { + quote! { expanded.simd_into(#token) } + }; + quote! { + let mask_bits = #token.#to_bitmask(mask) as u16; + #setup + let expanded = _mm_shuffle_epi8(values.into(), control); + #finish + } + } + _ => unreachable!("only compact operations use the x86 implementation"), + }) + } + + fn handle_x86_wide_compact(&self, op: Op, vec_ty: &VecType) -> TokenStream { + assert!(*self == Self::Sse4_2, "wide x86 compact requires SSE4.2"); + assert!( + matches!(vec_ty.len, 32 | 64), + "wide x86 compact requires a 32- or 64-byte vector" + ); + let vec = vec_ty.rust(); + let len = Literal::usize_unsuffixed(vec_ty.len); + let block_count = vec_ty.len / 16; + let to_bitmask = generic_op_name("to_bitmask", &vec_ty.mask_ty()); + let block_value = |name: &str, block: usize| { + let name = format_ident!("{name}"); + match (*self, vec_ty.len) { + (Self::Sse4_2, _) => { + let block = Literal::usize_unsuffixed(block); + quote! { #name.val.0[#block] } + } + (Self::Avx2, 32) if block == 0 => { + quote! { _mm256_castsi256_si128(#name.val.0) } + } + (Self::Avx2, 32) => quote! { _mm256_extracti128_si256::<1>(#name.val.0) }, + (Self::Avx2, 64) => { + let half = Literal::usize_unsuffixed(block / 2); + if block.is_multiple_of(2) { + quote! { _mm256_castsi256_si128(#name.val.0[#half]) } + } else { + quote! { _mm256_extracti128_si256::<1>(#name.val.0[#half]) } + } + } + _ => unreachable!(), + } + }; + + self.kernel_method(op, vec_ty, |token| match op.sig { + OpSig::Compress { merge } => { + let compact_blocks = (0..block_count).map(|block| { + let shifted_mask = shifted_mask_bits(block * 16); + let values = block_value("values", block); + let (setup, control, low_count) = compact_16_control(quote! { block_bits }, ""); + let advance = if merge || block + 1 < block_count { + quote! { output_lane += block_bits.count_ones() as usize; } + } else { + TokenStream::new() + }; + quote! { + let block_bits = (#shifted_mask & 0xffff) as usize; + #setup + let compacted = _mm_shuffle_epi8(#values, #control); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[#low_count]; + let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let compressed = _mm_shuffle_epi8(compacted, splice); + unsafe { + _mm_storeu_si128( + output.as_mut_ptr().add(output_lane).cast::<__m128i>(), + compressed, + ); + } + #advance + } + }); + let finish = if merge { + let blend_blocks = (0..block_count).map(|block| { + let offset = Literal::usize_unsuffixed(block * 16); + let merge = block_value("merge", block); + quote! { + let remaining = output_lane.saturating_sub(#offset).min(16); + let prefix = unsafe { + _mm_load_si128( + core::ptr::from_ref( + &crate::support::COMPACT_PREFIX_MASKS[remaining], + ) + .cast::<__m128i>(), + ) + }; + let compressed = unsafe { + _mm_loadu_si128(output.as_ptr().add(#offset).cast::<__m128i>()) + }; + let blended = _mm_blendv_epi8(#merge, compressed, prefix); + unsafe { + _mm_storeu_si128( + output.as_mut_ptr().add(#offset).cast::<__m128i>(), + blended, + ); + } + } + }); + quote! { #(#blend_blocks)* } + } else { + TokenStream::new() + }; + quote! { + let mask_bits = #token.#to_bitmask(mask); + let mut output = [0u8; #len]; + let mut output_lane = 0; + #(#compact_blocks)* + #finish + #vec::simd_from(#token, output) + } + } + OpSig::Expand { merge } => { + let input = quote! { <[u8; #len]>::from(values) }; + let expand_blocks = (0..block_count).map(|block| { + let shifted_mask = shifted_mask_bits(block * 16); + let offset = Literal::usize_unsuffixed(block * 16); + let setup = expand_16_control(quote! { block_bits }); + let result = if merge { + let merge = block_value("merge", block); + let mask = block_value("mask", block); + quote! { _mm_blendv_epi8(#merge, expanded, #mask) } + } else { + quote! { expanded } + }; + let advance = if block + 1 < block_count { + quote! { input_lane += block_bits.count_ones() as usize; } + } else { + TokenStream::new() + }; + quote! { + let block_bits = (#shifted_mask & 0xffff) as usize; + #setup + let packed = unsafe { + _mm_loadu_si128(input.as_ptr().add(input_lane).cast::<__m128i>()) + }; + let expanded = _mm_shuffle_epi8(packed, control); + let result = #result; + unsafe { + _mm_storeu_si128( + output.as_mut_ptr().add(#offset).cast::<__m128i>(), + result, + ); + } + #advance + } + }); + quote! { + let mask_bits = #token.#to_bitmask(mask); + let input = #input; + let mut output = [0u8; #len]; + let mut input_lane = 0; + #(#expand_blocks)* + #vec::simd_from(#token, output) + } + } + _ => unreachable!("only compact operations use the x86 implementation"), + }) + } + + fn handle_avx2_sse_expand(&self, op: Op, vec_ty: &VecType) -> TokenStream { + assert!( + *self == Self::Avx2, + "lower-width expand helpers require AVX2" + ); + assert!( + matches!(vec_ty.len, 32 | 64), + "lower-width expand helpers require a 32- or 64-byte vector" + ); + assert!( + matches!(op.sig, OpSig::Expand { .. }), + "lower-width expand helpers only implement expand operations" + ); + let vec = vec_ty.rust(); + let len = Literal::usize_unsuffixed(vec_ty.len); + let block_count = vec_ty.len / 16; + + op.simd_trait_kernel_method_with_target_features( + self.token(), + SSE4_2_FEATURES, + vec_ty, + |token| { + let input_setup = quote! { let input_values = <[u8; #len]>::from(values); }; + let input = quote! { input_values.as_ptr() }; + let mask_parts = (0..block_count).map(|block| { + let block_literal = Literal::usize_unsuffixed(block); + let movemask = quote! { + _mm_movemask_epi8(unsafe { + _mm_load_si128( + core::ptr::from_ref(&mask.val.0) + .cast::<__m128i>() + .add(#block_literal), + ) + }) as u64 + }; + if block == 0 { + movemask + } else { + let shift = Literal::usize_unsuffixed(block * 16); + quote! { (#movemask) << #shift } + } + }); + let blocks = (0..block_count).map(|block| { + let shifted_mask = shifted_mask_bits(block * 16); + let offset = Literal::usize_unsuffixed(block * 16); + let setup = expand_16_control(quote! { block_bits }); + let blend = if matches!(op.sig, OpSig::Expand { merge: true }) { + let block = Literal::usize_unsuffixed(block); + quote! { + unsafe { + _mm_blendv_epi8( + _mm_load_si128( + core::ptr::from_ref(&merge.val.0) + .cast::<__m128i>() + .add(#block), + ), + expanded, + _mm_load_si128( + core::ptr::from_ref(&mask.val.0) + .cast::<__m128i>() + .add(#block), + ), + ) + } + } + } else { + quote! { expanded } + }; + let advance = if block + 1 < block_count { + quote! { input_lane += block_bits.count_ones() as usize; } + } else { + TokenStream::new() + }; + quote! { + let block_bits = (#shifted_mask & 0xffff) as usize; + #setup + let packed = unsafe { + _mm_loadu_si128(input.add(input_lane).cast::<__m128i>()) + }; + let expanded = _mm_shuffle_epi8(packed, control); + let result = #blend; + unsafe { + _mm_storeu_si128( + output.as_mut_ptr().add(#offset).cast::<__m128i>(), + result, + ); + } + #advance + } + }); + quote! { + let mask_bits = #(#mask_parts)|*; + #input_setup + let input = #input; + let mut output = [0u8; #len]; + let mut input_lane = 0usize; + #(#blocks)* + #vec::simd_from(#token, output) + } + }, + ) + } + + fn handle_avx2_shuffle_compact(&self, op: Op, vec_ty: &VecType) -> TokenStream { + assert!(*self == Self::Avx2, "wide shuffle compact requires AVX2"); + assert!( + matches!(vec_ty.len, 32 | 64), + "wide shuffle compact requires a 32- or 64-byte vector" + ); + let pair_count = vec_ty.len / 32; + let to_bitmask = generic_op_name("to_bitmask", &vec_ty.mask_ty()); + + self.kernel_method(op, vec_ty, |token| match op.sig { + OpSig::Compress { merge } => { + let vec = vec_ty.rust(); + let empty = if merge { + quote! { merge } + } else { + quote! { #vec::splat(#token, 0) } + }; + let all_bits = if vec_ty.len == 64 { + quote! { u64::MAX } + } else { + let all_bits = Literal::u64_unsuffixed((1_u64 << vec_ty.len) - 1); + quote! { #all_bits } + }; + // The 64-byte path has an additional cross-YMM stitching stage. Empty, full, and + // single-bit masks can bypass nearly all of it and are more than twice as fast in + // the corresponding mask-density benchmarks. The shorter 32-byte path does not + // save enough work to justify these data-dependent branches. + let edge_mask_fast_path = if vec_ty.len == 64 { + quote! { + if mask_bits == 0 || mask_bits == #all_bits { + return if mask_bits == 0 { #empty } else { values }; + } + } + } else { + TokenStream::new() + }; + let single_lane_fast_path = if vec_ty.len == 64 { + let result = if merge { + quote! { + let low = _mm_insert_epi8::<0>( + _mm256_castsi256_si128(merge.val.0[0]), + i32::from(selected), + ); + let first = + _mm256_inserti128_si256::<0>(merge.val.0[0], low); + u8x64 { + val: crate::support::Aligned512([first, merge.val.0[1]]), + simd: #token, + } + } + } else { + quote! { + let first = _mm256_set_epi64x(0, 0, 0, i64::from(selected)); + u8x64 { + val: crate::support::Aligned512([first, _mm256_setzero_si256()]), + simd: #token, + } + } + }; + quote! { + if mask_bits.is_power_of_two() { + let selected = values.as_array()[mask_bits.trailing_zeros() as usize]; + return { #result }; + } + } + } else { + TokenStream::new() + }; + let compact_pairs = (0..pair_count).map(|pair| { + let low_block = pair * 2; + let high_block = low_block + 1; + let low_shifted_mask = shifted_mask_bits(low_block * 16); + let high_shifted_mask = shifted_mask_bits(high_block * 16); + let input = if vec_ty.len == 32 { + quote! { values.val.0 } + } else { + let pair = Literal::usize_unsuffixed(pair); + quote! { values.val.0[#pair] } + }; + let bits0 = format_ident!("block_bits_{low_block}"); + let bits1 = format_ident!("block_bits_{high_block}"); + let splice0 = format_ident!("splice_{low_block}"); + let splice1 = format_ident!("splice_{high_block}"); + let compressed_pair = format_ident!("compressed_{pair}"); + let (setup0, control0, low_count0) = + compact_16_control(quote! { #bits0 }, &format!("_{low_block}")); + let (setup1, control1, low_count1) = + compact_16_control(quote! { #bits1 }, &format!("_{high_block}")); + let output_lane = if vec_ty.len == 32 && merge { + quote! { + let output_lane = + (#bits0.count_ones() + #bits1.count_ones()) as usize; + } + } else { + TokenStream::new() + }; + let finish_pair = quote! { + let splice = unsafe { + _mm256_load_si256( + core::ptr::from_ref( + &crate::support::COMPACT_16_SPLICE_CONTROLS + [#bits0.count_ones() as usize], + ) + .cast::<__m256i>(), + ) + }; + let compressed_low = + _mm256_permute2x128_si256::<0x80>(compressed, compressed); + let compressed_high = + _mm256_permute2x128_si256::<0x11>(compressed, compressed); + let #compressed_pair = _mm256_or_si256( + compressed_low, + _mm256_shuffle_epi8(compressed_high, splice), + ); + #output_lane + }; + quote! { + let #bits0 = (#low_shifted_mask & 0xffff) as usize; + #setup0 + + let #bits1 = (#high_shifted_mask & 0xffff) as usize; + #setup1 + + let control = _mm256_set_m128i(#control1, #control0); + let compacted = _mm256_shuffle_epi8(#input, control); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[#low_count0]; + let #splice0 = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[#low_count1]; + let #splice1 = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let splice = _mm256_set_m128i(#splice1, #splice0); + let compressed = _mm256_shuffle_epi8(compacted, splice); + #finish_pair + } + }); + let stitch_pairs = if vec_ty.len == 64 { + let output_lane = if merge { + quote! { let output_lane = mask_bits.count_ones() as usize; } + } else { + TokenStream::new() + }; + quote! { + let first_count = + (block_bits_0.count_ones() + block_bits_1.count_ones()) as usize; + let controls = + &crate::support::COMPACT_32_STITCH_CONTROLS[first_count]; + let left_same = unsafe { + _mm256_load_si256( + core::ptr::from_ref(&controls.left_same).cast::<__m256i>(), + ) + }; + let left_cross = unsafe { + _mm256_load_si256( + core::ptr::from_ref(&controls.left_cross).cast::<__m256i>(), + ) + }; + let right_same = unsafe { + _mm256_load_si256( + core::ptr::from_ref(&controls.right_same).cast::<__m256i>(), + ) + }; + let right_cross = unsafe { + _mm256_load_si256( + core::ptr::from_ref(&controls.right_cross).cast::<__m256i>(), + ) + }; + + let low_to_high = + _mm256_permute2x128_si256::<0x08>(compressed_1, compressed_1); + let shifted_left = _mm256_or_si256( + _mm256_shuffle_epi8(compressed_1, left_same), + _mm256_shuffle_epi8(low_to_high, left_cross), + ); + let compressed_0 = _mm256_or_si256(compressed_0, shifted_left); + + let high_to_low = + _mm256_permute2x128_si256::<0x81>(compressed_1, compressed_1); + let compressed_1 = _mm256_or_si256( + _mm256_shuffle_epi8(compressed_1, right_same), + _mm256_shuffle_epi8(high_to_low, right_cross), + ); + #output_lane + } + } else { + TokenStream::new() + }; + let result_chunks = (0..pair_count).map(|pair| { + let offset = Literal::usize_unsuffixed(pair * 32); + let compressed = format_ident!("compressed_{pair}"); + let result = format_ident!("result_{pair}"); + if merge { + let remaining = if pair == 0 { + quote! { output_lane.min(32) } + } else { + quote! { output_lane.saturating_sub(#offset).min(32) } + }; + let merge = if vec_ty.len == 32 { + quote! { merge.val.0 } + } else { + let pair = Literal::usize_unsuffixed(pair); + quote! { merge.val.0[#pair] } + }; + quote! { + let prefix = unsafe { + _mm256_load_si256( + core::ptr::from_ref( + &crate::support::COMPACT_PREFIX_MASKS[#remaining], + ) + .cast::<__m256i>(), + ) + }; + let #result = _mm256_blendv_epi8(#merge, #compressed, prefix); + } + } else { + quote! { + let #result = #compressed; + } + } + }); + let finish = if vec_ty.len == 32 { + quote! { result_0.simd_into(#token) } + } else { + quote! { + u8x64 { + val: crate::support::Aligned512([result_0, result_1]), + simd: #token, + } + } + }; + quote! { + let mask_bits = #token.#to_bitmask(mask); + #edge_mask_fast_path + #single_lane_fast_path + #(#compact_pairs)* + #stitch_pairs + #(#result_chunks)* + #finish + } + } + _ => unreachable!("only compact operations use the AVX2 implementation"), + }) + } + + fn handle_avx512_compact_op(&self, op: Op, vec_ty: &VecType) -> TokenStream { + assert!( + *self == Self::Avx512, + "compact byte intrinsics require AVX-512" + ); + assert_eq!( + vec_ty.scalar, + ScalarType::Unsigned, + "compact byte intrinsics require unsigned vectors" + ); + assert_eq!( + vec_ty.scalar_bits, 8, + "compact byte intrinsics require byte lanes" + ); + + let prefix = match vec_ty.len { + 16 => "_mm", + 32 => "_mm256", + 64 => "_mm512", + _ => unreachable!(), + }; + let mask_ty = match vec_ty.len { + 16 => quote! { u16 }, + 32 => quote! { u32 }, + 64 => quote! { u64 }, + _ => unreachable!(), + }; + let mask_bits = avx512_mask_bits_expr(quote! { mask }); + + self.kernel_method(op, vec_ty, |token| match op.sig { + OpSig::Compress { merge: false } => { + let intrinsic = format_ident!("{prefix}_maskz_compress_epi8"); + quote! { + #intrinsic(#mask_bits as #mask_ty, values.into()).simd_into(#token) + } + } + OpSig::Compress { merge: true } => { + let intrinsic = format_ident!("{prefix}_mask_compress_epi8"); + quote! { + #intrinsic(merge.into(), #mask_bits as #mask_ty, values.into()).simd_into(#token) + } + } + OpSig::Expand { merge: false } => { + let intrinsic = format_ident!("{prefix}_maskz_expand_epi8"); + quote! { + #intrinsic(#mask_bits as #mask_ty, values.into()).simd_into(#token) + } + } + OpSig::Expand { merge: true } => { + let intrinsic = format_ident!("{prefix}_mask_expand_epi8"); + quote! { + #intrinsic(merge.into(), #mask_bits as #mask_ty, values.into()).simd_into(#token) + } + } + _ => unreachable!(), + }) + } + fn handle_count_ones(&self, op: Op, vec_ty: &VecType) -> TokenStream { match *self { Self::Avx512 => { diff --git a/fearless_simd_gen/src/ops.rs b/fearless_simd_gen/src/ops.rs index cfb406e50..7da96d815 100644 --- a/fearless_simd_gen/src/ops.rs +++ b/fearless_simd_gen/src/ops.rs @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 OR MIT use anyhow::{Context, anyhow}; -use proc_macro2::{Ident, Span, TokenStream}; +use proc_macro2::{Ident, Literal, Span, TokenStream}; use quote::{format_ident, quote}; use std::fmt::Write; @@ -108,6 +108,10 @@ pub(crate) enum OpSig { /// Takes two vectors and a same-width byte-index vector, and returns the original vector type with its bytes /// dynamically selected from the concatenation of both vectors. Out-of-range indices produce zero. ConcatSwizzleDynPrecise, + /// Takes a byte vector, a same-width byte-lane mask, and optionally a merge vector. + Compress { merge: bool }, + /// Takes a byte vector, a same-width byte-lane mask, and optionally a merge vector. + Expand { merge: bool }, /// Takes a single argument of the source vector type, and returns a vector type of the target scalar type and the /// same length. Cvt { @@ -280,6 +284,46 @@ impl Op { } } + /// Generate a SIMD trait method whose local kernel uses an explicit target-feature set rather + /// than all features associated with the method's backend. This allows a higher backend to + /// reuse a lower backend's instruction formulation when wider instructions do not improve it. + pub(crate) fn simd_trait_kernel_method_with_target_features( + &self, + method_level: Ident, + target_features: &str, + vec_ty: &VecType, + body: impl FnOnce(&Ident) -> TokenStream, + ) -> TokenStream { + assert!( + !matches!(self.sig, OpSig::Slide { .. }), + "kernel! does not support const-generic methods" + ); + + let method_sig = self.simd_trait_method_sig(vec_ty); + let token = Ident::new("token", Span::call_site()); + let target_features = Literal::string(target_features); + let kernel_body = body(&token); + let sig = self.simd_trait_sig_parts(vec_ty, quote! { #method_level }); + let arg_decls = sig.arg_decls(); + let call_args = &sig.arg_names; + let ret = &sig.ret; + + quote! { + #method_sig { + #[inline] + #[target_feature(enable = #target_features)] + unsafe fn kernel( + #token: #method_level #(, #arg_decls)* + ) -> #ret { + #kernel_body + } + + // SAFETY: the backend token guarantees this kernel's feature subset. + unsafe { kernel(self #(, #call_args)*) } + } + } + } + fn simd_trait_sig_parts(&self, vec_ty: &VecType, simd_ty: TokenStream) -> SimdTraitSigParts { let ty = vec_ty.rust(); let arg_names = self @@ -352,6 +396,17 @@ impl Op { vec, ) } + OpSig::Compress { merge: false } | OpSig::Expand { merge: false } => { + let mask = vec_ty.mask_ty().rust(); + (vec![vec.clone(), quote! { #mask<#simd_ty> }], vec) + } + OpSig::Compress { merge: true } | OpSig::Expand { merge: true } => { + let mask = vec_ty.mask_ty().rust(); + ( + vec![vec.clone(), quote! { #mask<#simd_ty> }, vec.clone()], + vec, + ) + } OpSig::Cvt { target_ty, scalar_bits, @@ -412,7 +467,10 @@ impl Op { let arg1 = &arg_names[1]; quote! { (#arg0: S, #arg1: Self::Element) -> Self } } - OpSig::LoadInterleaved { .. } | OpSig::StoreInterleaved { .. } => { + OpSig::LoadInterleaved { .. } + | OpSig::StoreInterleaved { .. } + | OpSig::Compress { .. } + | OpSig::Expand { .. } => { return None; } OpSig::MaskFromBitmask | OpSig::MaskToBitmask | OpSig::MaskSet => return None, @@ -673,6 +731,37 @@ const BASE_OPS: &[Op] = &[ ), ]; +const U8_ONLY_OPS: &[Op] = &[ + Op::new( + "compress", + OpKind::AssociatedOnly, + OpSig::Compress { merge: false }, + "Compact the bytes selected by `{arg1}` into consecutive low lanes.\n\n\ + Lanes above the number of selected bytes are zero.", + ), + Op::new( + "compress_merge", + OpKind::AssociatedOnly, + OpSig::Compress { merge: true }, + "Compact the bytes selected by `{arg1}` into consecutive low lanes.\n\n\ + Lanes above the number of selected bytes retain the corresponding values from `{arg2}`.", + ), + Op::new( + "expand", + OpKind::AssociatedOnly, + OpSig::Expand { merge: false }, + "Expand consecutive low bytes from `{arg0}` into the lanes selected by `{arg1}`.\n\n\ + Unselected lanes are zero.", + ), + Op::new( + "expand_merge", + OpKind::AssociatedOnly, + OpSig::Expand { merge: true }, + "Expand consecutive low bytes from `{arg0}` into the lanes selected by `{arg1}`.\n\n\ + Unselected lanes retain the corresponding values from `{arg2}`.", + ), +]; + const COMMON_BASE_OPS: &[Op] = &[ Op::new( "reduce_max", @@ -1541,6 +1630,10 @@ pub(crate) fn ops_for_type(ty: &VecType) -> Vec { } } + if ty.scalar == ScalarType::Unsigned && ty.scalar_bits == 8 { + ops.extend_from_slice(U8_ONLY_OPS); + } + if let Some(combined_ty) = ty.combine_operand() { ops.push(Op::new( "combine", @@ -1857,6 +1950,8 @@ impl OpSig { | Self::SwizzleDynPrecise | Self::ConcatSwizzleDyn | Self::ConcatSwizzleDynPrecise + | Self::Compress { .. } + | Self::Expand { .. } | Self::Slide { granularity: SlideGranularity::AcrossBlocks, .. @@ -1907,6 +2002,10 @@ impl OpSig { &["a", "indices"] } Self::ConcatSwizzleDyn | Self::ConcatSwizzleDynPrecise => &["a", "b", "indices"], + Self::Compress { merge: false } | Self::Expand { merge: false } => &["values", "mask"], + Self::Compress { merge: true } | Self::Expand { merge: true } => { + &["values", "mask", "merge"] + } Self::Binary | Self::Compare | Self::Combine { .. } @@ -1930,7 +2029,9 @@ impl OpSig { | Self::StoreInterleaved { .. } | Self::MaskFromBitmask | Self::MaskToBitmask - | Self::MaskSet => &[], + | Self::MaskSet + | Self::Compress { .. } + | Self::Expand { .. } => &[], Self::Unary | Self::Reduce { .. } | Self::RotateElements { .. } @@ -2009,6 +2110,8 @@ impl OpSig { | Self::SwizzleDynWithinBlocks | Self::SwizzleDyn | Self::SwizzleDynPrecise + | Self::Compress { .. } + | Self::Expand { .. } | Self::Slide { .. } => return None, }; Some(args) diff --git a/fearless_simd_tests/tests/harness/ops/compress.rs b/fearless_simd_tests/tests/harness/ops/compress.rs new file mode 100644 index 000000000..c47a5caed --- /dev/null +++ b/fearless_simd_tests/tests/harness/ops/compress.rs @@ -0,0 +1,62 @@ +// Copyright 2026 the Fearless_SIMD Authors +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use fearless_simd::*; +use fearless_simd_dev_macros::simd_test; + +#[simd_test] +fn compress_all_widths(simd: S) { + macro_rules! check_width { + ($bytes:ident, $mask:ident, $lanes:literal, $compress:ident, $compress_merge:ident) => {{ + let values: [u8; $lanes] = core::array::from_fn(|lane| (lane * 3 + 1) as u8); + let merge: [u8; $lanes] = core::array::from_fn(|lane| 0xe0_u8.wrapping_add(lane as u8)); + let patterned_mask: u64 = (0..$lanes) + .filter(|lane| lane % 3 == 0 || lane % 7 == 2) + .fold(0, |mask, lane| mask | (1_u64 << lane)); + let values_vec = $bytes::simd_from(simd, values); + let merge_vec = $bytes::simd_from(simd, merge); + let all_lanes = u64::MAX >> (64 - $lanes); + + let repeated_byte_masks = (0_u64..=255).map(|byte_mask| { + (0..($lanes / 8)).fold(0, |mask, block| mask | (byte_mask << (block * 8))) + }); + let stitch_masks = ($lanes == 64).then(|| { + (0..=32).map(|low_count| { + let low = if low_count == 32 { + u64::from(u32::MAX) + } else { + (1_u64 << low_count) - 1 + }; + low | (0xa5a5_a5a5_u64 << 32) + }) + }); + for mask_bits in [0, 1, patterned_mask, all_lanes] + .into_iter() + .chain(repeated_byte_masks) + .chain(stitch_masks.into_iter().flatten()) + { + let mask = $mask::from_bitmask(simd, mask_bits); + let mut expected = [0; $lanes]; + let mut expected_merge = merge; + let mut output_lane = 0; + for (input_lane, value) in values.into_iter().enumerate() { + if mask_bits & (1_u64 << input_lane) != 0 { + expected[output_lane] = value; + expected_merge[output_lane] = value; + output_lane += 1; + } + } + + assert_eq!(*simd.$compress(values_vec, mask), expected); + assert_eq!( + *simd.$compress_merge(values_vec, mask, merge_vec), + expected_merge + ); + } + }}; + } + + check_width!(u8x16, mask8x16, 16, compress_u8x16, compress_merge_u8x16); + check_width!(u8x32, mask8x32, 32, compress_u8x32, compress_merge_u8x32); + check_width!(u8x64, mask8x64, 64, compress_u8x64, compress_merge_u8x64); +} diff --git a/fearless_simd_tests/tests/harness/ops/expand.rs b/fearless_simd_tests/tests/harness/ops/expand.rs new file mode 100644 index 000000000..a1f6be9f4 --- /dev/null +++ b/fearless_simd_tests/tests/harness/ops/expand.rs @@ -0,0 +1,51 @@ +// Copyright 2026 the Fearless_SIMD Authors +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use fearless_simd::*; +use fearless_simd_dev_macros::simd_test; + +#[simd_test] +fn expand_all_widths(simd: S) { + macro_rules! check_width { + ($bytes:ident, $mask:ident, $lanes:literal, $expand:ident, $expand_merge:ident) => {{ + let values: [u8; $lanes] = core::array::from_fn(|lane| (lane * 3 + 1) as u8); + let merge: [u8; $lanes] = core::array::from_fn(|lane| 0xe0_u8.wrapping_add(lane as u8)); + let patterned_mask: u64 = (0..$lanes) + .filter(|lane| lane % 3 == 0 || lane % 7 == 2) + .fold(0, |mask, lane| mask | (1_u64 << lane)); + let values_vec = $bytes::simd_from(simd, values); + let merge_vec = $bytes::simd_from(simd, merge); + let all_lanes = u64::MAX >> (64 - $lanes); + + let repeated_byte_masks = (0_u64..=255).map(|byte_mask| { + (0..($lanes / 8)).fold(0, |mask, block| mask | (byte_mask << (block * 8))) + }); + for mask_bits in [0, 1, patterned_mask, all_lanes] + .into_iter() + .chain(repeated_byte_masks) + { + let mask = $mask::from_bitmask(simd, mask_bits); + let mut expected = [0; $lanes]; + let mut expected_merge = merge; + let mut input_lane = 0; + for lane in 0..$lanes { + if mask_bits & (1_u64 << lane) != 0 { + expected[lane] = values[input_lane]; + expected_merge[lane] = values[input_lane]; + input_lane += 1; + } + } + + assert_eq!(*simd.$expand(values_vec, mask), expected); + assert_eq!( + *simd.$expand_merge(values_vec, mask, merge_vec), + expected_merge + ); + } + }}; + } + + check_width!(u8x16, mask8x16, 16, expand_u8x16, expand_merge_u8x16); + check_width!(u8x32, mask8x32, 32, expand_u8x32, expand_merge_u8x32); + check_width!(u8x64, mask8x64, 64, expand_u8x64, expand_merge_u8x64); +} diff --git a/fearless_simd_tests/tests/harness/ops/mod.rs b/fearless_simd_tests/tests/harness/ops/mod.rs index a84d9881f..b7b8c34a3 100644 --- a/fearless_simd_tests/tests/harness/ops/mod.rs +++ b/fearless_simd_tests/tests/harness/ops/mod.rs @@ -17,6 +17,7 @@ mod bitcast; mod block_splat; mod ceil; mod combine; +mod compress; mod concat_swizzle_dyn; mod concat_swizzle_dyn_precise; mod copysign; @@ -34,6 +35,7 @@ mod cvt_u64; mod cvt_u64_precise; mod deinterleave; mod div; +mod expand; mod floor; mod fp_classify; mod fract; From 261407e8a0b9c259797735a9fab687a444a7e961 Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Wed, 30 Sep 2026 10:59:26 +0100 Subject: [PATCH 2/8] Route wider compress/expand ops through byte-level ones, so that all SimdBase vectors can implement compress/expand --- fearless_simd/src/generated/avx2.rs | 2486 ++++++++++++---- fearless_simd/src/generated/avx512.rs | 2511 +++++++++++++++-- fearless_simd/src/generated/fallback.rs | 1472 +++++++++- fearless_simd/src/generated/neon.rs | 1600 ++++++++++- fearless_simd/src/generated/simd_trait.rs | 602 +++- fearless_simd/src/generated/simd_types.rs | 540 ++++ fearless_simd/src/generated/sse2.rs | 1472 +++++++++- fearless_simd/src/generated/sse4_2.rs | 1698 +++++++++-- fearless_simd/src/generated/wasm.rs | 1552 +++++++++- fearless_simd_gen/src/generic.rs | 28 + fearless_simd_gen/src/mk_fallback.rs | 8 +- fearless_simd_gen/src/mk_neon.rs | 9 +- fearless_simd_gen/src/mk_wasm.rs | 12 +- fearless_simd_gen/src/mk_x86.rs | 53 +- fearless_simd_gen/src/ops.rs | 62 +- .../tests/harness/ops/compress.rs | 2080 ++++++++++++++ 16 files changed, 14694 insertions(+), 1491 deletions(-) diff --git a/fearless_simd/src/generated/avx2.rs b/fearless_simd/src/generated/avx2.rs index e01c2d6ac..23aa97bed 100644 --- a/fearless_simd/src/generated/avx2.rs +++ b/fearless_simd/src/generated/avx2.rs @@ -166,6 +166,56 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn compress_f32x4(self, values: f32x4, mask: mask32x4) -> f32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_f32x4( + self, + values: f32x4, + mask: mask32x4, + merge: f32x4, + ) -> f32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_f32x4(self, values: f32x4, mask: mask32x4) -> f32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_f32x4( + self, + values: f32x4, + mask: mask32x4, + merge: f32x4, + ) -> f32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn neg_f32x4(self, a: f32x4) -> f32x4 { crate::kernel!( #[inline(always)] @@ -808,6 +858,56 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn compress_i8x16(self, values: i8x16, mask: mask8x16) -> i8x16 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i8x16( + self, + values: i8x16, + mask: mask8x16, + merge: i8x16, + ) -> i8x16 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i8x16(self, values: i8x16, mask: mask8x16) -> i8x16 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i8x16( + self, + values: i8x16, + mask: mask8x16, + merge: i8x16, + ) -> i8x16 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_i8x16(self, a: i8x16) -> i8x16 { crate::kernel!( #[inline(always)] @@ -1439,6 +1539,121 @@ impl Simd for Avx2 { Bytes::from_bytes(result_bytes) } #[inline(always)] + fn compress_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: u8x16, mask: mask8x16) -> u8x16 { + let mask_bits = token.to_bitmask_mask8x16(mask) as u16; + let low_mask = (usize::from(mask_bits)) & 0xff; + let high_mask = (usize::from(mask_bits)) >> 8; + let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask]; + let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask]; + let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + 0x0808_0808_0808_0808) + | (high_control & 0x8080_8080_8080_8080); + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let low_count = low_mask.count_ones() as usize; + let compacted = _mm_shuffle_epi8(values.into(), control); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count]; + let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let compressed = _mm_shuffle_epi8(compacted, splice); + compressed.simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_u8x16( + self, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + let mask_bits = token.to_bitmask_mask8x16(mask) as u16; + let low_mask = (usize::from(mask_bits)) & 0xff; + let high_mask = (usize::from(mask_bits)) >> 8; + let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask]; + let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask]; + let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + 0x0808_0808_0808_0808) + | (high_control & 0x8080_8080_8080_8080); + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let low_count = low_mask.count_ones() as usize; + let compacted = _mm_shuffle_epi8(values.into(), control); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count]; + let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let compressed = _mm_shuffle_epi8(compacted, splice); + let count = mask_bits.count_ones() as usize; + let prefix = unsafe { + _mm_load_si128( + core::ptr::from_ref(&crate::support::COMPACT_PREFIX_MASKS[count]) + .cast::<__m128i>(), + ) + }; + _mm_blendv_epi8(merge.into(), compressed, prefix).simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: u8x16, mask: mask8x16) -> u8x16 { + let mask_bits = token.to_bitmask_mask8x16(mask) as u16; + let low_mask = (usize::from(mask_bits)) & 0xff; + let high_mask = (usize::from(mask_bits)) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) + | (high_control & 0x8080_8080_8080_8080); + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let expanded = _mm_shuffle_epi8(values.into(), control); + expanded.simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_u8x16( + self, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + let mask_bits = token.to_bitmask_mask8x16(mask) as u16; + let low_mask = (usize::from(mask_bits)) & 0xff; + let high_mask = (usize::from(mask_bits)) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) + | (high_control & 0x8080_8080_8080_8080); + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let expanded = _mm_shuffle_epi8(values.into(), control); + _mm_blendv_epi8(merge.into(), expanded, mask.into()).simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn count_ones_u8x16(self, a: u8x16) -> u8x16 { crate::kernel!( #[inline(always)] @@ -1845,175 +2060,60 @@ impl Simd for Avx2 { kernel(self, a, b, c) } #[inline(always)] - fn compress_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16 { + fn combine_u8x16(self, a: u8x16, b: u8x16) -> u8x32 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, values: u8x16, mask: mask8x16) -> u8x16 { - let mask_bits = token.to_bitmask_mask8x16(mask) as u16; - let low_mask = (usize::from(mask_bits)) & 0xff; - let high_mask = (usize::from(mask_bits)) >> 8; - let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask]; - let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask]; - let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + 0x0808_0808_0808_0808) - | (high_control & 0x8080_8080_8080_8080); - let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); - let low_count = low_mask.count_ones() as usize; - let compacted = _mm_shuffle_epi8(values.into(), control); - let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count]; - let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64); - let compressed = _mm_shuffle_epi8(compacted, splice); - compressed.simd_into(token) + fn kernel(token: Avx2, a: u8x16, b: u8x16) -> u8x32 { + _mm256_setr_m128i(a.into(), b.into()).simd_into(token) } ); - kernel(self, values, mask) + kernel(self, a, b) } #[inline(always)] - fn compress_merge_u8x16( - self, - values: u8x16, - mask: mask8x16, - merge: u8x16, - ) -> u8x16 { + fn load_four_interleaved_u8x16(self, src: &[u8; 64usize]) -> [u8x16; 4usize] { crate::kernel!( #[inline(always)] - fn kernel( - token: Avx2, - values: u8x16, - mask: mask8x16, - merge: u8x16, - ) -> u8x16 { - let mask_bits = token.to_bitmask_mask8x16(mask) as u16; - let low_mask = (usize::from(mask_bits)) & 0xff; - let high_mask = (usize::from(mask_bits)) >> 8; - let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask]; - let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask]; - let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + 0x0808_0808_0808_0808) - | (high_control & 0x8080_8080_8080_8080); - let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); - let low_count = low_mask.count_ones() as usize; - let compacted = _mm_shuffle_epi8(values.into(), control); - let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count]; - let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64); - let compressed = _mm_shuffle_epi8(compacted, splice); - let count = mask_bits.count_ones() as usize; - let prefix = unsafe { - _mm_load_si128( - core::ptr::from_ref(&crate::support::COMPACT_PREFIX_MASKS[count]) - .cast::<__m128i>(), - ) + fn kernel(token: Avx2, src: &[u8; 64usize]) -> [u8x16; 4usize] { + let (chunks, []) = src.as_chunks::<16usize>() else { + unreachable!() }; - _mm_blendv_epi8(merge.into(), compressed, prefix).simd_into(token) + let v0: __m128i = + crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[0]); + let v1: __m128i = + crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[1]); + let v2: __m128i = + crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[2]); + let v3: __m128i = + crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[3]); + let mask = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); + let v0 = _mm_shuffle_epi8(v0, mask); + let v1 = _mm_shuffle_epi8(v1, mask); + let v2 = _mm_shuffle_epi8(v2, mask); + let v3 = _mm_shuffle_epi8(v3, mask); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + [ + out0.simd_into(token), + out1.simd_into(token), + out2.simd_into(token), + out3.simd_into(token), + ] } ); - kernel(self, values, mask, merge) + kernel(self, src) } #[inline(always)] - fn expand_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, values: u8x16, mask: mask8x16) -> u8x16 { - let mask_bits = token.to_bitmask_mask8x16(mask) as u16; - let low_mask = (usize::from(mask_bits)) & 0xff; - let high_mask = (usize::from(mask_bits)) >> 8; - let low_count = low_mask.count_ones() as u64; - let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; - let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; - let high_base = low_count * 0x0101_0101_0101_0101; - let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) - | (high_control & 0x8080_8080_8080_8080); - let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); - let expanded = _mm_shuffle_epi8(values.into(), control); - expanded.simd_into(token) - } - ); - kernel(self, values, mask) - } - #[inline(always)] - fn expand_merge_u8x16( - self, - values: u8x16, - mask: mask8x16, - merge: u8x16, - ) -> u8x16 { - crate::kernel!( - #[inline(always)] - fn kernel( - token: Avx2, - values: u8x16, - mask: mask8x16, - merge: u8x16, - ) -> u8x16 { - let mask_bits = token.to_bitmask_mask8x16(mask) as u16; - let low_mask = (usize::from(mask_bits)) & 0xff; - let high_mask = (usize::from(mask_bits)) >> 8; - let low_count = low_mask.count_ones() as u64; - let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; - let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; - let high_base = low_count * 0x0101_0101_0101_0101; - let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) - | (high_control & 0x8080_8080_8080_8080); - let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); - let expanded = _mm_shuffle_epi8(values.into(), control); - _mm_blendv_epi8(merge.into(), expanded, mask.into()).simd_into(token) - } - ); - kernel(self, values, mask, merge) - } - #[inline(always)] - fn combine_u8x16(self, a: u8x16, b: u8x16) -> u8x32 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, a: u8x16, b: u8x16) -> u8x32 { - _mm256_setr_m128i(a.into(), b.into()).simd_into(token) - } - ); - kernel(self, a, b) - } - #[inline(always)] - fn load_four_interleaved_u8x16(self, src: &[u8; 64usize]) -> [u8x16; 4usize] { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, src: &[u8; 64usize]) -> [u8x16; 4usize] { - let (chunks, []) = src.as_chunks::<16usize>() else { - unreachable!() - }; - let v0: __m128i = - crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[0]); - let v1: __m128i = - crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[1]); - let v2: __m128i = - crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[2]); - let v3: __m128i = - crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[3]); - let mask = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); - let v0 = _mm_shuffle_epi8(v0, mask); - let v1 = _mm_shuffle_epi8(v1, mask); - let v2 = _mm_shuffle_epi8(v2, mask); - let v3 = _mm_shuffle_epi8(v3, mask); - let tmp0 = _mm_unpacklo_epi32(v0, v1); - let tmp1 = _mm_unpackhi_epi32(v0, v1); - let tmp2 = _mm_unpacklo_epi32(v2, v3); - let tmp3 = _mm_unpackhi_epi32(v2, v3); - let out0 = _mm_unpacklo_epi64(tmp0, tmp2); - let out1 = _mm_unpackhi_epi64(tmp0, tmp2); - let out2 = _mm_unpacklo_epi64(tmp1, tmp3); - let out3 = _mm_unpackhi_epi64(tmp1, tmp3); - [ - out0.simd_into(token), - out1.simd_into(token), - out2.simd_into(token), - out3.simd_into(token), - ] - } - ); - kernel(self, src) - } - #[inline(always)] - fn store_four_interleaved_u8x16( - self, - vectors: [u8x16; 4usize], - dest: &mut [u8; 64usize], - ) -> () { + fn store_four_interleaved_u8x16( + self, + vectors: [u8x16; 4usize], + dest: &mut [u8; 64usize], + ) -> () { crate::kernel!( #[inline(always)] fn kernel(token: Avx2, vectors: [u8x16; 4usize], dest: &mut [u8; 64usize]) -> () { @@ -2337,6 +2437,56 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn compress_i16x8(self, values: i16x8, mask: mask16x8) -> i16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i16x8( + self, + values: i16x8, + mask: mask16x8, + merge: i16x8, + ) -> i16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i16x8(self, values: i16x8, mask: mask16x8) -> i16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i16x8( + self, + values: i16x8, + mask: mask16x8, + merge: i16x8, + ) -> i16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_i16x8(self, a: i16x8) -> i16x8 { crate::kernel!( #[inline(always)] @@ -2885,6 +3035,56 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn compress_u16x8(self, values: u16x8, mask: mask16x8) -> u16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u16x8( + self, + values: u16x8, + mask: mask16x8, + merge: u16x8, + ) -> u16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u16x8(self, values: u16x8, mask: mask16x8) -> u16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u16x8( + self, + values: u16x8, + mask: mask16x8, + merge: u16x8, + ) -> u16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_u16x8(self, a: u16x8) -> u16x8 { crate::kernel!( #[inline(always)] @@ -3674,6 +3874,56 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn compress_i32x4(self, values: i32x4, mask: mask32x4) -> i32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i32x4( + self, + values: i32x4, + mask: mask32x4, + merge: i32x4, + ) -> i32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i32x4(self, values: i32x4, mask: mask32x4) -> i32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i32x4( + self, + values: i32x4, + mask: mask32x4, + merge: i32x4, + ) -> i32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_i32x4(self, a: i32x4) -> i32x4 { crate::kernel!( #[inline(always)] @@ -4213,6 +4463,56 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn compress_u32x4(self, values: u32x4, mask: mask32x4) -> u32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u32x4( + self, + values: u32x4, + mask: mask32x4, + merge: u32x4, + ) -> u32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u32x4(self, values: u32x4, mask: mask32x4) -> u32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u32x4( + self, + values: u32x4, + mask: mask32x4, + merge: u32x4, + ) -> u32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_u32x4(self, a: u32x4) -> u32x4 { crate::kernel!( #[inline(always)] @@ -4981,6 +5281,56 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn compress_f64x2(self, values: f64x2, mask: mask64x2) -> f64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_f64x2( + self, + values: f64x2, + mask: mask64x2, + merge: f64x2, + ) -> f64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_f64x2(self, values: f64x2, mask: mask64x2) -> f64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_f64x2( + self, + values: f64x2, + mask: mask64x2, + merge: f64x2, + ) -> f64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn neg_f64x2(self, a: f64x2) -> f64x2 { crate::kernel!( #[inline(always)] @@ -5623,6 +5973,56 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn compress_i64x2(self, values: i64x2, mask: mask64x2) -> i64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i64x2( + self, + values: i64x2, + mask: mask64x2, + merge: i64x2, + ) -> i64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i64x2(self, values: i64x2, mask: mask64x2) -> i64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i64x2( + self, + values: i64x2, + mask: mask64x2, + merge: i64x2, + ) -> i64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_i64x2(self, a: i64x2) -> i64x2 { crate::kernel!( #[inline(always)] @@ -6120,6 +6520,56 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn compress_u64x2(self, values: u64x2, mask: mask64x2) -> u64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u64x2( + self, + values: u64x2, + mask: mask64x2, + merge: u64x2, + ) -> u64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u64x2(self, values: u64x2, mask: mask64x2) -> u64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u64x2( + self, + values: u64x2, + mask: mask64x2, + merge: u64x2, + ) -> u64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_u64x2(self, a: u64x2) -> u64x2 { crate::kernel!( #[inline(always)] @@ -6847,6 +7297,56 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn compress_f32x8(self, values: f32x8, mask: mask32x8) -> f32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_f32x8( + self, + values: f32x8, + mask: mask32x8, + merge: f32x8, + ) -> f32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_f32x8(self, values: f32x8, mask: mask32x8) -> f32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_f32x8( + self, + values: f32x8, + mask: mask32x8, + merge: f32x8, + ) -> f32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn neg_f32x8(self, a: f32x8) -> f32x8 { crate::kernel!( #[inline(always)] @@ -7380,22 +7880,72 @@ impl Simd for Avx2 { }) } #[inline(always)] - fn count_ones_i8x32(self, a: i8x32) -> i8x32 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, a: i8x32) -> i8x32 { - let value = a.into(); - let nibble_mask = _mm256_set1_epi8(0x0f); - let lookup = _mm256_setr_epi8( - 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, - 2, 3, 2, 3, 3, 4, - ); - let low = _mm256_and_si256(value, nibble_mask); - let high = _mm256_and_si256(_mm256_srli_epi16::<4>(value), nibble_mask); - let byte_counts = _mm256_add_epi8( - _mm256_shuffle_epi8(lookup, low), - _mm256_shuffle_epi8(lookup, high), - ); + fn compress_i8x32(self, values: i8x32, mask: mask8x32) -> i8x32 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i8x32( + self, + values: i8x32, + mask: mask8x32, + merge: i8x32, + ) -> i8x32 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i8x32(self, values: i8x32, mask: mask8x32) -> i8x32 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i8x32( + self, + values: i8x32, + mask: mask8x32, + merge: i8x32, + ) -> i8x32 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn count_ones_i8x32(self, a: i8x32) -> i8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i8x32) -> i8x32 { + let value = a.into(); + let nibble_mask = _mm256_set1_epi8(0x0f); + let lookup = _mm256_setr_epi8( + 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, + 2, 3, 2, 3, 3, 4, + ); + let low = _mm256_and_si256(value, nibble_mask); + let high = _mm256_and_si256(_mm256_srli_epi16::<4>(value), nibble_mask); + let byte_counts = _mm256_add_epi8( + _mm256_shuffle_epi8(lookup, low), + _mm256_shuffle_epi8(lookup, high), + ); byte_counts.simd_into(token) } ); @@ -7998,74 +8548,331 @@ impl Simd for Avx2 { kernel(self, a, b, indices) } #[inline(always)] - fn count_ones_u8x32(self, a: u8x32) -> u8x32 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, a: u8x32) -> u8x32 { - let value = a.into(); - let nibble_mask = _mm256_set1_epi8(0x0f); - let lookup = _mm256_setr_epi8( - 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, - 2, 3, 2, 3, 3, 4, - ); - let low = _mm256_and_si256(value, nibble_mask); - let high = _mm256_and_si256(_mm256_srli_epi16::<4>(value), nibble_mask); - let byte_counts = _mm256_add_epi8( - _mm256_shuffle_epi8(lookup, low), - _mm256_shuffle_epi8(lookup, high), - ); - byte_counts.simd_into(token) - } - ); - kernel(self, a) - } - #[inline(always)] - fn count_zeros_u8x32(self, a: u8x32) -> u8x32 { - self.count_ones_u8x32(self.not_u8x32(a)) - } - #[inline(always)] - fn add_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + fn compress_u8x32(self, values: u8x32, mask: mask8x32) -> u8x32 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u8x32, b: u8x32) -> u8x32 { - _mm256_add_epi8(a.into(), b.into()).simd_into(token) + fn kernel(token: Avx2, values: u8x32, mask: mask8x32) -> u8x32 { + let mask_bits = token.to_bitmask_mask8x32(mask); + let block_bits_0 = (mask_bits & 0xffff) as usize; + let low_mask_0 = (block_bits_0) & 0xff; + let high_mask_0 = (block_bits_0) >> 8; + let low_control_0 = crate::support::COMPRESS_8_CONTROLS[low_mask_0]; + let high_control_0 = crate::support::COMPRESS_8_CONTROLS[high_mask_0]; + let high_control_0 = ((high_control_0 & 0x7f7f_7f7f_7f7f_7f7f) + + 0x0808_0808_0808_0808) + | (high_control_0 & 0x8080_8080_8080_8080); + let control_0 = + _mm_set_epi64x(high_control_0.cast_signed(), low_control_0.cast_signed()); + let low_count_0 = low_mask_0.count_ones() as usize; + let block_bits_1 = (mask_bits >> 16 & 0xffff) as usize; + let low_mask_1 = (block_bits_1) & 0xff; + let high_mask_1 = (block_bits_1) >> 8; + let low_control_1 = crate::support::COMPRESS_8_CONTROLS[low_mask_1]; + let high_control_1 = crate::support::COMPRESS_8_CONTROLS[high_mask_1]; + let high_control_1 = ((high_control_1 & 0x7f7f_7f7f_7f7f_7f7f) + + 0x0808_0808_0808_0808) + | (high_control_1 & 0x8080_8080_8080_8080); + let control_1 = + _mm_set_epi64x(high_control_1.cast_signed(), low_control_1.cast_signed()); + let low_count_1 = low_mask_1.count_ones() as usize; + let control = _mm256_set_m128i(control_1, control_0); + let compacted = _mm256_shuffle_epi8(values.val.0, control); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count_0]; + let splice_0 = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count_1]; + let splice_1 = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let splice = _mm256_set_m128i(splice_1, splice_0); + let compressed = _mm256_shuffle_epi8(compacted, splice); + let splice = unsafe { + _mm256_load_si256( + core::ptr::from_ref( + &crate::support::COMPACT_16_SPLICE_CONTROLS + [block_bits_0.count_ones() as usize], + ) + .cast::<__m256i>(), + ) + }; + let compressed_low = _mm256_permute2x128_si256::<0x80>(compressed, compressed); + let compressed_high = _mm256_permute2x128_si256::<0x11>(compressed, compressed); + let compressed_0 = + _mm256_or_si256(compressed_low, _mm256_shuffle_epi8(compressed_high, splice)); + let result_0 = compressed_0; + result_0.simd_into(token) } ); - kernel(self, a, b) + kernel(self, values, mask) } #[inline(always)] - fn saturating_add_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + fn compress_merge_u8x32( + self, + values: u8x32, + mask: mask8x32, + merge: u8x32, + ) -> u8x32 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx2, a: u8x32, b: u8x32) -> u8x32 { - _mm256_adds_epu8(a.into(), b.into()).simd_into(token) + fn kernel( + token: Avx2, + values: u8x32, + mask: mask8x32, + merge: u8x32, + ) -> u8x32 { + let mask_bits = token.to_bitmask_mask8x32(mask); + let block_bits_0 = (mask_bits & 0xffff) as usize; + let low_mask_0 = (block_bits_0) & 0xff; + let high_mask_0 = (block_bits_0) >> 8; + let low_control_0 = crate::support::COMPRESS_8_CONTROLS[low_mask_0]; + let high_control_0 = crate::support::COMPRESS_8_CONTROLS[high_mask_0]; + let high_control_0 = ((high_control_0 & 0x7f7f_7f7f_7f7f_7f7f) + + 0x0808_0808_0808_0808) + | (high_control_0 & 0x8080_8080_8080_8080); + let control_0 = + _mm_set_epi64x(high_control_0.cast_signed(), low_control_0.cast_signed()); + let low_count_0 = low_mask_0.count_ones() as usize; + let block_bits_1 = (mask_bits >> 16 & 0xffff) as usize; + let low_mask_1 = (block_bits_1) & 0xff; + let high_mask_1 = (block_bits_1) >> 8; + let low_control_1 = crate::support::COMPRESS_8_CONTROLS[low_mask_1]; + let high_control_1 = crate::support::COMPRESS_8_CONTROLS[high_mask_1]; + let high_control_1 = ((high_control_1 & 0x7f7f_7f7f_7f7f_7f7f) + + 0x0808_0808_0808_0808) + | (high_control_1 & 0x8080_8080_8080_8080); + let control_1 = + _mm_set_epi64x(high_control_1.cast_signed(), low_control_1.cast_signed()); + let low_count_1 = low_mask_1.count_ones() as usize; + let control = _mm256_set_m128i(control_1, control_0); + let compacted = _mm256_shuffle_epi8(values.val.0, control); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count_0]; + let splice_0 = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count_1]; + let splice_1 = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let splice = _mm256_set_m128i(splice_1, splice_0); + let compressed = _mm256_shuffle_epi8(compacted, splice); + let splice = unsafe { + _mm256_load_si256( + core::ptr::from_ref( + &crate::support::COMPACT_16_SPLICE_CONTROLS + [block_bits_0.count_ones() as usize], + ) + .cast::<__m256i>(), + ) + }; + let compressed_low = _mm256_permute2x128_si256::<0x80>(compressed, compressed); + let compressed_high = _mm256_permute2x128_si256::<0x11>(compressed, compressed); + let compressed_0 = + _mm256_or_si256(compressed_low, _mm256_shuffle_epi8(compressed_high, splice)); + let output_lane = (block_bits_0.count_ones() + block_bits_1.count_ones()) as usize; + let prefix = unsafe { + _mm256_load_si256( + core::ptr::from_ref( + &crate::support::COMPACT_PREFIX_MASKS[output_lane.min(32)], + ) + .cast::<__m256i>(), + ) + }; + let result_0 = _mm256_blendv_epi8(merge.val.0, compressed_0, prefix); + result_0.simd_into(token) } ); - kernel(self, a, b) + kernel(self, values, mask, merge) } #[inline(always)] - fn sub_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, a: u8x32, b: u8x32) -> u8x32 { - _mm256_sub_epi8(a.into(), b.into()).simd_into(token) + fn expand_u8x32(self, values: u8x32, mask: mask8x32) -> u8x32 { + #[inline] + #[target_feature(enable = "fxsr,sse4.2,cmpxchg16b,popcnt")] + unsafe fn kernel(token: Avx2, values: u8x32, mask: mask8x32) -> u8x32 { + let mask_bits = _mm_movemask_epi8(unsafe { + _mm_load_si128(core::ptr::from_ref(&mask.val.0).cast::<__m128i>().add(0)) + }) as u64 + | (_mm_movemask_epi8(unsafe { + _mm_load_si128(core::ptr::from_ref(&mask.val.0).cast::<__m128i>().add(1)) + }) as u64) + << 16; + let input_values = <[u8; 32]>::from(values); + let input = input_values.as_ptr(); + let mut output = [0u8; 32]; + let mut input_lane = 0usize; + let block_bits = (mask_bits & 0xffff) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) + | (high_control & 0x8080_8080_8080_8080); + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = unsafe { _mm_loadu_si128(input.add(input_lane).cast::<__m128i>()) }; + let expanded = _mm_shuffle_epi8(packed, control); + let result = expanded; + unsafe { + _mm_storeu_si128(output.as_mut_ptr().add(0).cast::<__m128i>(), result); } - ); - kernel(self, a, b) - } - #[inline(always)] - fn saturating_sub_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, a: u8x32, b: u8x32) -> u8x32 { - _mm256_subs_epu8(a.into(), b.into()).simd_into(token) + input_lane += block_bits.count_ones() as usize; + let block_bits = (mask_bits >> 16 & 0xffff) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) + | (high_control & 0x8080_8080_8080_8080); + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = unsafe { _mm_loadu_si128(input.add(input_lane).cast::<__m128i>()) }; + let expanded = _mm_shuffle_epi8(packed, control); + let result = expanded; + unsafe { + _mm_storeu_si128(output.as_mut_ptr().add(16).cast::<__m128i>(), result); } - ); - kernel(self, a, b) + u8x32::simd_from(token, output) + } + unsafe { kernel(self, values, mask) } } #[inline(always)] - fn mul_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { - crate::kernel!( + fn expand_merge_u8x32( + self, + values: u8x32, + mask: mask8x32, + merge: u8x32, + ) -> u8x32 { + #[inline] + #[target_feature(enable = "fxsr,sse4.2,cmpxchg16b,popcnt")] + unsafe fn kernel( + token: Avx2, + values: u8x32, + mask: mask8x32, + merge: u8x32, + ) -> u8x32 { + let mask_bits = _mm_movemask_epi8(unsafe { + _mm_load_si128(core::ptr::from_ref(&mask.val.0).cast::<__m128i>().add(0)) + }) as u64 + | (_mm_movemask_epi8(unsafe { + _mm_load_si128(core::ptr::from_ref(&mask.val.0).cast::<__m128i>().add(1)) + }) as u64) + << 16; + let input_values = <[u8; 32]>::from(values); + let input = input_values.as_ptr(); + let mut output = [0u8; 32]; + let mut input_lane = 0usize; + let block_bits = (mask_bits & 0xffff) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) + | (high_control & 0x8080_8080_8080_8080); + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = unsafe { _mm_loadu_si128(input.add(input_lane).cast::<__m128i>()) }; + let expanded = _mm_shuffle_epi8(packed, control); + let result = unsafe { + _mm_blendv_epi8( + _mm_load_si128(core::ptr::from_ref(&merge.val.0).cast::<__m128i>().add(0)), + expanded, + _mm_load_si128(core::ptr::from_ref(&mask.val.0).cast::<__m128i>().add(0)), + ) + }; + unsafe { + _mm_storeu_si128(output.as_mut_ptr().add(0).cast::<__m128i>(), result); + } + input_lane += block_bits.count_ones() as usize; + let block_bits = (mask_bits >> 16 & 0xffff) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) + | (high_control & 0x8080_8080_8080_8080); + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = unsafe { _mm_loadu_si128(input.add(input_lane).cast::<__m128i>()) }; + let expanded = _mm_shuffle_epi8(packed, control); + let result = unsafe { + _mm_blendv_epi8( + _mm_load_si128(core::ptr::from_ref(&merge.val.0).cast::<__m128i>().add(1)), + expanded, + _mm_load_si128(core::ptr::from_ref(&mask.val.0).cast::<__m128i>().add(1)), + ) + }; + unsafe { + _mm_storeu_si128(output.as_mut_ptr().add(16).cast::<__m128i>(), result); + } + u8x32::simd_from(token, output) + } + unsafe { kernel(self, values, mask, merge) } + } + #[inline(always)] + fn count_ones_u8x32(self, a: u8x32) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u8x32) -> u8x32 { + let value = a.into(); + let nibble_mask = _mm256_set1_epi8(0x0f); + let lookup = _mm256_setr_epi8( + 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, + 2, 3, 2, 3, 3, 4, + ); + let low = _mm256_and_si256(value, nibble_mask); + let high = _mm256_and_si256(_mm256_srli_epi16::<4>(value), nibble_mask); + let byte_counts = _mm256_add_epi8( + _mm256_shuffle_epi8(lookup, low), + _mm256_shuffle_epi8(lookup, high), + ); + byte_counts.simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn count_zeros_u8x32(self, a: u8x32) -> u8x32 { + self.count_ones_u8x32(self.not_u8x32(a)) + } + #[inline(always)] + fn add_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u8x32, b: u8x32) -> u8x32 { + _mm256_add_epi8(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn saturating_add_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u8x32, b: u8x32) -> u8x32 { + _mm256_adds_epu8(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn sub_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u8x32, b: u8x32) -> u8x32 { + _mm256_sub_epi8(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn saturating_sub_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u8x32, b: u8x32) -> u8x32 { + _mm256_subs_epu8(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn mul_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + crate::kernel!( #[inline(always)] fn kernel(token: Avx2, a: u8x32, b: u8x32) -> u8x32 { let a = a.into(); @@ -8303,360 +9110,103 @@ impl Simd for Avx2 { kernel(self, a, b) } #[inline(always)] - fn unzip_low_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, a: u8x32, b: u8x32) -> u8x32 { - let t1 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8( - a.into(), - _mm256_setr_epi8( - 0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, 0, 2, 4, 6, 8, 10, - 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, - ), - )); - let t2 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8( - b.into(), - _mm256_setr_epi8( - 0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, 0, 2, 4, 6, 8, 10, - 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, - ), - )); - _mm256_permute2x128_si256::<0b0010_0000>(t1, t2).simd_into(token) - } - ); - kernel(self, a, b) - } - #[inline(always)] - fn unzip_high_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, a: u8x32, b: u8x32) -> u8x32 { - let t1 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8( - a.into(), - _mm256_setr_epi8( - 0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, 0, 2, 4, 6, 8, 10, - 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, - ), - )); - let t2 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8( - b.into(), - _mm256_setr_epi8( - 0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, 0, 2, 4, 6, 8, 10, - 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, - ), - )); - _mm256_permute2x128_si256::<0b0011_0001>(t1, t2).simd_into(token) - } - ); - kernel(self, a, b) - } - #[inline(always)] - fn interleave_u8x32(self, a: u8x32, b: u8x32) -> (u8x32, u8x32) { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, a: u8x32, b: u8x32) -> (u8x32, u8x32) { - let lo = _mm256_unpacklo_epi8(a.into(), b.into()); - let hi = _mm256_unpackhi_epi8(a.into(), b.into()); - ( - _mm256_permute2x128_si256::<0b0010_0000>(lo, hi).simd_into(token), - _mm256_permute2x128_si256::<0b0011_0001>(lo, hi).simd_into(token), - ) - } - ); - kernel(self, a, b) - } - #[inline(always)] - fn deinterleave_u8x32(self, a: u8x32, b: u8x32) -> (u8x32, u8x32) { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, a: u8x32, b: u8x32) -> (u8x32, u8x32) { - let mask = _mm256_setr_epi8( - 0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, 0, 2, 4, 6, 8, 10, 12, - 14, 1, 3, 5, 7, 9, 11, 13, 15, - ); - let a = _mm256_shuffle_epi8(a.into(), mask); - let b = _mm256_shuffle_epi8(b.into(), mask); - let low = _mm256_permute2x128_si256::<0b0010_0000>(a, b); - let high = _mm256_permute2x128_si256::<0b0011_0001>(a, b); - ( - _mm256_unpacklo_epi64(low, high).simd_into(token), - _mm256_unpackhi_epi64(low, high).simd_into(token), - ) - } - ); - kernel(self, a, b) - } - #[inline(always)] - fn select_u8x32(self, a: mask8x32, b: u8x32, c: u8x32) -> u8x32 { - crate::kernel!( - #[inline(always)] - fn kernel( - token: Avx2, - a: mask8x32, - b: u8x32, - c: u8x32, - ) -> u8x32 { - _mm256_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token) - } - ); - kernel(self, a, b, c) - } - #[inline(always)] - fn compress_u8x32(self, values: u8x32, mask: mask8x32) -> u8x32 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx2, values: u8x32, mask: mask8x32) -> u8x32 { - let mask_bits = token.to_bitmask_mask8x32(mask); - let block_bits_0 = (mask_bits & 0xffff) as usize; - let low_mask_0 = (block_bits_0) & 0xff; - let high_mask_0 = (block_bits_0) >> 8; - let low_control_0 = crate::support::COMPRESS_8_CONTROLS[low_mask_0]; - let high_control_0 = crate::support::COMPRESS_8_CONTROLS[high_mask_0]; - let high_control_0 = ((high_control_0 & 0x7f7f_7f7f_7f7f_7f7f) - + 0x0808_0808_0808_0808) - | (high_control_0 & 0x8080_8080_8080_8080); - let control_0 = - _mm_set_epi64x(high_control_0.cast_signed(), low_control_0.cast_signed()); - let low_count_0 = low_mask_0.count_ones() as usize; - let block_bits_1 = (mask_bits >> 16 & 0xffff) as usize; - let low_mask_1 = (block_bits_1) & 0xff; - let high_mask_1 = (block_bits_1) >> 8; - let low_control_1 = crate::support::COMPRESS_8_CONTROLS[low_mask_1]; - let high_control_1 = crate::support::COMPRESS_8_CONTROLS[high_mask_1]; - let high_control_1 = ((high_control_1 & 0x7f7f_7f7f_7f7f_7f7f) - + 0x0808_0808_0808_0808) - | (high_control_1 & 0x8080_8080_8080_8080); - let control_1 = - _mm_set_epi64x(high_control_1.cast_signed(), low_control_1.cast_signed()); - let low_count_1 = low_mask_1.count_ones() as usize; - let control = _mm256_set_m128i(control_1, control_0); - let compacted = _mm256_shuffle_epi8(values.val.0, control); - let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count_0]; - let splice_0 = _mm_set_epi64x((splice >> 64) as i64, splice as i64); - let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count_1]; - let splice_1 = _mm_set_epi64x((splice >> 64) as i64, splice as i64); - let splice = _mm256_set_m128i(splice_1, splice_0); - let compressed = _mm256_shuffle_epi8(compacted, splice); - let splice = unsafe { - _mm256_load_si256( - core::ptr::from_ref( - &crate::support::COMPACT_16_SPLICE_CONTROLS - [block_bits_0.count_ones() as usize], - ) - .cast::<__m256i>(), - ) - }; - let compressed_low = _mm256_permute2x128_si256::<0x80>(compressed, compressed); - let compressed_high = _mm256_permute2x128_si256::<0x11>(compressed, compressed); - let compressed_0 = - _mm256_or_si256(compressed_low, _mm256_shuffle_epi8(compressed_high, splice)); - let result_0 = compressed_0; - result_0.simd_into(token) - } - ); - kernel(self, values, mask) - } - #[inline(always)] - fn compress_merge_u8x32( - self, - values: u8x32, - mask: mask8x32, - merge: u8x32, - ) -> u8x32 { - crate::kernel!( - #[inline(always)] - fn kernel( - token: Avx2, - values: u8x32, - mask: mask8x32, - merge: u8x32, - ) -> u8x32 { - let mask_bits = token.to_bitmask_mask8x32(mask); - let block_bits_0 = (mask_bits & 0xffff) as usize; - let low_mask_0 = (block_bits_0) & 0xff; - let high_mask_0 = (block_bits_0) >> 8; - let low_control_0 = crate::support::COMPRESS_8_CONTROLS[low_mask_0]; - let high_control_0 = crate::support::COMPRESS_8_CONTROLS[high_mask_0]; - let high_control_0 = ((high_control_0 & 0x7f7f_7f7f_7f7f_7f7f) - + 0x0808_0808_0808_0808) - | (high_control_0 & 0x8080_8080_8080_8080); - let control_0 = - _mm_set_epi64x(high_control_0.cast_signed(), low_control_0.cast_signed()); - let low_count_0 = low_mask_0.count_ones() as usize; - let block_bits_1 = (mask_bits >> 16 & 0xffff) as usize; - let low_mask_1 = (block_bits_1) & 0xff; - let high_mask_1 = (block_bits_1) >> 8; - let low_control_1 = crate::support::COMPRESS_8_CONTROLS[low_mask_1]; - let high_control_1 = crate::support::COMPRESS_8_CONTROLS[high_mask_1]; - let high_control_1 = ((high_control_1 & 0x7f7f_7f7f_7f7f_7f7f) - + 0x0808_0808_0808_0808) - | (high_control_1 & 0x8080_8080_8080_8080); - let control_1 = - _mm_set_epi64x(high_control_1.cast_signed(), low_control_1.cast_signed()); - let low_count_1 = low_mask_1.count_ones() as usize; - let control = _mm256_set_m128i(control_1, control_0); - let compacted = _mm256_shuffle_epi8(values.val.0, control); - let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count_0]; - let splice_0 = _mm_set_epi64x((splice >> 64) as i64, splice as i64); - let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count_1]; - let splice_1 = _mm_set_epi64x((splice >> 64) as i64, splice as i64); - let splice = _mm256_set_m128i(splice_1, splice_0); - let compressed = _mm256_shuffle_epi8(compacted, splice); - let splice = unsafe { - _mm256_load_si256( - core::ptr::from_ref( - &crate::support::COMPACT_16_SPLICE_CONTROLS - [block_bits_0.count_ones() as usize], - ) - .cast::<__m256i>(), - ) - }; - let compressed_low = _mm256_permute2x128_si256::<0x80>(compressed, compressed); - let compressed_high = _mm256_permute2x128_si256::<0x11>(compressed, compressed); - let compressed_0 = - _mm256_or_si256(compressed_low, _mm256_shuffle_epi8(compressed_high, splice)); - let output_lane = (block_bits_0.count_ones() + block_bits_1.count_ones()) as usize; - let prefix = unsafe { - _mm256_load_si256( - core::ptr::from_ref( - &crate::support::COMPACT_PREFIX_MASKS[output_lane.min(32)], - ) - .cast::<__m256i>(), - ) - }; - let result_0 = _mm256_blendv_epi8(merge.val.0, compressed_0, prefix); - result_0.simd_into(token) + fn unzip_low_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u8x32, b: u8x32) -> u8x32 { + let t1 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8( + a.into(), + _mm256_setr_epi8( + 0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, 0, 2, 4, 6, 8, 10, + 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, + ), + )); + let t2 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8( + b.into(), + _mm256_setr_epi8( + 0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, 0, 2, 4, 6, 8, 10, + 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, + ), + )); + _mm256_permute2x128_si256::<0b0010_0000>(t1, t2).simd_into(token) } ); - kernel(self, values, mask, merge) + kernel(self, a, b) } #[inline(always)] - fn expand_u8x32(self, values: u8x32, mask: mask8x32) -> u8x32 { - #[inline] - #[target_feature(enable = "fxsr,sse4.2,cmpxchg16b,popcnt")] - unsafe fn kernel(token: Avx2, values: u8x32, mask: mask8x32) -> u8x32 { - let mask_bits = _mm_movemask_epi8(unsafe { - _mm_load_si128(core::ptr::from_ref(&mask.val.0).cast::<__m128i>().add(0)) - }) as u64 - | (_mm_movemask_epi8(unsafe { - _mm_load_si128(core::ptr::from_ref(&mask.val.0).cast::<__m128i>().add(1)) - }) as u64) - << 16; - let input_values = <[u8; 32]>::from(values); - let input = input_values.as_ptr(); - let mut output = [0u8; 32]; - let mut input_lane = 0usize; - let block_bits = (mask_bits & 0xffff) as usize; - let low_mask = (block_bits) & 0xff; - let high_mask = (block_bits) >> 8; - let low_count = low_mask.count_ones() as u64; - let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; - let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; - let high_base = low_count * 0x0101_0101_0101_0101; - let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) - | (high_control & 0x8080_8080_8080_8080); - let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); - let packed = unsafe { _mm_loadu_si128(input.add(input_lane).cast::<__m128i>()) }; - let expanded = _mm_shuffle_epi8(packed, control); - let result = expanded; - unsafe { - _mm_storeu_si128(output.as_mut_ptr().add(0).cast::<__m128i>(), result); - } - input_lane += block_bits.count_ones() as usize; - let block_bits = (mask_bits >> 16 & 0xffff) as usize; - let low_mask = (block_bits) & 0xff; - let high_mask = (block_bits) >> 8; - let low_count = low_mask.count_ones() as u64; - let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; - let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; - let high_base = low_count * 0x0101_0101_0101_0101; - let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) - | (high_control & 0x8080_8080_8080_8080); - let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); - let packed = unsafe { _mm_loadu_si128(input.add(input_lane).cast::<__m128i>()) }; - let expanded = _mm_shuffle_epi8(packed, control); - let result = expanded; - unsafe { - _mm_storeu_si128(output.as_mut_ptr().add(16).cast::<__m128i>(), result); + fn unzip_high_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u8x32, b: u8x32) -> u8x32 { + let t1 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8( + a.into(), + _mm256_setr_epi8( + 0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, 0, 2, 4, 6, 8, 10, + 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, + ), + )); + let t2 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8( + b.into(), + _mm256_setr_epi8( + 0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, 0, 2, 4, 6, 8, 10, + 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, + ), + )); + _mm256_permute2x128_si256::<0b0011_0001>(t1, t2).simd_into(token) } - u8x32::simd_from(token, output) - } - unsafe { kernel(self, values, mask) } + ); + kernel(self, a, b) } #[inline(always)] - fn expand_merge_u8x32( - self, - values: u8x32, - mask: mask8x32, - merge: u8x32, - ) -> u8x32 { - #[inline] - #[target_feature(enable = "fxsr,sse4.2,cmpxchg16b,popcnt")] - unsafe fn kernel( - token: Avx2, - values: u8x32, - mask: mask8x32, - merge: u8x32, - ) -> u8x32 { - let mask_bits = _mm_movemask_epi8(unsafe { - _mm_load_si128(core::ptr::from_ref(&mask.val.0).cast::<__m128i>().add(0)) - }) as u64 - | (_mm_movemask_epi8(unsafe { - _mm_load_si128(core::ptr::from_ref(&mask.val.0).cast::<__m128i>().add(1)) - }) as u64) - << 16; - let input_values = <[u8; 32]>::from(values); - let input = input_values.as_ptr(); - let mut output = [0u8; 32]; - let mut input_lane = 0usize; - let block_bits = (mask_bits & 0xffff) as usize; - let low_mask = (block_bits) & 0xff; - let high_mask = (block_bits) >> 8; - let low_count = low_mask.count_ones() as u64; - let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; - let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; - let high_base = low_count * 0x0101_0101_0101_0101; - let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) - | (high_control & 0x8080_8080_8080_8080); - let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); - let packed = unsafe { _mm_loadu_si128(input.add(input_lane).cast::<__m128i>()) }; - let expanded = _mm_shuffle_epi8(packed, control); - let result = unsafe { - _mm_blendv_epi8( - _mm_load_si128(core::ptr::from_ref(&merge.val.0).cast::<__m128i>().add(0)), - expanded, - _mm_load_si128(core::ptr::from_ref(&mask.val.0).cast::<__m128i>().add(0)), + fn interleave_u8x32(self, a: u8x32, b: u8x32) -> (u8x32, u8x32) { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u8x32, b: u8x32) -> (u8x32, u8x32) { + let lo = _mm256_unpacklo_epi8(a.into(), b.into()); + let hi = _mm256_unpackhi_epi8(a.into(), b.into()); + ( + _mm256_permute2x128_si256::<0b0010_0000>(lo, hi).simd_into(token), + _mm256_permute2x128_si256::<0b0011_0001>(lo, hi).simd_into(token), ) - }; - unsafe { - _mm_storeu_si128(output.as_mut_ptr().add(0).cast::<__m128i>(), result); } - input_lane += block_bits.count_ones() as usize; - let block_bits = (mask_bits >> 16 & 0xffff) as usize; - let low_mask = (block_bits) & 0xff; - let high_mask = (block_bits) >> 8; - let low_count = low_mask.count_ones() as u64; - let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; - let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; - let high_base = low_count * 0x0101_0101_0101_0101; - let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) - | (high_control & 0x8080_8080_8080_8080); - let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); - let packed = unsafe { _mm_loadu_si128(input.add(input_lane).cast::<__m128i>()) }; - let expanded = _mm_shuffle_epi8(packed, control); - let result = unsafe { - _mm_blendv_epi8( - _mm_load_si128(core::ptr::from_ref(&merge.val.0).cast::<__m128i>().add(1)), - expanded, - _mm_load_si128(core::ptr::from_ref(&mask.val.0).cast::<__m128i>().add(1)), + ); + kernel(self, a, b) + } + #[inline(always)] + fn deinterleave_u8x32(self, a: u8x32, b: u8x32) -> (u8x32, u8x32) { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u8x32, b: u8x32) -> (u8x32, u8x32) { + let mask = _mm256_setr_epi8( + 0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, 0, 2, 4, 6, 8, 10, 12, + 14, 1, 3, 5, 7, 9, 11, 13, 15, + ); + let a = _mm256_shuffle_epi8(a.into(), mask); + let b = _mm256_shuffle_epi8(b.into(), mask); + let low = _mm256_permute2x128_si256::<0b0010_0000>(a, b); + let high = _mm256_permute2x128_si256::<0b0011_0001>(a, b); + ( + _mm256_unpacklo_epi64(low, high).simd_into(token), + _mm256_unpackhi_epi64(low, high).simd_into(token), ) - }; - unsafe { - _mm_storeu_si128(output.as_mut_ptr().add(16).cast::<__m128i>(), result); } - u8x32::simd_from(token, output) - } - unsafe { kernel(self, values, mask, merge) } + ); + kernel(self, a, b) + } + #[inline(always)] + fn select_u8x32(self, a: mask8x32, b: u8x32, c: u8x32) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + a: mask8x32, + b: u8x32, + c: u8x32, + ) -> u8x32 { + _mm256_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token) + } + ); + kernel(self, a, b, c) } #[inline(always)] fn combine_u8x32(self, a: u8x32, b: u8x32) -> u8x64 { @@ -8995,6 +9545,56 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn compress_i16x16(self, values: i16x16, mask: mask16x16) -> i16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i16x16( + self, + values: i16x16, + mask: mask16x16, + merge: i16x16, + ) -> i16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i16x16(self, values: i16x16, mask: mask16x16) -> i16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i16x16( + self, + values: i16x16, + mask: mask16x16, + merge: i16x16, + ) -> i16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_i16x16(self, a: i16x16) -> i16x16 { crate::kernel!( #[inline(always)] @@ -9488,6 +10088,56 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn compress_u16x16(self, values: u16x16, mask: mask16x16) -> u16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u16x16( + self, + values: u16x16, + mask: mask16x16, + merge: u16x16, + ) -> u16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u16x16(self, values: u16x16, mask: mask16x16) -> u16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u16x16( + self, + values: u16x16, + mask: mask16x16, + merge: u16x16, + ) -> u16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_u16x16(self, a: u16x16) -> u16x16 { crate::kernel!( #[inline(always)] @@ -10236,7 +10886,57 @@ impl Simd for Avx2 { Bytes::from_bytes(u8x32 { val: crate::support::Aligned256(result), simd: self, - }) + }) + } + #[inline(always)] + fn compress_i32x8(self, values: i32x8, mask: mask32x8) -> i32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i32x8( + self, + values: i32x8, + mask: mask32x8, + merge: i32x8, + ) -> i32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i32x8(self, values: i32x8, mask: mask32x8) -> i32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i32x8( + self, + values: i32x8, + mask: mask32x8, + merge: i32x8, + ) -> i32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) } #[inline(always)] fn count_ones_i32x8(self, a: i32x8) -> i32x8 { @@ -10724,6 +11424,56 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn compress_u32x8(self, values: u32x8, mask: mask32x8) -> u32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u32x8( + self, + values: u32x8, + mask: mask32x8, + merge: u32x8, + ) -> u32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u32x8(self, values: u32x8, mask: mask32x8) -> u32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u32x8( + self, + values: u32x8, + mask: mask32x8, + merge: u32x8, + ) -> u32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_u32x8(self, a: u32x8) -> u32x8 { crate::kernel!( #[inline(always)] @@ -11453,6 +12203,56 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn compress_f64x4(self, values: f64x4, mask: mask64x4) -> f64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_f64x4( + self, + values: f64x4, + mask: mask64x4, + merge: f64x4, + ) -> f64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_f64x4(self, values: f64x4, mask: mask64x4) -> f64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_f64x4( + self, + values: f64x4, + mask: mask64x4, + merge: f64x4, + ) -> f64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn neg_f64x4(self, a: f64x4) -> f64x4 { crate::kernel!( #[inline(always)] @@ -12025,6 +12825,56 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn compress_i64x4(self, values: i64x4, mask: mask64x4) -> i64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i64x4( + self, + values: i64x4, + mask: mask64x4, + merge: i64x4, + ) -> i64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i64x4(self, values: i64x4, mask: mask64x4) -> i64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i64x4( + self, + values: i64x4, + mask: mask64x4, + merge: i64x4, + ) -> i64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_i64x4(self, a: i64x4) -> i64x4 { crate::kernel!( #[inline(always)] @@ -12496,6 +13346,56 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn compress_u64x4(self, values: u64x4, mask: mask64x4) -> u64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u64x4( + self, + values: u64x4, + mask: mask64x4, + merge: u64x4, + ) -> u64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u64x4(self, values: u64x4, mask: mask64x4) -> u64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u64x4( + self, + values: u64x4, + mask: mask64x4, + merge: u64x4, + ) -> u64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_u64x4(self, a: u64x4) -> u64x4 { crate::kernel!( #[inline(always)] @@ -13133,6 +14033,56 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn compress_f32x16(self, values: f32x16, mask: mask32x16) -> f32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_f32x16( + self, + values: f32x16, + mask: mask32x16, + merge: f32x16, + ) -> f32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_f32x16(self, values: f32x16, mask: mask32x16) -> f32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_f32x16( + self, + values: f32x16, + mask: mask32x16, + merge: f32x16, + ) -> f32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn is_nan_f32x16(self, a: f32x16) -> mask32x16 { !a.simd_eq(a) } @@ -13194,6 +14144,56 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn compress_i8x64(self, values: i8x64, mask: mask8x64) -> i8x64 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i8x64( + self, + values: i8x64, + mask: mask8x64, + merge: i8x64, + ) -> i8x64 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i8x64(self, values: i8x64, mask: mask8x64) -> i8x64 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i8x64( + self, + values: i8x64, + mask: mask8x64, + merge: i8x64, + ) -> i8x64 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_i8x64(self, a: i8x64) -> (i8x32, i8x32) { ( i8x32 { @@ -14044,6 +15044,56 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn compress_i16x32(self, values: i16x32, mask: mask16x32) -> i16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i16x32( + self, + values: i16x32, + mask: mask16x32, + merge: i16x32, + ) -> i16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i16x32(self, values: i16x32, mask: mask16x32) -> i16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i16x32( + self, + values: i16x32, + mask: mask16x32, + merge: i16x32, + ) -> i16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_i16x32(self, a: i16x32) -> (i16x16, i16x16) { ( i16x16 { @@ -14073,6 +15123,56 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn compress_u16x32(self, values: u16x32, mask: mask16x32) -> u16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u16x32( + self, + values: u16x32, + mask: mask16x32, + merge: u16x32, + ) -> u16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u16x32(self, values: u16x32, mask: mask16x32) -> u16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u16x32( + self, + values: u16x32, + mask: mask16x32, + merge: u16x32, + ) -> u16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_u16x32(self, a: u16x32) -> (u16x16, u16x16) { ( u16x16 { @@ -14172,6 +15272,56 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn compress_i32x16(self, values: i32x16, mask: mask32x16) -> i32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i32x16( + self, + values: i32x16, + mask: mask32x16, + merge: i32x16, + ) -> i32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i32x16(self, values: i32x16, mask: mask32x16) -> i32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i32x16( + self, + values: i32x16, + mask: mask32x16, + merge: i32x16, + ) -> i32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_i32x16(self, a: i32x16) -> (i32x8, i32x8) { ( i32x8 { @@ -14201,6 +15351,56 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn compress_u32x16(self, values: u32x16, mask: mask32x16) -> u32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u32x16( + self, + values: u32x16, + mask: mask32x16, + merge: u32x16, + ) -> u32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u32x16(self, values: u32x16, mask: mask32x16) -> u32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u32x16( + self, + values: u32x16, + mask: mask32x16, + merge: u32x16, + ) -> u32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_u32x16(self, a: u32x16) -> (u32x8, u32x8) { ( u32x8 { @@ -14311,6 +15511,56 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn compress_f64x8(self, values: f64x8, mask: mask64x8) -> f64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_f64x8( + self, + values: f64x8, + mask: mask64x8, + merge: f64x8, + ) -> f64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_f64x8(self, values: f64x8, mask: mask64x8) -> f64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_f64x8( + self, + values: f64x8, + mask: mask64x8, + merge: f64x8, + ) -> f64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn is_nan_f64x8(self, a: f64x8) -> mask64x8 { !a.simd_eq(a) } @@ -14372,6 +15622,56 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn compress_i64x8(self, values: i64x8, mask: mask64x8) -> i64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i64x8( + self, + values: i64x8, + mask: mask64x8, + merge: i64x8, + ) -> i64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i64x8(self, values: i64x8, mask: mask64x8) -> i64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i64x8( + self, + values: i64x8, + mask: mask64x8, + merge: i64x8, + ) -> i64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_i64x8(self, a: i64x8) -> (i64x4, i64x4) { ( i64x4 { @@ -14401,6 +15701,56 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn compress_u64x8(self, values: u64x8, mask: mask64x8) -> u64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u64x8( + self, + values: u64x8, + mask: mask64x8, + merge: u64x8, + ) -> u64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u64x8(self, values: u64x8, mask: mask64x8) -> u64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u64x8( + self, + values: u64x8, + mask: mask64x8, + merge: u64x8, + ) -> u64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_u64x8(self, a: u64x8) -> (u64x4, u64x4) { ( u64x4 { diff --git a/fearless_simd/src/generated/avx512.rs b/fearless_simd/src/generated/avx512.rs index a7cdcf9d6..6c4f5daef 100644 --- a/fearless_simd/src/generated/avx512.rs +++ b/fearless_simd/src/generated/avx512.rs @@ -440,6 +440,76 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn compress_f32x4(self, values: f32x4, mask: mask32x4) -> f32x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: f32x4, + mask: mask32x4, + ) -> f32x4 { + _mm_maskz_compress_ps(u64::from((mask).val) as u8, values.into()).simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_f32x4( + self, + values: f32x4, + mask: mask32x4, + merge: f32x4, + ) -> f32x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: f32x4, + mask: mask32x4, + merge: f32x4, + ) -> f32x4 { + _mm_mask_compress_ps(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_f32x4(self, values: f32x4, mask: mask32x4) -> f32x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: f32x4, + mask: mask32x4, + ) -> f32x4 { + _mm_maskz_expand_ps(u64::from((mask).val) as u8, values.into()).simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_f32x4( + self, + values: f32x4, + mask: mask32x4, + merge: f32x4, + ) -> f32x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: f32x4, + mask: mask32x4, + merge: f32x4, + ) -> f32x4 { + _mm_mask_expand_ps(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn neg_f32x4(self, a: f32x4) -> f32x4 { crate::kernel!( #[inline(always)] @@ -1076,6 +1146,77 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn compress_i8x16(self, values: i8x16, mask: mask8x16) -> i8x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i8x16, + mask: mask8x16, + ) -> i8x16 { + _mm_maskz_compress_epi8(u64::from((mask).val) as u16, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_i8x16( + self, + values: i8x16, + mask: mask8x16, + merge: i8x16, + ) -> i8x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i8x16, + mask: mask8x16, + merge: i8x16, + ) -> i8x16 { + _mm_mask_compress_epi8(merge.into(), u64::from((mask).val) as u16, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_i8x16(self, values: i8x16, mask: mask8x16) -> i8x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i8x16, + mask: mask8x16, + ) -> i8x16 { + _mm_maskz_expand_epi8(u64::from((mask).val) as u16, values.into()).simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_i8x16( + self, + values: i8x16, + mask: mask8x16, + merge: i8x16, + ) -> i8x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i8x16, + mask: mask8x16, + merge: i8x16, + ) -> i8x16 { + _mm_mask_expand_epi8(merge.into(), u64::from((mask).val) as u16, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn count_ones_i8x16(self, a: i8x16) -> i8x16 { crate::kernel!( #[inline(always)] @@ -1723,6 +1864,77 @@ impl Simd for Avx512 { kernel(self, a, b, indices) } #[inline(always)] + fn compress_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u8x16, + mask: mask8x16, + ) -> u8x16 { + _mm_maskz_compress_epi8(u64::from((mask).val) as u16, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_u8x16( + self, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + _mm_mask_compress_epi8(merge.into(), u64::from((mask).val) as u16, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u8x16, + mask: mask8x16, + ) -> u8x16 { + _mm_maskz_expand_epi8(u64::from((mask).val) as u16, values.into()).simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_u8x16( + self, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + _mm_mask_expand_epi8(merge.into(), u64::from((mask).val) as u16, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn count_ones_u8x16(self, a: u8x16) -> u8x16 { crate::kernel!( #[inline(always)] @@ -2135,77 +2347,6 @@ impl Simd for Avx512 { kernel(self, a, b, c) } #[inline(always)] - fn compress_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16 { - crate::kernel!( - #[inline(always)] - fn kernel( - token: Avx512, - values: u8x16, - mask: mask8x16, - ) -> u8x16 { - _mm_maskz_compress_epi8(u64::from((mask).val) as u16, values.into()) - .simd_into(token) - } - ); - kernel(self, values, mask) - } - #[inline(always)] - fn compress_merge_u8x16( - self, - values: u8x16, - mask: mask8x16, - merge: u8x16, - ) -> u8x16 { - crate::kernel!( - #[inline(always)] - fn kernel( - token: Avx512, - values: u8x16, - mask: mask8x16, - merge: u8x16, - ) -> u8x16 { - _mm_mask_compress_epi8(merge.into(), u64::from((mask).val) as u16, values.into()) - .simd_into(token) - } - ); - kernel(self, values, mask, merge) - } - #[inline(always)] - fn expand_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16 { - crate::kernel!( - #[inline(always)] - fn kernel( - token: Avx512, - values: u8x16, - mask: mask8x16, - ) -> u8x16 { - _mm_maskz_expand_epi8(u64::from((mask).val) as u16, values.into()).simd_into(token) - } - ); - kernel(self, values, mask) - } - #[inline(always)] - fn expand_merge_u8x16( - self, - values: u8x16, - mask: mask8x16, - merge: u8x16, - ) -> u8x16 { - crate::kernel!( - #[inline(always)] - fn kernel( - token: Avx512, - values: u8x16, - mask: mask8x16, - merge: u8x16, - ) -> u8x16 { - _mm_mask_expand_epi8(merge.into(), u64::from((mask).val) as u16, values.into()) - .simd_into(token) - } - ); - kernel(self, values, mask, merge) - } - #[inline(always)] fn combine_u8x16(self, a: u8x16, b: u8x16) -> u8x32 { crate::kernel!( #[inline(always)] @@ -2485,21 +2626,92 @@ impl Simd for Avx512 { }) } #[inline(always)] - fn count_ones_i16x8(self, a: i16x8) -> i16x8 { + fn compress_i16x8(self, values: i16x8, mask: mask16x8) -> i16x8 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx512, a: i16x8) -> i16x8 { - _mm_popcnt_epi16(a.into()).simd_into(token) + fn kernel( + token: Avx512, + values: i16x8, + mask: mask16x8, + ) -> i16x8 { + _mm_maskz_compress_epi16(u64::from((mask).val) as u8, values.into()) + .simd_into(token) } ); - kernel(self, a) - } - #[inline(always)] - fn count_zeros_i16x8(self, a: i16x8) -> i16x8 { - self.count_ones_i16x8(self.not_i16x8(a)) + kernel(self, values, mask) } #[inline(always)] - fn add_i16x8(self, a: i16x8, b: i16x8) -> i16x8 { + fn compress_merge_i16x8( + self, + values: i16x8, + mask: mask16x8, + merge: i16x8, + ) -> i16x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i16x8, + mask: mask16x8, + merge: i16x8, + ) -> i16x8 { + _mm_mask_compress_epi16(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_i16x8(self, values: i16x8, mask: mask16x8) -> i16x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i16x8, + mask: mask16x8, + ) -> i16x8 { + _mm_maskz_expand_epi16(u64::from((mask).val) as u8, values.into()).simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_i16x8( + self, + values: i16x8, + mask: mask16x8, + merge: i16x8, + ) -> i16x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i16x8, + mask: mask16x8, + merge: i16x8, + ) -> i16x8 { + _mm_mask_expand_epi16(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn count_ones_i16x8(self, a: i16x8) -> i16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x8) -> i16x8 { + _mm_popcnt_epi16(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn count_zeros_i16x8(self, a: i16x8) -> i16x8 { + self.count_ones_i16x8(self.not_i16x8(a)) + } + #[inline(always)] + fn add_i16x8(self, a: i16x8, b: i16x8) -> i16x8 { crate::kernel!( #[inline(always)] fn kernel(token: Avx512, a: i16x8, b: i16x8) -> i16x8 { @@ -3001,6 +3213,77 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn compress_u16x8(self, values: u16x8, mask: mask16x8) -> u16x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u16x8, + mask: mask16x8, + ) -> u16x8 { + _mm_maskz_compress_epi16(u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_u16x8( + self, + values: u16x8, + mask: mask16x8, + merge: u16x8, + ) -> u16x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u16x8, + mask: mask16x8, + merge: u16x8, + ) -> u16x8 { + _mm_mask_compress_epi16(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_u16x8(self, values: u16x8, mask: mask16x8) -> u16x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u16x8, + mask: mask16x8, + ) -> u16x8 { + _mm_maskz_expand_epi16(u64::from((mask).val) as u8, values.into()).simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_u16x8( + self, + values: u16x8, + mask: mask16x8, + merge: u16x8, + ) -> u16x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u16x8, + mask: mask16x8, + merge: u16x8, + ) -> u16x8 { + _mm_mask_expand_epi16(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn count_ones_u16x8(self, a: u16x8) -> u16x8 { crate::kernel!( #[inline(always)] @@ -3677,6 +3960,77 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn compress_i32x4(self, values: i32x4, mask: mask32x4) -> i32x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i32x4, + mask: mask32x4, + ) -> i32x4 { + _mm_maskz_compress_epi32(u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_i32x4( + self, + values: i32x4, + mask: mask32x4, + merge: i32x4, + ) -> i32x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i32x4, + mask: mask32x4, + merge: i32x4, + ) -> i32x4 { + _mm_mask_compress_epi32(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_i32x4(self, values: i32x4, mask: mask32x4) -> i32x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i32x4, + mask: mask32x4, + ) -> i32x4 { + _mm_maskz_expand_epi32(u64::from((mask).val) as u8, values.into()).simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_i32x4( + self, + values: i32x4, + mask: mask32x4, + merge: i32x4, + ) -> i32x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i32x4, + mask: mask32x4, + merge: i32x4, + ) -> i32x4 { + _mm_mask_expand_epi32(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn count_ones_i32x4(self, a: i32x4) -> i32x4 { crate::kernel!( #[inline(always)] @@ -4188,6 +4542,77 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn compress_u32x4(self, values: u32x4, mask: mask32x4) -> u32x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u32x4, + mask: mask32x4, + ) -> u32x4 { + _mm_maskz_compress_epi32(u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_u32x4( + self, + values: u32x4, + mask: mask32x4, + merge: u32x4, + ) -> u32x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u32x4, + mask: mask32x4, + merge: u32x4, + ) -> u32x4 { + _mm_mask_compress_epi32(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_u32x4(self, values: u32x4, mask: mask32x4) -> u32x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u32x4, + mask: mask32x4, + ) -> u32x4 { + _mm_maskz_expand_epi32(u64::from((mask).val) as u8, values.into()).simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_u32x4( + self, + values: u32x4, + mask: mask32x4, + merge: u32x4, + ) -> u32x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u32x4, + mask: mask32x4, + merge: u32x4, + ) -> u32x4 { + _mm_mask_expand_epi32(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn count_ones_u32x4(self, a: u32x4) -> u32x4 { crate::kernel!( #[inline(always)] @@ -4867,6 +5292,76 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn compress_f64x2(self, values: f64x2, mask: mask64x2) -> f64x2 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: f64x2, + mask: mask64x2, + ) -> f64x2 { + _mm_maskz_compress_pd(u64::from((mask).val) as u8, values.into()).simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_f64x2( + self, + values: f64x2, + mask: mask64x2, + merge: f64x2, + ) -> f64x2 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: f64x2, + mask: mask64x2, + merge: f64x2, + ) -> f64x2 { + _mm_mask_compress_pd(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_f64x2(self, values: f64x2, mask: mask64x2) -> f64x2 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: f64x2, + mask: mask64x2, + ) -> f64x2 { + _mm_maskz_expand_pd(u64::from((mask).val) as u8, values.into()).simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_f64x2( + self, + values: f64x2, + mask: mask64x2, + merge: f64x2, + ) -> f64x2 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: f64x2, + mask: mask64x2, + merge: f64x2, + ) -> f64x2 { + _mm_mask_expand_pd(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn neg_f64x2(self, a: f64x2) -> f64x2 { crate::kernel!( #[inline(always)] @@ -5493,38 +5988,109 @@ impl Simd for Avx512 { }) } #[inline(always)] - fn count_ones_i64x2(self, a: i64x2) -> i64x2 { + fn compress_i64x2(self, values: i64x2, mask: mask64x2) -> i64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx512, a: i64x2) -> i64x2 { - _mm_popcnt_epi64(a.into()).simd_into(token) + fn kernel( + token: Avx512, + values: i64x2, + mask: mask64x2, + ) -> i64x2 { + _mm_maskz_compress_epi64(u64::from((mask).val) as u8, values.into()) + .simd_into(token) } ); - kernel(self, a) - } - #[inline(always)] - fn count_zeros_i64x2(self, a: i64x2) -> i64x2 { - self.count_ones_i64x2(self.not_i64x2(a)) + kernel(self, values, mask) } #[inline(always)] - fn add_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + fn compress_merge_i64x2( + self, + values: i64x2, + mask: mask64x2, + merge: i64x2, + ) -> i64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx512, a: i64x2, b: i64x2) -> i64x2 { - _mm_add_epi64(a.into(), b.into()).simd_into(token) + fn kernel( + token: Avx512, + values: i64x2, + mask: mask64x2, + merge: i64x2, + ) -> i64x2 { + _mm_mask_compress_epi64(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) } ); - kernel(self, a, b) + kernel(self, values, mask, merge) } #[inline(always)] - fn saturating_add_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + fn expand_i64x2(self, values: i64x2, mask: mask64x2) -> i64x2 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx512, a: i64x2, b: i64x2) -> i64x2 { - let a = a.into(); - let b = b.into(); - let wrapped = _mm_add_epi64(a, b); - let overflow_bits = _mm_ternarylogic_epi64::<0x42>(a, b, wrapped); + fn kernel( + token: Avx512, + values: i64x2, + mask: mask64x2, + ) -> i64x2 { + _mm_maskz_expand_epi64(u64::from((mask).val) as u8, values.into()).simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_i64x2( + self, + values: i64x2, + mask: mask64x2, + merge: i64x2, + ) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i64x2, + mask: mask64x2, + merge: i64x2, + ) -> i64x2 { + _mm_mask_expand_epi64(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn count_ones_i64x2(self, a: i64x2) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x2) -> i64x2 { + _mm_popcnt_epi64(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn count_zeros_i64x2(self, a: i64x2) -> i64x2 { + self.count_ones_i64x2(self.not_i64x2(a)) + } + #[inline(always)] + fn add_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x2, b: i64x2) -> i64x2 { + _mm_add_epi64(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn saturating_add_i64x2(self, a: i64x2, b: i64x2) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i64x2, b: i64x2) -> i64x2 { + let a = a.into(); + let b = b.into(); + let wrapped = _mm_add_epi64(a, b); + let overflow_bits = _mm_ternarylogic_epi64::<0x42>(a, b, wrapped); let overflow_mask = _mm_srai_epi64::<63>(overflow_bits); let direction = _mm_add_epi64(_mm_srli_epi64::<63>(a), _mm_set1_epi64x(i64::MAX)); _mm_ternarylogic_epi64::<0xca>(overflow_mask, direction, wrapped).simd_into(token) @@ -5965,6 +6531,77 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn compress_u64x2(self, values: u64x2, mask: mask64x2) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u64x2, + mask: mask64x2, + ) -> u64x2 { + _mm_maskz_compress_epi64(u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_u64x2( + self, + values: u64x2, + mask: mask64x2, + merge: u64x2, + ) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u64x2, + mask: mask64x2, + merge: u64x2, + ) -> u64x2 { + _mm_mask_compress_epi64(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_u64x2(self, values: u64x2, mask: mask64x2) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u64x2, + mask: mask64x2, + ) -> u64x2 { + _mm_maskz_expand_epi64(u64::from((mask).val) as u8, values.into()).simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_u64x2( + self, + values: u64x2, + mask: mask64x2, + merge: u64x2, + ) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u64x2, + mask: mask64x2, + merge: u64x2, + ) -> u64x2 { + _mm_mask_expand_epi64(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn count_ones_u64x2(self, a: u64x2) -> u64x2 { crate::kernel!( #[inline(always)] @@ -6630,6 +7267,77 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn compress_f32x8(self, values: f32x8, mask: mask32x8) -> f32x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: f32x8, + mask: mask32x8, + ) -> f32x8 { + _mm256_maskz_compress_ps(u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_f32x8( + self, + values: f32x8, + mask: mask32x8, + merge: f32x8, + ) -> f32x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: f32x8, + mask: mask32x8, + merge: f32x8, + ) -> f32x8 { + _mm256_mask_compress_ps(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_f32x8(self, values: f32x8, mask: mask32x8) -> f32x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: f32x8, + mask: mask32x8, + ) -> f32x8 { + _mm256_maskz_expand_ps(u64::from((mask).val) as u8, values.into()).simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_f32x8( + self, + values: f32x8, + mask: mask32x8, + merge: f32x8, + ) -> f32x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: f32x8, + mask: mask32x8, + merge: f32x8, + ) -> f32x8 { + _mm256_mask_expand_ps(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn neg_f32x8(self, a: f32x8) -> f32x8 { crate::kernel!( #[inline(always)] @@ -7241,6 +7949,78 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn compress_i8x32(self, values: i8x32, mask: mask8x32) -> i8x32 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i8x32, + mask: mask8x32, + ) -> i8x32 { + _mm256_maskz_compress_epi8(u64::from((mask).val) as u32, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_i8x32( + self, + values: i8x32, + mask: mask8x32, + merge: i8x32, + ) -> i8x32 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i8x32, + mask: mask8x32, + merge: i8x32, + ) -> i8x32 { + _mm256_mask_compress_epi8(merge.into(), u64::from((mask).val) as u32, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_i8x32(self, values: i8x32, mask: mask8x32) -> i8x32 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i8x32, + mask: mask8x32, + ) -> i8x32 { + _mm256_maskz_expand_epi8(u64::from((mask).val) as u32, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_i8x32( + self, + values: i8x32, + mask: mask8x32, + merge: i8x32, + ) -> i8x32 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i8x32, + mask: mask8x32, + merge: i8x32, + ) -> i8x32 { + _mm256_mask_expand_epi8(merge.into(), u64::from((mask).val) as u32, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn count_ones_i8x32(self, a: i8x32) -> i8x32 { crate::kernel!( #[inline(always)] @@ -7861,45 +8641,117 @@ impl Simd for Avx512 { kernel(self, a, b, indices) } #[inline(always)] - fn count_ones_u8x32(self, a: u8x32) -> u8x32 { + fn compress_u8x32(self, values: u8x32, mask: mask8x32) -> u8x32 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx512, a: u8x32) -> u8x32 { - _mm256_popcnt_epi8(a.into()).simd_into(token) + fn kernel( + token: Avx512, + values: u8x32, + mask: mask8x32, + ) -> u8x32 { + _mm256_maskz_compress_epi8(u64::from((mask).val) as u32, values.into()) + .simd_into(token) } ); - kernel(self, a) - } - #[inline(always)] - fn count_zeros_u8x32(self, a: u8x32) -> u8x32 { - self.count_ones_u8x32(self.not_u8x32(a)) + kernel(self, values, mask) } #[inline(always)] - fn add_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + fn compress_merge_u8x32( + self, + values: u8x32, + mask: mask8x32, + merge: u8x32, + ) -> u8x32 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx512, a: u8x32, b: u8x32) -> u8x32 { - _mm256_add_epi8(a.into(), b.into()).simd_into(token) + fn kernel( + token: Avx512, + values: u8x32, + mask: mask8x32, + merge: u8x32, + ) -> u8x32 { + _mm256_mask_compress_epi8(merge.into(), u64::from((mask).val) as u32, values.into()) + .simd_into(token) } ); - kernel(self, a, b) + kernel(self, values, mask, merge) } #[inline(always)] - fn saturating_add_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + fn expand_u8x32(self, values: u8x32, mask: mask8x32) -> u8x32 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx512, a: u8x32, b: u8x32) -> u8x32 { - _mm256_adds_epu8(a.into(), b.into()).simd_into(token) + fn kernel( + token: Avx512, + values: u8x32, + mask: mask8x32, + ) -> u8x32 { + _mm256_maskz_expand_epi8(u64::from((mask).val) as u32, values.into()) + .simd_into(token) } ); - kernel(self, a, b) + kernel(self, values, mask) } #[inline(always)] - fn sub_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { - crate::kernel!( - #[inline(always)] - fn kernel(token: Avx512, a: u8x32, b: u8x32) -> u8x32 { - _mm256_sub_epi8(a.into(), b.into()).simd_into(token) + fn expand_merge_u8x32( + self, + values: u8x32, + mask: mask8x32, + merge: u8x32, + ) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u8x32, + mask: mask8x32, + merge: u8x32, + ) -> u8x32 { + _mm256_mask_expand_epi8(merge.into(), u64::from((mask).val) as u32, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn count_ones_u8x32(self, a: u8x32) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x32) -> u8x32 { + _mm256_popcnt_epi8(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn count_zeros_u8x32(self, a: u8x32) -> u8x32 { + self.count_ones_u8x32(self.not_u8x32(a)) + } + #[inline(always)] + fn add_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x32, b: u8x32) -> u8x32 { + _mm256_add_epi8(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn saturating_add_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x32, b: u8x32) -> u8x32 { + _mm256_adds_epu8(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn sub_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x32, b: u8x32) -> u8x32 { + _mm256_sub_epi8(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) @@ -8255,78 +9107,6 @@ impl Simd for Avx512 { kernel(self, a, b, c) } #[inline(always)] - fn compress_u8x32(self, values: u8x32, mask: mask8x32) -> u8x32 { - crate::kernel!( - #[inline(always)] - fn kernel( - token: Avx512, - values: u8x32, - mask: mask8x32, - ) -> u8x32 { - _mm256_maskz_compress_epi8(u64::from((mask).val) as u32, values.into()) - .simd_into(token) - } - ); - kernel(self, values, mask) - } - #[inline(always)] - fn compress_merge_u8x32( - self, - values: u8x32, - mask: mask8x32, - merge: u8x32, - ) -> u8x32 { - crate::kernel!( - #[inline(always)] - fn kernel( - token: Avx512, - values: u8x32, - mask: mask8x32, - merge: u8x32, - ) -> u8x32 { - _mm256_mask_compress_epi8(merge.into(), u64::from((mask).val) as u32, values.into()) - .simd_into(token) - } - ); - kernel(self, values, mask, merge) - } - #[inline(always)] - fn expand_u8x32(self, values: u8x32, mask: mask8x32) -> u8x32 { - crate::kernel!( - #[inline(always)] - fn kernel( - token: Avx512, - values: u8x32, - mask: mask8x32, - ) -> u8x32 { - _mm256_maskz_expand_epi8(u64::from((mask).val) as u32, values.into()) - .simd_into(token) - } - ); - kernel(self, values, mask) - } - #[inline(always)] - fn expand_merge_u8x32( - self, - values: u8x32, - mask: mask8x32, - merge: u8x32, - ) -> u8x32 { - crate::kernel!( - #[inline(always)] - fn kernel( - token: Avx512, - values: u8x32, - mask: mask8x32, - merge: u8x32, - ) -> u8x32 { - _mm256_mask_expand_epi8(merge.into(), u64::from((mask).val) as u32, values.into()) - .simd_into(token) - } - ); - kernel(self, values, mask, merge) - } - #[inline(always)] fn combine_u8x32(self, a: u8x32, b: u8x32) -> u8x64 { crate::kernel!( #[inline(always)] @@ -8611,6 +9391,82 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn compress_i16x16(self, values: i16x16, mask: mask16x16) -> i16x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i16x16, + mask: mask16x16, + ) -> i16x16 { + _mm256_maskz_compress_epi16(u64::from((mask).val) as u16, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_i16x16( + self, + values: i16x16, + mask: mask16x16, + merge: i16x16, + ) -> i16x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i16x16, + mask: mask16x16, + merge: i16x16, + ) -> i16x16 { + _mm256_mask_compress_epi16( + merge.into(), + u64::from((mask).val) as u16, + values.into(), + ) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_i16x16(self, values: i16x16, mask: mask16x16) -> i16x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i16x16, + mask: mask16x16, + ) -> i16x16 { + _mm256_maskz_expand_epi16(u64::from((mask).val) as u16, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_i16x16( + self, + values: i16x16, + mask: mask16x16, + merge: i16x16, + ) -> i16x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i16x16, + mask: mask16x16, + merge: i16x16, + ) -> i16x16 { + _mm256_mask_expand_epi16(merge.into(), u64::from((mask).val) as u16, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn count_ones_i16x16(self, a: i16x16) -> i16x16 { crate::kernel!( #[inline(always)] @@ -9101,6 +9957,82 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn compress_u16x16(self, values: u16x16, mask: mask16x16) -> u16x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u16x16, + mask: mask16x16, + ) -> u16x16 { + _mm256_maskz_compress_epi16(u64::from((mask).val) as u16, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_u16x16( + self, + values: u16x16, + mask: mask16x16, + merge: u16x16, + ) -> u16x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u16x16, + mask: mask16x16, + merge: u16x16, + ) -> u16x16 { + _mm256_mask_compress_epi16( + merge.into(), + u64::from((mask).val) as u16, + values.into(), + ) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_u16x16(self, values: u16x16, mask: mask16x16) -> u16x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u16x16, + mask: mask16x16, + ) -> u16x16 { + _mm256_maskz_expand_epi16(u64::from((mask).val) as u16, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_u16x16( + self, + values: u16x16, + mask: mask16x16, + merge: u16x16, + ) -> u16x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u16x16, + mask: mask16x16, + merge: u16x16, + ) -> u16x16 { + _mm256_mask_expand_epi16(merge.into(), u64::from((mask).val) as u16, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn count_ones_u16x16(self, a: u16x16) -> u16x16 { crate::kernel!( #[inline(always)] @@ -9765,6 +10697,78 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn compress_i32x8(self, values: i32x8, mask: mask32x8) -> i32x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i32x8, + mask: mask32x8, + ) -> i32x8 { + _mm256_maskz_compress_epi32(u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_i32x8( + self, + values: i32x8, + mask: mask32x8, + merge: i32x8, + ) -> i32x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i32x8, + mask: mask32x8, + merge: i32x8, + ) -> i32x8 { + _mm256_mask_compress_epi32(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_i32x8(self, values: i32x8, mask: mask32x8) -> i32x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i32x8, + mask: mask32x8, + ) -> i32x8 { + _mm256_maskz_expand_epi32(u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_i32x8( + self, + values: i32x8, + mask: mask32x8, + merge: i32x8, + ) -> i32x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i32x8, + mask: mask32x8, + merge: i32x8, + ) -> i32x8 { + _mm256_mask_expand_epi32(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn count_ones_i32x8(self, a: i32x8) -> i32x8 { crate::kernel!( #[inline(always)] @@ -10259,6 +11263,78 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn compress_u32x8(self, values: u32x8, mask: mask32x8) -> u32x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u32x8, + mask: mask32x8, + ) -> u32x8 { + _mm256_maskz_compress_epi32(u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_u32x8( + self, + values: u32x8, + mask: mask32x8, + merge: u32x8, + ) -> u32x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u32x8, + mask: mask32x8, + merge: u32x8, + ) -> u32x8 { + _mm256_mask_compress_epi32(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_u32x8(self, values: u32x8, mask: mask32x8) -> u32x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u32x8, + mask: mask32x8, + ) -> u32x8 { + _mm256_maskz_expand_epi32(u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_u32x8( + self, + values: u32x8, + mask: mask32x8, + merge: u32x8, + ) -> u32x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u32x8, + mask: mask32x8, + merge: u32x8, + ) -> u32x8 { + _mm256_mask_expand_epi32(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn count_ones_u32x8(self, a: u32x8) -> u32x8 { crate::kernel!( #[inline(always)] @@ -10917,6 +11993,77 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn compress_f64x4(self, values: f64x4, mask: mask64x4) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: f64x4, + mask: mask64x4, + ) -> f64x4 { + _mm256_maskz_compress_pd(u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_f64x4( + self, + values: f64x4, + mask: mask64x4, + merge: f64x4, + ) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: f64x4, + mask: mask64x4, + merge: f64x4, + ) -> f64x4 { + _mm256_mask_compress_pd(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_f64x4(self, values: f64x4, mask: mask64x4) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: f64x4, + mask: mask64x4, + ) -> f64x4 { + _mm256_maskz_expand_pd(u64::from((mask).val) as u8, values.into()).simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_f64x4( + self, + values: f64x4, + mask: mask64x4, + merge: f64x4, + ) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: f64x4, + mask: mask64x4, + merge: f64x4, + ) -> f64x4 { + _mm256_mask_expand_pd(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn neg_f64x4(self, a: f64x4) -> f64x4 { crate::kernel!( #[inline(always)] @@ -11515,6 +12662,78 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn compress_i64x4(self, values: i64x4, mask: mask64x4) -> i64x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i64x4, + mask: mask64x4, + ) -> i64x4 { + _mm256_maskz_compress_epi64(u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_i64x4( + self, + values: i64x4, + mask: mask64x4, + merge: i64x4, + ) -> i64x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i64x4, + mask: mask64x4, + merge: i64x4, + ) -> i64x4 { + _mm256_mask_compress_epi64(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_i64x4(self, values: i64x4, mask: mask64x4) -> i64x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i64x4, + mask: mask64x4, + ) -> i64x4 { + _mm256_maskz_expand_epi64(u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_i64x4( + self, + values: i64x4, + mask: mask64x4, + merge: i64x4, + ) -> i64x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i64x4, + mask: mask64x4, + merge: i64x4, + ) -> i64x4 { + _mm256_mask_expand_epi64(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn count_ones_i64x4(self, a: i64x4) -> i64x4 { crate::kernel!( #[inline(always)] @@ -11979,6 +13198,78 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn compress_u64x4(self, values: u64x4, mask: mask64x4) -> u64x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u64x4, + mask: mask64x4, + ) -> u64x4 { + _mm256_maskz_compress_epi64(u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_u64x4( + self, + values: u64x4, + mask: mask64x4, + merge: u64x4, + ) -> u64x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u64x4, + mask: mask64x4, + merge: u64x4, + ) -> u64x4 { + _mm256_mask_compress_epi64(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_u64x4(self, values: u64x4, mask: mask64x4) -> u64x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u64x4, + mask: mask64x4, + ) -> u64x4 { + _mm256_maskz_expand_epi64(u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_u64x4( + self, + values: u64x4, + mask: mask64x4, + merge: u64x4, + ) -> u64x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u64x4, + mask: mask64x4, + merge: u64x4, + ) -> u64x4 { + _mm256_mask_expand_epi64(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn count_ones_u64x4(self, a: u64x4) -> u64x4 { crate::kernel!( #[inline(always)] @@ -12609,6 +13900,77 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn compress_f32x16(self, values: f32x16, mask: mask32x16) -> f32x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: f32x16, + mask: mask32x16, + ) -> f32x16 { + _mm512_maskz_compress_ps(u64::from((mask).val) as u16, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_f32x16( + self, + values: f32x16, + mask: mask32x16, + merge: f32x16, + ) -> f32x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: f32x16, + mask: mask32x16, + merge: f32x16, + ) -> f32x16 { + _mm512_mask_compress_ps(merge.into(), u64::from((mask).val) as u16, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_f32x16(self, values: f32x16, mask: mask32x16) -> f32x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: f32x16, + mask: mask32x16, + ) -> f32x16 { + _mm512_maskz_expand_ps(u64::from((mask).val) as u16, values.into()).simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_f32x16( + self, + values: f32x16, + mask: mask32x16, + merge: f32x16, + ) -> f32x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: f32x16, + mask: mask32x16, + merge: f32x16, + ) -> f32x16 { + _mm512_mask_expand_ps(merge.into(), u64::from((mask).val) as u16, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn neg_f32x16(self, a: f32x16) -> f32x16 { crate::kernel!( #[inline(always)] @@ -13225,24 +14587,96 @@ impl Simd for Avx512 { kernel(self, a, b, SHIFT) } #[inline(always)] - fn slide_within_blocks_i8x64( + fn slide_within_blocks_i8x64( + self, + a: i8x64, + b: i8x64, + ) -> i8x64 { + if SHIFT == 0 { + return a; + } + if SHIFT >= 16usize { + return b; + } + let a = Bytes::to_bytes(a).val.0; + let b = Bytes::to_bytes(b).val.0; + let result = dyn_alignr_512(self, b, a, SHIFT); + Bytes::from_bytes(u8x64 { + val: crate::support::Aligned512(result), + simd: self, + }) + } + #[inline(always)] + fn compress_i8x64(self, values: i8x64, mask: mask8x64) -> i8x64 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i8x64, + mask: mask8x64, + ) -> i8x64 { + _mm512_maskz_compress_epi8(u64::from((mask).val) as u64, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_i8x64( + self, + values: i8x64, + mask: mask8x64, + merge: i8x64, + ) -> i8x64 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i8x64, + mask: mask8x64, + merge: i8x64, + ) -> i8x64 { + _mm512_mask_compress_epi8(merge.into(), u64::from((mask).val) as u64, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_i8x64(self, values: i8x64, mask: mask8x64) -> i8x64 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i8x64, + mask: mask8x64, + ) -> i8x64 { + _mm512_maskz_expand_epi8(u64::from((mask).val) as u64, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_i8x64( self, - a: i8x64, - b: i8x64, + values: i8x64, + mask: mask8x64, + merge: i8x64, ) -> i8x64 { - if SHIFT == 0 { - return a; - } - if SHIFT >= 16usize { - return b; - } - let a = Bytes::to_bytes(a).val.0; - let b = Bytes::to_bytes(b).val.0; - let result = dyn_alignr_512(self, b, a, SHIFT); - Bytes::from_bytes(u8x64 { - val: crate::support::Aligned512(result), - simd: self, - }) + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i8x64, + mask: mask8x64, + merge: i8x64, + ) -> i8x64 { + _mm512_mask_expand_epi8(merge.into(), u64::from((mask).val) as u64, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) } #[inline(always)] fn count_ones_i8x64(self, a: i8x64) -> i8x64 { @@ -13875,6 +15309,78 @@ impl Simd for Avx512 { kernel(self, a, b, indices) } #[inline(always)] + fn compress_u8x64(self, values: u8x64, mask: mask8x64) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u8x64, + mask: mask8x64, + ) -> u8x64 { + _mm512_maskz_compress_epi8(u64::from((mask).val) as u64, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_u8x64( + self, + values: u8x64, + mask: mask8x64, + merge: u8x64, + ) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u8x64, + mask: mask8x64, + merge: u8x64, + ) -> u8x64 { + _mm512_mask_compress_epi8(merge.into(), u64::from((mask).val) as u64, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_u8x64(self, values: u8x64, mask: mask8x64) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u8x64, + mask: mask8x64, + ) -> u8x64 { + _mm512_maskz_expand_epi8(u64::from((mask).val) as u64, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_u8x64( + self, + values: u8x64, + mask: mask8x64, + merge: u8x64, + ) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u8x64, + mask: mask8x64, + merge: u8x64, + ) -> u8x64 { + _mm512_mask_expand_epi8(merge.into(), u64::from((mask).val) as u64, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn count_ones_u8x64(self, a: u8x64) -> u8x64 { crate::kernel!( #[inline(always)] @@ -14286,78 +15792,6 @@ impl Simd for Avx512 { kernel(self, a, b, c) } #[inline(always)] - fn compress_u8x64(self, values: u8x64, mask: mask8x64) -> u8x64 { - crate::kernel!( - #[inline(always)] - fn kernel( - token: Avx512, - values: u8x64, - mask: mask8x64, - ) -> u8x64 { - _mm512_maskz_compress_epi8(u64::from((mask).val) as u64, values.into()) - .simd_into(token) - } - ); - kernel(self, values, mask) - } - #[inline(always)] - fn compress_merge_u8x64( - self, - values: u8x64, - mask: mask8x64, - merge: u8x64, - ) -> u8x64 { - crate::kernel!( - #[inline(always)] - fn kernel( - token: Avx512, - values: u8x64, - mask: mask8x64, - merge: u8x64, - ) -> u8x64 { - _mm512_mask_compress_epi8(merge.into(), u64::from((mask).val) as u64, values.into()) - .simd_into(token) - } - ); - kernel(self, values, mask, merge) - } - #[inline(always)] - fn expand_u8x64(self, values: u8x64, mask: mask8x64) -> u8x64 { - crate::kernel!( - #[inline(always)] - fn kernel( - token: Avx512, - values: u8x64, - mask: mask8x64, - ) -> u8x64 { - _mm512_maskz_expand_epi8(u64::from((mask).val) as u64, values.into()) - .simd_into(token) - } - ); - kernel(self, values, mask) - } - #[inline(always)] - fn expand_merge_u8x64( - self, - values: u8x64, - mask: mask8x64, - merge: u8x64, - ) -> u8x64 { - crate::kernel!( - #[inline(always)] - fn kernel( - token: Avx512, - values: u8x64, - mask: mask8x64, - merge: u8x64, - ) -> u8x64 { - _mm512_mask_expand_epi8(merge.into(), u64::from((mask).val) as u64, values.into()) - .simd_into(token) - } - ); - kernel(self, values, mask, merge) - } - #[inline(always)] fn split_u8x64(self, a: u8x64) -> (u8x32, u8x32) { crate::kernel!( #[inline(always)] @@ -14627,6 +16061,82 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn compress_i16x32(self, values: i16x32, mask: mask16x32) -> i16x32 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i16x32, + mask: mask16x32, + ) -> i16x32 { + _mm512_maskz_compress_epi16(u64::from((mask).val) as u32, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_i16x32( + self, + values: i16x32, + mask: mask16x32, + merge: i16x32, + ) -> i16x32 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i16x32, + mask: mask16x32, + merge: i16x32, + ) -> i16x32 { + _mm512_mask_compress_epi16( + merge.into(), + u64::from((mask).val) as u32, + values.into(), + ) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_i16x32(self, values: i16x32, mask: mask16x32) -> i16x32 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i16x32, + mask: mask16x32, + ) -> i16x32 { + _mm512_maskz_expand_epi16(u64::from((mask).val) as u32, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_i16x32( + self, + values: i16x32, + mask: mask16x32, + merge: i16x32, + ) -> i16x32 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i16x32, + mask: mask16x32, + merge: i16x32, + ) -> i16x32 { + _mm512_mask_expand_epi16(merge.into(), u64::from((mask).val) as u32, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn count_ones_i16x32(self, a: i16x32) -> i16x32 { crate::kernel!( #[inline(always)] @@ -15128,6 +16638,82 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn compress_u16x32(self, values: u16x32, mask: mask16x32) -> u16x32 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u16x32, + mask: mask16x32, + ) -> u16x32 { + _mm512_maskz_compress_epi16(u64::from((mask).val) as u32, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_u16x32( + self, + values: u16x32, + mask: mask16x32, + merge: u16x32, + ) -> u16x32 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u16x32, + mask: mask16x32, + merge: u16x32, + ) -> u16x32 { + _mm512_mask_compress_epi16( + merge.into(), + u64::from((mask).val) as u32, + values.into(), + ) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_u16x32(self, values: u16x32, mask: mask16x32) -> u16x32 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u16x32, + mask: mask16x32, + ) -> u16x32 { + _mm512_maskz_expand_epi16(u64::from((mask).val) as u32, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_u16x32( + self, + values: u16x32, + mask: mask16x32, + merge: u16x32, + ) -> u16x32 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u16x32, + mask: mask16x32, + merge: u16x32, + ) -> u16x32 { + _mm512_mask_expand_epi16(merge.into(), u64::from((mask).val) as u32, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn count_ones_u16x32(self, a: u16x32) -> u16x32 { crate::kernel!( #[inline(always)] @@ -15795,6 +17381,82 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn compress_i32x16(self, values: i32x16, mask: mask32x16) -> i32x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i32x16, + mask: mask32x16, + ) -> i32x16 { + _mm512_maskz_compress_epi32(u64::from((mask).val) as u16, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_i32x16( + self, + values: i32x16, + mask: mask32x16, + merge: i32x16, + ) -> i32x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i32x16, + mask: mask32x16, + merge: i32x16, + ) -> i32x16 { + _mm512_mask_compress_epi32( + merge.into(), + u64::from((mask).val) as u16, + values.into(), + ) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_i32x16(self, values: i32x16, mask: mask32x16) -> i32x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i32x16, + mask: mask32x16, + ) -> i32x16 { + _mm512_maskz_expand_epi32(u64::from((mask).val) as u16, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_i32x16( + self, + values: i32x16, + mask: mask32x16, + merge: i32x16, + ) -> i32x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i32x16, + mask: mask32x16, + merge: i32x16, + ) -> i32x16 { + _mm512_mask_expand_epi32(merge.into(), u64::from((mask).val) as u16, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn count_ones_i32x16(self, a: i32x16) -> i32x16 { crate::kernel!( #[inline(always)] @@ -16304,6 +17966,82 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn compress_u32x16(self, values: u32x16, mask: mask32x16) -> u32x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u32x16, + mask: mask32x16, + ) -> u32x16 { + _mm512_maskz_compress_epi32(u64::from((mask).val) as u16, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_u32x16( + self, + values: u32x16, + mask: mask32x16, + merge: u32x16, + ) -> u32x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u32x16, + mask: mask32x16, + merge: u32x16, + ) -> u32x16 { + _mm512_mask_compress_epi32( + merge.into(), + u64::from((mask).val) as u16, + values.into(), + ) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_u32x16(self, values: u32x16, mask: mask32x16) -> u32x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u32x16, + mask: mask32x16, + ) -> u32x16 { + _mm512_maskz_expand_epi32(u64::from((mask).val) as u16, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_u32x16( + self, + values: u32x16, + mask: mask32x16, + merge: u32x16, + ) -> u32x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u32x16, + mask: mask32x16, + merge: u32x16, + ) -> u32x16 { + _mm512_mask_expand_epi32(merge.into(), u64::from((mask).val) as u16, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn count_ones_u32x16(self, a: u32x16) -> u32x16 { crate::kernel!( #[inline(always)] @@ -16968,6 +18706,77 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn compress_f64x8(self, values: f64x8, mask: mask64x8) -> f64x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: f64x8, + mask: mask64x8, + ) -> f64x8 { + _mm512_maskz_compress_pd(u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_f64x8( + self, + values: f64x8, + mask: mask64x8, + merge: f64x8, + ) -> f64x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: f64x8, + mask: mask64x8, + merge: f64x8, + ) -> f64x8 { + _mm512_mask_compress_pd(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_f64x8(self, values: f64x8, mask: mask64x8) -> f64x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: f64x8, + mask: mask64x8, + ) -> f64x8 { + _mm512_maskz_expand_pd(u64::from((mask).val) as u8, values.into()).simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_f64x8( + self, + values: f64x8, + mask: mask64x8, + merge: f64x8, + ) -> f64x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: f64x8, + mask: mask64x8, + merge: f64x8, + ) -> f64x8 { + _mm512_mask_expand_pd(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn neg_f64x8(self, a: f64x8) -> f64x8 { crate::kernel!( #[inline(always)] @@ -17579,6 +19388,78 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn compress_i64x8(self, values: i64x8, mask: mask64x8) -> i64x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i64x8, + mask: mask64x8, + ) -> i64x8 { + _mm512_maskz_compress_epi64(u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_i64x8( + self, + values: i64x8, + mask: mask64x8, + merge: i64x8, + ) -> i64x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i64x8, + mask: mask64x8, + merge: i64x8, + ) -> i64x8 { + _mm512_mask_compress_epi64(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_i64x8(self, values: i64x8, mask: mask64x8) -> i64x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i64x8, + mask: mask64x8, + ) -> i64x8 { + _mm512_maskz_expand_epi64(u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_i64x8( + self, + values: i64x8, + mask: mask64x8, + merge: i64x8, + ) -> i64x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i64x8, + mask: mask64x8, + merge: i64x8, + ) -> i64x8 { + _mm512_mask_expand_epi64(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn count_ones_i64x8(self, a: i64x8) -> i64x8 { crate::kernel!( #[inline(always)] @@ -18052,6 +19933,78 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn compress_u64x8(self, values: u64x8, mask: mask64x8) -> u64x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u64x8, + mask: mask64x8, + ) -> u64x8 { + _mm512_maskz_compress_epi64(u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_u64x8( + self, + values: u64x8, + mask: mask64x8, + merge: u64x8, + ) -> u64x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u64x8, + mask: mask64x8, + merge: u64x8, + ) -> u64x8 { + _mm512_mask_compress_epi64(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_u64x8(self, values: u64x8, mask: mask64x8) -> u64x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u64x8, + mask: mask64x8, + ) -> u64x8 { + _mm512_maskz_expand_epi64(u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_u64x8( + self, + values: u64x8, + mask: mask64x8, + merge: u64x8, + ) -> u64x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u64x8, + mask: mask64x8, + merge: u64x8, + ) -> u64x8 { + _mm512_mask_expand_epi64(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn count_ones_u64x8(self, a: u64x8) -> u64x8 { crate::kernel!( #[inline(always)] diff --git a/fearless_simd/src/generated/fallback.rs b/fearless_simd/src/generated/fallback.rs index b26da3e8f..ddd9d56ca 100644 --- a/fearless_simd/src/generated/fallback.rs +++ b/fearless_simd/src/generated/fallback.rs @@ -141,6 +141,56 @@ impl Simd for Fallback { dest.simd_into(self) } #[inline(always)] + fn compress_f32x4(self, values: f32x4, mask: mask32x4) -> f32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_f32x4( + self, + values: f32x4, + mask: mask32x4, + merge: f32x4, + ) -> f32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_f32x4(self, values: f32x4, mask: mask32x4) -> f32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_f32x4( + self, + values: f32x4, + mask: mask32x4, + merge: f32x4, + ) -> f32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn neg_f32x4(self, a: f32x4) -> f32x4 { [ f32::neg(a[0usize]), @@ -611,6 +661,56 @@ impl Simd for Fallback { dest.simd_into(self) } #[inline(always)] + fn compress_i8x16(self, values: i8x16, mask: mask8x16) -> i8x16 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i8x16( + self, + values: i8x16, + mask: mask8x16, + merge: i8x16, + ) -> i8x16 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i8x16(self, values: i8x16, mask: mask8x16) -> i8x16 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i8x16( + self, + values: i8x16, + mask: mask8x16, + merge: i8x16, + ) -> i8x16 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_i8x16(self, a: i8x16) -> i8x16 { [ i8::try_from(a[0usize].count_ones()).unwrap(), @@ -1649,6 +1749,53 @@ impl Simd for Fallback { Bytes::from_bytes(result) } #[inline(always)] + fn compress_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16 { + self.compress_merge_u8x16(values, mask, u8x16::splat(self, 0)) + } + #[inline(always)] + fn compress_merge_u8x16( + self, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + let mask = self.to_bitmask_mask8x16(mask); + let mut merge_padded = [0u8; 16 + 1]; + merge_padded[..16].copy_from_slice(&*merge); + let mut compacted = merge_padded; + let mut output_lane = 0; + for input_lane in 0..16 { + compacted[output_lane] = values[input_lane]; + output_lane += ((mask >> input_lane) & 1) as usize; + } + compacted[output_lane] = merge_padded[output_lane]; + let mut result = merge; + result.copy_from_slice(&compacted[..16]); + result + } + #[inline(always)] + fn expand_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16 { + self.expand_merge_u8x16(values, mask, u8x16::splat(self, 0)) + } + #[inline(always)] + fn expand_merge_u8x16( + self, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + let mask = self.to_bitmask_mask8x16(mask); + let mut result = merge; + let mut input_lane = 0; + for output_lane in 0..16 { + let bit = ((mask >> output_lane) & 1) as u8; + let keep = 0u8.wrapping_sub(bit); + result[output_lane] = (values[input_lane] & keep) | (result[output_lane] & !keep); + input_lane += usize::from(bit); + } + result + } + #[inline(always)] fn count_ones_u8x16(self, a: u8x16) -> u8x16 { [ u8::try_from(a[0usize].count_ones()).unwrap(), @@ -2318,53 +2465,6 @@ impl Simd for Fallback { .simd_into(self) } #[inline(always)] - fn compress_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16 { - self.compress_merge_u8x16(values, mask, u8x16::splat(self, 0)) - } - #[inline(always)] - fn compress_merge_u8x16( - self, - values: u8x16, - mask: mask8x16, - merge: u8x16, - ) -> u8x16 { - let mask = self.to_bitmask_mask8x16(mask); - let mut merge_padded = [0u8; 16 + 1]; - merge_padded[..16].copy_from_slice(&*merge); - let mut compacted = merge_padded; - let mut output_lane = 0; - for input_lane in 0..16 { - compacted[output_lane] = values[input_lane]; - output_lane += ((mask >> input_lane) & 1) as usize; - } - compacted[output_lane] = merge_padded[output_lane]; - let mut result = merge; - result.copy_from_slice(&compacted[..16]); - result - } - #[inline(always)] - fn expand_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16 { - self.expand_merge_u8x16(values, mask, u8x16::splat(self, 0)) - } - #[inline(always)] - fn expand_merge_u8x16( - self, - values: u8x16, - mask: mask8x16, - merge: u8x16, - ) -> u8x16 { - let mask = self.to_bitmask_mask8x16(mask); - let mut result = merge; - let mut input_lane = 0; - for output_lane in 0..16 { - let bit = ((mask >> output_lane) & 1) as u8; - let keep = 0u8.wrapping_sub(bit); - result[output_lane] = (values[input_lane] & keep) | (result[output_lane] & !keep); - input_lane += usize::from(bit); - } - result - } - #[inline(always)] fn combine_u8x16(self, a: u8x16, b: u8x16) -> u8x32 { let mut result = [0; 32usize]; result[0..16usize].copy_from_slice(&a.val.0); @@ -2999,6 +3099,56 @@ impl Simd for Fallback { dest.simd_into(self) } #[inline(always)] + fn compress_i16x8(self, values: i16x8, mask: mask16x8) -> i16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i16x8( + self, + values: i16x8, + mask: mask16x8, + merge: i16x8, + ) -> i16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i16x8(self, values: i16x8, mask: mask16x8) -> i16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i16x8( + self, + values: i16x8, + mask: mask16x8, + merge: i16x8, + ) -> i16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_i16x8(self, a: i16x8) -> i16x8 { [ i16::try_from(a[0usize].count_ones()).unwrap(), @@ -3624,6 +3774,56 @@ impl Simd for Fallback { dest.simd_into(self) } #[inline(always)] + fn compress_u16x8(self, values: u16x8, mask: mask16x8) -> u16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u16x8( + self, + values: u16x8, + mask: mask16x8, + merge: u16x8, + ) -> u16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u16x8(self, values: u16x8, mask: mask16x8) -> u16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u16x8( + self, + values: u16x8, + mask: mask16x8, + merge: u16x8, + ) -> u16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_u16x8(self, a: u16x8) -> u16x8 { [ u16::try_from(a[0usize].count_ones()).unwrap(), @@ -4542,6 +4742,56 @@ impl Simd for Fallback { dest.simd_into(self) } #[inline(always)] + fn compress_i32x4(self, values: i32x4, mask: mask32x4) -> i32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i32x4( + self, + values: i32x4, + mask: mask32x4, + merge: i32x4, + ) -> i32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i32x4(self, values: i32x4, mask: mask32x4) -> i32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i32x4( + self, + values: i32x4, + mask: mask32x4, + merge: i32x4, + ) -> i32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_i32x4(self, a: i32x4) -> i32x4 { [ a[0usize].count_ones().cast_signed(), @@ -4950,6 +5200,56 @@ impl Simd for Fallback { dest.simd_into(self) } #[inline(always)] + fn compress_u32x4(self, values: u32x4, mask: mask32x4) -> u32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u32x4( + self, + values: u32x4, + mask: mask32x4, + merge: u32x4, + ) -> u32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u32x4(self, values: u32x4, mask: mask32x4) -> u32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u32x4( + self, + values: u32x4, + mask: mask32x4, + merge: u32x4, + ) -> u32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_u32x4(self, a: u32x4) -> u32x4 { [ a[0usize].count_ones(), @@ -5556,6 +5856,56 @@ impl Simd for Fallback { dest.simd_into(self) } #[inline(always)] + fn compress_f64x2(self, values: f64x2, mask: mask64x2) -> f64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_f64x2( + self, + values: f64x2, + mask: mask64x2, + merge: f64x2, + ) -> f64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_f64x2(self, values: f64x2, mask: mask64x2) -> f64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_f64x2( + self, + values: f64x2, + mask: mask64x2, + merge: f64x2, + ) -> f64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn neg_f64x2(self, a: f64x2) -> f64x2 { [f64::neg(a[0usize]), f64::neg(a[1usize])].simd_into(self) } @@ -5898,6 +6248,56 @@ impl Simd for Fallback { dest.simd_into(self) } #[inline(always)] + fn compress_i64x2(self, values: i64x2, mask: mask64x2) -> i64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i64x2( + self, + values: i64x2, + mask: mask64x2, + merge: i64x2, + ) -> i64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i64x2(self, values: i64x2, mask: mask64x2) -> i64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i64x2( + self, + values: i64x2, + mask: mask64x2, + merge: i64x2, + ) -> i64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_i64x2(self, a: i64x2) -> i64x2 { [ i64::from(a[0usize].count_ones()), @@ -6201,20 +6601,70 @@ impl Simd for Fallback { dest.simd_into(self) } #[inline(always)] - fn count_ones_u64x2(self, a: u64x2) -> u64x2 { - [ - u64::from(a[0usize].count_ones()), - u64::from(a[1usize].count_ones()), - ] - .simd_into(self) - } - #[inline(always)] - fn count_zeros_u64x2(self, a: u64x2) -> u64x2 { - [ - u64::from(a[0usize].count_zeros()), - u64::from(a[1usize].count_zeros()), - ] - .simd_into(self) + fn compress_u64x2(self, values: u64x2, mask: mask64x2) -> u64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u64x2( + self, + values: u64x2, + mask: mask64x2, + merge: u64x2, + ) -> u64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u64x2(self, values: u64x2, mask: mask64x2) -> u64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u64x2( + self, + values: u64x2, + mask: mask64x2, + merge: u64x2, + ) -> u64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn count_ones_u64x2(self, a: u64x2) -> u64x2 { + [ + u64::from(a[0usize].count_ones()), + u64::from(a[1usize].count_ones()), + ] + .simd_into(self) + } + #[inline(always)] + fn count_zeros_u64x2(self, a: u64x2) -> u64x2 { + [ + u64::from(a[0usize].count_zeros()), + u64::from(a[1usize].count_zeros()), + ] + .simd_into(self) } #[inline(always)] fn add_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { @@ -6653,6 +7103,56 @@ impl Simd for Fallback { dest.simd_into(self) } #[inline(always)] + fn compress_f32x8(self, values: f32x8, mask: mask32x8) -> f32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_f32x8( + self, + values: f32x8, + mask: mask32x8, + merge: f32x8, + ) -> f32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_f32x8(self, values: f32x8, mask: mask32x8) -> f32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_f32x8( + self, + values: f32x8, + mask: mask32x8, + merge: f32x8, + ) -> f32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn is_nan_f32x8(self, a: f32x8) -> mask32x8 { !a.simd_eq(a) } @@ -6707,6 +7207,56 @@ impl Simd for Fallback { dest.simd_into(self) } #[inline(always)] + fn compress_i8x32(self, values: i8x32, mask: mask8x32) -> i8x32 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i8x32( + self, + values: i8x32, + mask: mask8x32, + merge: i8x32, + ) -> i8x32 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i8x32(self, values: i8x32, mask: mask8x32) -> i8x32 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i8x32( + self, + values: i8x32, + mask: mask8x32, + merge: i8x32, + ) -> i8x32 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn combine_i8x32(self, a: i8x32, b: i8x32) -> i8x64 { let mut result = [0; 64usize]; result[0..32usize].copy_from_slice(&a.val.0); @@ -6921,6 +7471,56 @@ impl Simd for Fallback { dest.simd_into(self) } #[inline(always)] + fn compress_i16x16(self, values: i16x16, mask: mask16x16) -> i16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i16x16( + self, + values: i16x16, + mask: mask16x16, + merge: i16x16, + ) -> i16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i16x16(self, values: i16x16, mask: mask16x16) -> i16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i16x16( + self, + values: i16x16, + mask: mask16x16, + merge: i16x16, + ) -> i16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn combine_i16x16(self, a: i16x16, b: i16x16) -> i16x32 { let mut result = [0; 32usize]; result[0..16usize].copy_from_slice(&a.val.0); @@ -6943,6 +7543,56 @@ impl Simd for Fallback { dest.simd_into(self) } #[inline(always)] + fn compress_u16x16(self, values: u16x16, mask: mask16x16) -> u16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u16x16( + self, + values: u16x16, + mask: mask16x16, + merge: u16x16, + ) -> u16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u16x16(self, values: u16x16, mask: mask16x16) -> u16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u16x16( + self, + values: u16x16, + mask: mask16x16, + merge: u16x16, + ) -> u16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn combine_u16x16(self, a: u16x16, b: u16x16) -> u16x32 { let mut result = [0; 32usize]; result[0..16usize].copy_from_slice(&a.val.0); @@ -7021,6 +7671,56 @@ impl Simd for Fallback { dest.simd_into(self) } #[inline(always)] + fn compress_i32x8(self, values: i32x8, mask: mask32x8) -> i32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i32x8( + self, + values: i32x8, + mask: mask32x8, + merge: i32x8, + ) -> i32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i32x8(self, values: i32x8, mask: mask32x8) -> i32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i32x8( + self, + values: i32x8, + mask: mask32x8, + merge: i32x8, + ) -> i32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn combine_i32x8(self, a: i32x8, b: i32x8) -> i32x16 { let mut result = [0; 16usize]; result[0..8usize].copy_from_slice(&a.val.0); @@ -7043,6 +7743,56 @@ impl Simd for Fallback { dest.simd_into(self) } #[inline(always)] + fn compress_u32x8(self, values: u32x8, mask: mask32x8) -> u32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u32x8( + self, + values: u32x8, + mask: mask32x8, + merge: u32x8, + ) -> u32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u32x8(self, values: u32x8, mask: mask32x8) -> u32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u32x8( + self, + values: u32x8, + mask: mask32x8, + merge: u32x8, + ) -> u32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn combine_u32x8(self, a: u32x8, b: u32x8) -> u32x16 { let mut result = [0; 16usize]; result[0..8usize].copy_from_slice(&a.val.0); @@ -7121,6 +7871,56 @@ impl Simd for Fallback { dest.simd_into(self) } #[inline(always)] + fn compress_f64x4(self, values: f64x4, mask: mask64x4) -> f64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_f64x4( + self, + values: f64x4, + mask: mask64x4, + merge: f64x4, + ) -> f64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_f64x4(self, values: f64x4, mask: mask64x4) -> f64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_f64x4( + self, + values: f64x4, + mask: mask64x4, + merge: f64x4, + ) -> f64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn is_nan_f64x4(self, a: f64x4) -> mask64x4 { !a.simd_eq(a) } @@ -7175,6 +7975,56 @@ impl Simd for Fallback { dest.simd_into(self) } #[inline(always)] + fn compress_i64x4(self, values: i64x4, mask: mask64x4) -> i64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i64x4( + self, + values: i64x4, + mask: mask64x4, + merge: i64x4, + ) -> i64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i64x4(self, values: i64x4, mask: mask64x4) -> i64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i64x4( + self, + values: i64x4, + mask: mask64x4, + merge: i64x4, + ) -> i64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn combine_i64x4(self, a: i64x4, b: i64x4) -> i64x8 { let mut result = [0; 8usize]; result[0..4usize].copy_from_slice(&a.val.0); @@ -7197,6 +8047,56 @@ impl Simd for Fallback { dest.simd_into(self) } #[inline(always)] + fn compress_u64x4(self, values: u64x4, mask: mask64x4) -> u64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u64x4( + self, + values: u64x4, + mask: mask64x4, + merge: u64x4, + ) -> u64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u64x4(self, values: u64x4, mask: mask64x4) -> u64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u64x4( + self, + values: u64x4, + mask: mask64x4, + merge: u64x4, + ) -> u64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn combine_u64x4(self, a: u64x4, b: u64x4) -> u64x8 { let mut result = [0; 8usize]; result[0..4usize].copy_from_slice(&a.val.0); @@ -7275,6 +8175,56 @@ impl Simd for Fallback { dest.simd_into(self) } #[inline(always)] + fn compress_f32x16(self, values: f32x16, mask: mask32x16) -> f32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_f32x16( + self, + values: f32x16, + mask: mask32x16, + merge: f32x16, + ) -> f32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_f32x16(self, values: f32x16, mask: mask32x16) -> f32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_f32x16( + self, + values: f32x16, + mask: mask32x16, + merge: f32x16, + ) -> f32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn is_nan_f32x16(self, a: f32x16) -> mask32x16 { !a.simd_eq(a) } @@ -7322,6 +8272,56 @@ impl Simd for Fallback { dest.simd_into(self) } #[inline(always)] + fn compress_i8x64(self, values: i8x64, mask: mask8x64) -> i8x64 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i8x64( + self, + values: i8x64, + mask: mask8x64, + merge: i8x64, + ) -> i8x64 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i8x64(self, values: i8x64, mask: mask8x64) -> i8x64 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i8x64( + self, + values: i8x64, + mask: mask8x64, + merge: i8x64, + ) -> i8x64 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_i8x64(self, a: i8x64) -> (i8x32, i8x32) { let mut b0 = [0; 32usize]; let mut b1 = [0; 32usize]; @@ -7515,6 +8515,56 @@ impl Simd for Fallback { dest.simd_into(self) } #[inline(always)] + fn compress_i16x32(self, values: i16x32, mask: mask16x32) -> i16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i16x32( + self, + values: i16x32, + mask: mask16x32, + merge: i16x32, + ) -> i16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i16x32(self, values: i16x32, mask: mask16x32) -> i16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i16x32( + self, + values: i16x32, + mask: mask16x32, + merge: i16x32, + ) -> i16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_i16x32(self, a: i16x32) -> (i16x16, i16x16) { let mut b0 = [0; 16usize]; let mut b1 = [0; 16usize]; @@ -7530,6 +8580,56 @@ impl Simd for Fallback { dest.simd_into(self) } #[inline(always)] + fn compress_u16x32(self, values: u16x32, mask: mask16x32) -> u16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u16x32( + self, + values: u16x32, + mask: mask16x32, + merge: u16x32, + ) -> u16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u16x32(self, values: u16x32, mask: mask16x32) -> u16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u16x32( + self, + values: u16x32, + mask: mask16x32, + merge: u16x32, + ) -> u16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_u16x32(self, a: u16x32) -> (u16x16, u16x16) { let mut b0 = [0; 16usize]; let mut b1 = [0; 16usize]; @@ -7594,6 +8694,56 @@ impl Simd for Fallback { dest.simd_into(self) } #[inline(always)] + fn compress_i32x16(self, values: i32x16, mask: mask32x16) -> i32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i32x16( + self, + values: i32x16, + mask: mask32x16, + merge: i32x16, + ) -> i32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i32x16(self, values: i32x16, mask: mask32x16) -> i32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i32x16( + self, + values: i32x16, + mask: mask32x16, + merge: i32x16, + ) -> i32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_i32x16(self, a: i32x16) -> (i32x8, i32x8) { let mut b0 = [0; 8usize]; let mut b1 = [0; 8usize]; @@ -7609,6 +8759,56 @@ impl Simd for Fallback { dest.simd_into(self) } #[inline(always)] + fn compress_u32x16(self, values: u32x16, mask: mask32x16) -> u32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u32x16( + self, + values: u32x16, + mask: mask32x16, + merge: u32x16, + ) -> u32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u32x16(self, values: u32x16, mask: mask32x16) -> u32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u32x16( + self, + values: u32x16, + mask: mask32x16, + merge: u32x16, + ) -> u32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_u32x16(self, a: u32x16) -> (u32x8, u32x8) { let mut b0 = [0; 8usize]; let mut b1 = [0; 8usize]; @@ -7673,6 +8873,56 @@ impl Simd for Fallback { dest.simd_into(self) } #[inline(always)] + fn compress_f64x8(self, values: f64x8, mask: mask64x8) -> f64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_f64x8( + self, + values: f64x8, + mask: mask64x8, + merge: f64x8, + ) -> f64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_f64x8(self, values: f64x8, mask: mask64x8) -> f64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_f64x8( + self, + values: f64x8, + mask: mask64x8, + merge: f64x8, + ) -> f64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn is_nan_f64x8(self, a: f64x8) -> mask64x8 { !a.simd_eq(a) } @@ -7720,6 +8970,56 @@ impl Simd for Fallback { dest.simd_into(self) } #[inline(always)] + fn compress_i64x8(self, values: i64x8, mask: mask64x8) -> i64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i64x8( + self, + values: i64x8, + mask: mask64x8, + merge: i64x8, + ) -> i64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i64x8(self, values: i64x8, mask: mask64x8) -> i64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i64x8( + self, + values: i64x8, + mask: mask64x8, + merge: i64x8, + ) -> i64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_i64x8(self, a: i64x8) -> (i64x4, i64x4) { let mut b0 = [0; 4usize]; let mut b1 = [0; 4usize]; @@ -7735,6 +9035,56 @@ impl Simd for Fallback { dest.simd_into(self) } #[inline(always)] + fn compress_u64x8(self, values: u64x8, mask: mask64x8) -> u64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u64x8( + self, + values: u64x8, + mask: mask64x8, + merge: u64x8, + ) -> u64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u64x8(self, values: u64x8, mask: mask64x8) -> u64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u64x8( + self, + values: u64x8, + mask: mask64x8, + merge: u64x8, + ) -> u64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_u64x8(self, a: u64x8) -> (u64x4, u64x4) { let mut b0 = [0; 4usize]; let mut b1 = [0; 4usize]; diff --git a/fearless_simd/src/generated/neon.rs b/fearless_simd/src/generated/neon.rs index ef02ec3fd..526afc83f 100644 --- a/fearless_simd/src/generated/neon.rs +++ b/fearless_simd/src/generated/neon.rs @@ -159,6 +159,56 @@ impl Simd for Neon { }) } #[inline(always)] + fn compress_f32x4(self, values: f32x4, mask: mask32x4) -> f32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_f32x4( + self, + values: f32x4, + mask: mask32x4, + merge: f32x4, + ) -> f32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_f32x4(self, values: f32x4, mask: mask32x4) -> f32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_f32x4( + self, + values: f32x4, + mask: mask32x4, + merge: f32x4, + ) -> f32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn neg_f32x4(self, a: f32x4) -> f32x4 { crate::kernel!( #[inline(always)] @@ -662,6 +712,56 @@ impl Simd for Neon { }) } #[inline(always)] + fn compress_i8x16(self, values: i8x16, mask: mask8x16) -> i8x16 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i8x16( + self, + values: i8x16, + mask: mask8x16, + merge: i8x16, + ) -> i8x16 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i8x16(self, values: i8x16, mask: mask8x16) -> i8x16 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i8x16( + self, + values: i8x16, + mask: mask8x16, + merge: i8x16, + ) -> i8x16 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_i8x16(self, a: i8x16) -> i8x16 { crate::kernel!( #[inline(always)] @@ -1124,6 +1224,130 @@ impl Simd for Neon { kernel(self, a, b, indices) } #[inline(always)] + fn compress_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16 { + let mask_bits = self.to_bitmask_mask8x16(mask); + let mut control = [u8::MAX; 16]; + let mut output_lane = 0; + for block in 0..16 / 8 { + let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; + let packed = crate::support::COMPRESS_8_CONTROLS[block_mask]; + let base = (block * 8) as u64 * 0x0101_0101_0101_0101; + let adjusted = + ((packed & 0x7f7f_7f7f_7f7f_7f7f) + base) | (packed & 0x8080_8080_8080_8080); + let adjusted = adjusted.to_le_bytes(); + let write_len = core::cmp::min(8, 16 - output_lane); + control[output_lane..output_lane + write_len].copy_from_slice(&adjusted[..write_len]); + output_lane += crate::support::COMPACT_8_COUNTS[block_mask] as usize; + } + let control = u8x16::simd_from(self, control); + self.swizzle_dyn_precise_u8x16(values, control) + } + #[inline(always)] + fn compress_merge_u8x16( + self, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + let mask_bits = self.to_bitmask_mask8x16(mask); + let mut control = [u8::MAX; 16]; + let mut output_lane = 0; + for block in 0..16 / 8 { + let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; + let packed = crate::support::COMPRESS_8_CONTROLS[block_mask]; + let base = (block * 8) as u64 * 0x0101_0101_0101_0101; + let adjusted = + ((packed & 0x7f7f_7f7f_7f7f_7f7f) + base) | (packed & 0x8080_8080_8080_8080); + let adjusted = adjusted.to_le_bytes(); + let write_len = core::cmp::min(8, 16 - output_lane); + control[output_lane..output_lane + write_len].copy_from_slice(&adjusted[..write_len]); + output_lane += crate::support::COMPACT_8_COUNTS[block_mask] as usize; + } + let control = u8x16::simd_from(self, control); + { + crate::kernel!( + #[inline(always)] + fn merge_swizzle( + token: Neon, + values: u8x16, + control: u8x16, + merge: u8x16, + ) -> u8x16 { + let values: uint8x16_t = values.into(); + let control: uint8x16_t = control.into(); + let merge: uint8x16_t = merge.into(); + let result = vqtbx1q_u8(merge, values, control); + u8x16 { + val: crate::support::Aligned128(result), + simd: token, + } + } + ); + merge_swizzle(self, values, control, merge) + } + } + #[inline(always)] + fn expand_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16 { + let mask_bits = self.to_bitmask_mask8x16(mask); + let mut control = [u8::MAX; 16]; + let mut input_lane = 0; + for block in 0..16 / 8 { + let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; + let packed = crate::support::EXPAND_8_CONTROLS[block_mask]; + let base = input_lane as u64 * 0x0101_0101_0101_0101; + let adjusted = + ((packed & 0x7f7f_7f7f_7f7f_7f7f) + base) | (packed & 0x8080_8080_8080_8080); + let output_lane = block * 8; + control[output_lane..output_lane + 8].copy_from_slice(&adjusted.to_le_bytes()); + input_lane += crate::support::COMPACT_8_COUNTS[block_mask] as usize; + } + let control = u8x16::simd_from(self, control); + self.swizzle_dyn_precise_u8x16(values, control) + } + #[inline(always)] + fn expand_merge_u8x16( + self, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + let mask_bits = self.to_bitmask_mask8x16(mask); + let mut control = [u8::MAX; 16]; + let mut input_lane = 0; + for block in 0..16 / 8 { + let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; + let packed = crate::support::EXPAND_8_CONTROLS[block_mask]; + let base = input_lane as u64 * 0x0101_0101_0101_0101; + let adjusted = + ((packed & 0x7f7f_7f7f_7f7f_7f7f) + base) | (packed & 0x8080_8080_8080_8080); + let output_lane = block * 8; + control[output_lane..output_lane + 8].copy_from_slice(&adjusted.to_le_bytes()); + input_lane += crate::support::COMPACT_8_COUNTS[block_mask] as usize; + } + let control = u8x16::simd_from(self, control); + { + crate::kernel!( + #[inline(always)] + fn merge_swizzle( + token: Neon, + values: u8x16, + control: u8x16, + merge: u8x16, + ) -> u8x16 { + let values: uint8x16_t = values.into(); + let control: uint8x16_t = control.into(); + let merge: uint8x16_t = merge.into(); + let result = vqtbx1q_u8(merge, values, control); + u8x16 { + val: crate::support::Aligned128(result), + simd: token, + } + } + ); + merge_swizzle(self, values, control, merge) + } + } + #[inline(always)] fn count_ones_u8x16(self, a: u8x16) -> u8x16 { crate::kernel!( #[inline(always)] @@ -1434,130 +1658,6 @@ impl Simd for Neon { kernel(self, a, b, c) } #[inline(always)] - fn compress_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16 { - let mask_bits = self.to_bitmask_mask8x16(mask); - let mut control = [u8::MAX; 16]; - let mut output_lane = 0; - for block in 0..16 / 8 { - let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; - let packed = crate::support::COMPRESS_8_CONTROLS[block_mask]; - let base = (block * 8) as u64 * 0x0101_0101_0101_0101; - let adjusted = - ((packed & 0x7f7f_7f7f_7f7f_7f7f) + base) | (packed & 0x8080_8080_8080_8080); - let adjusted = adjusted.to_le_bytes(); - let write_len = core::cmp::min(8, 16 - output_lane); - control[output_lane..output_lane + write_len].copy_from_slice(&adjusted[..write_len]); - output_lane += crate::support::COMPACT_8_COUNTS[block_mask] as usize; - } - let control = u8x16::simd_from(self, control); - self.swizzle_dyn_precise_u8x16(values, control) - } - #[inline(always)] - fn compress_merge_u8x16( - self, - values: u8x16, - mask: mask8x16, - merge: u8x16, - ) -> u8x16 { - let mask_bits = self.to_bitmask_mask8x16(mask); - let mut control = [u8::MAX; 16]; - let mut output_lane = 0; - for block in 0..16 / 8 { - let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; - let packed = crate::support::COMPRESS_8_CONTROLS[block_mask]; - let base = (block * 8) as u64 * 0x0101_0101_0101_0101; - let adjusted = - ((packed & 0x7f7f_7f7f_7f7f_7f7f) + base) | (packed & 0x8080_8080_8080_8080); - let adjusted = adjusted.to_le_bytes(); - let write_len = core::cmp::min(8, 16 - output_lane); - control[output_lane..output_lane + write_len].copy_from_slice(&adjusted[..write_len]); - output_lane += crate::support::COMPACT_8_COUNTS[block_mask] as usize; - } - let control = u8x16::simd_from(self, control); - { - crate::kernel!( - #[inline(always)] - fn merge_swizzle( - token: Neon, - values: u8x16, - control: u8x16, - merge: u8x16, - ) -> u8x16 { - let values: uint8x16_t = values.into(); - let control: uint8x16_t = control.into(); - let merge: uint8x16_t = merge.into(); - let result = vqtbx1q_u8(merge, values, control); - u8x16 { - val: crate::support::Aligned128(result), - simd: token, - } - } - ); - merge_swizzle(self, values, control, merge) - } - } - #[inline(always)] - fn expand_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16 { - let mask_bits = self.to_bitmask_mask8x16(mask); - let mut control = [u8::MAX; 16]; - let mut input_lane = 0; - for block in 0..16 / 8 { - let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; - let packed = crate::support::EXPAND_8_CONTROLS[block_mask]; - let base = input_lane as u64 * 0x0101_0101_0101_0101; - let adjusted = - ((packed & 0x7f7f_7f7f_7f7f_7f7f) + base) | (packed & 0x8080_8080_8080_8080); - let output_lane = block * 8; - control[output_lane..output_lane + 8].copy_from_slice(&adjusted.to_le_bytes()); - input_lane += crate::support::COMPACT_8_COUNTS[block_mask] as usize; - } - let control = u8x16::simd_from(self, control); - self.swizzle_dyn_precise_u8x16(values, control) - } - #[inline(always)] - fn expand_merge_u8x16( - self, - values: u8x16, - mask: mask8x16, - merge: u8x16, - ) -> u8x16 { - let mask_bits = self.to_bitmask_mask8x16(mask); - let mut control = [u8::MAX; 16]; - let mut input_lane = 0; - for block in 0..16 / 8 { - let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; - let packed = crate::support::EXPAND_8_CONTROLS[block_mask]; - let base = input_lane as u64 * 0x0101_0101_0101_0101; - let adjusted = - ((packed & 0x7f7f_7f7f_7f7f_7f7f) + base) | (packed & 0x8080_8080_8080_8080); - let output_lane = block * 8; - control[output_lane..output_lane + 8].copy_from_slice(&adjusted.to_le_bytes()); - input_lane += crate::support::COMPACT_8_COUNTS[block_mask] as usize; - } - let control = u8x16::simd_from(self, control); - { - crate::kernel!( - #[inline(always)] - fn merge_swizzle( - token: Neon, - values: u8x16, - control: u8x16, - merge: u8x16, - ) -> u8x16 { - let values: uint8x16_t = values.into(); - let control: uint8x16_t = control.into(); - let merge: uint8x16_t = merge.into(); - let result = vqtbx1q_u8(merge, values, control); - u8x16 { - val: crate::support::Aligned128(result), - simd: token, - } - } - ); - merge_swizzle(self, values, control, merge) - } - } - #[inline(always)] fn combine_u8x16(self, a: u8x16, b: u8x16) -> u8x32 { u8x32 { val: crate::support::Aligned256(uint8x16x2_t(a.val.0, b.val.0)), @@ -1875,6 +1975,56 @@ impl Simd for Neon { }) } #[inline(always)] + fn compress_i16x8(self, values: i16x8, mask: mask16x8) -> i16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i16x8( + self, + values: i16x8, + mask: mask16x8, + merge: i16x8, + ) -> i16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i16x8(self, values: i16x8, mask: mask16x8) -> i16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i16x8( + self, + values: i16x8, + mask: mask16x8, + merge: i16x8, + ) -> i16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_i16x8(self, a: i16x8) -> i16x8 { crate::kernel!( #[inline(always)] @@ -2302,6 +2452,56 @@ impl Simd for Neon { }) } #[inline(always)] + fn compress_u16x8(self, values: u16x8, mask: mask16x8) -> u16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u16x8( + self, + values: u16x8, + mask: mask16x8, + merge: u16x8, + ) -> u16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u16x8(self, values: u16x8, mask: mask16x8) -> u16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u16x8( + self, + values: u16x8, + mask: mask16x8, + merge: u16x8, + ) -> u16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_u16x8(self, a: u16x8) -> u16x8 { crate::kernel!( #[inline(always)] @@ -2961,6 +3161,56 @@ impl Simd for Neon { }) } #[inline(always)] + fn compress_i32x4(self, values: i32x4, mask: mask32x4) -> i32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i32x4( + self, + values: i32x4, + mask: mask32x4, + merge: i32x4, + ) -> i32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i32x4(self, values: i32x4, mask: mask32x4) -> i32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i32x4( + self, + values: i32x4, + mask: mask32x4, + merge: i32x4, + ) -> i32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_i32x4(self, a: i32x4) -> i32x4 { crate::kernel!( #[inline(always)] @@ -3399,6 +3649,56 @@ impl Simd for Neon { }) } #[inline(always)] + fn compress_u32x4(self, values: u32x4, mask: mask32x4) -> u32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u32x4( + self, + values: u32x4, + mask: mask32x4, + merge: u32x4, + ) -> u32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u32x4(self, values: u32x4, mask: mask32x4) -> u32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u32x4( + self, + values: u32x4, + mask: mask32x4, + merge: u32x4, + ) -> u32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_u32x4(self, a: u32x4) -> u32x4 { crate::kernel!( #[inline(always)] @@ -4066,6 +4366,56 @@ impl Simd for Neon { }) } #[inline(always)] + fn compress_f64x2(self, values: f64x2, mask: mask64x2) -> f64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_f64x2( + self, + values: f64x2, + mask: mask64x2, + merge: f64x2, + ) -> f64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_f64x2(self, values: f64x2, mask: mask64x2) -> f64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_f64x2( + self, + values: f64x2, + mask: mask64x2, + merge: f64x2, + ) -> f64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn neg_f64x2(self, a: f64x2) -> f64x2 { crate::kernel!( #[inline(always)] @@ -4572,6 +4922,56 @@ impl Simd for Neon { }) } #[inline(always)] + fn compress_i64x2(self, values: i64x2, mask: mask64x2) -> i64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i64x2( + self, + values: i64x2, + mask: mask64x2, + merge: i64x2, + ) -> i64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i64x2(self, values: i64x2, mask: mask64x2) -> i64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i64x2( + self, + values: i64x2, + mask: mask64x2, + merge: i64x2, + ) -> i64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_i64x2(self, a: i64x2) -> i64x2 { crate::kernel!( #[inline(always)] @@ -4973,6 +5373,56 @@ impl Simd for Neon { }) } #[inline(always)] + fn compress_u64x2(self, values: u64x2, mask: mask64x2) -> u64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u64x2( + self, + values: u64x2, + mask: mask64x2, + merge: u64x2, + ) -> u64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u64x2(self, values: u64x2, mask: mask64x2) -> u64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u64x2( + self, + values: u64x2, + mask: mask64x2, + merge: u64x2, + ) -> u64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_u64x2(self, a: u64x2) -> u64x2 { crate::kernel!( #[inline(always)] @@ -5587,6 +6037,56 @@ impl Simd for Neon { }) } #[inline(always)] + fn compress_f32x8(self, values: f32x8, mask: mask32x8) -> f32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_f32x8( + self, + values: f32x8, + mask: mask32x8, + merge: f32x8, + ) -> f32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_f32x8(self, values: f32x8, mask: mask32x8) -> f32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_f32x8( + self, + values: f32x8, + mask: mask32x8, + merge: f32x8, + ) -> f32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn is_nan_f32x8(self, a: f32x8) -> mask32x8 { !a.simd_eq(a) } @@ -5678,6 +6178,56 @@ impl Simd for Neon { }) } #[inline(always)] + fn compress_i8x32(self, values: i8x32, mask: mask8x32) -> i8x32 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i8x32( + self, + values: i8x32, + mask: mask8x32, + merge: i8x32, + ) -> i8x32 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i8x32(self, values: i8x32, mask: mask8x32) -> i8x32 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i8x32( + self, + values: i8x32, + mask: mask8x32, + merge: i8x32, + ) -> i8x32 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn combine_i8x32(self, a: i8x32, b: i8x32) -> i8x64 { i8x64 { val: crate::support::Aligned512(int8x16x4_t( @@ -6070,6 +6620,56 @@ impl Simd for Neon { }) } #[inline(always)] + fn compress_i16x16(self, values: i16x16, mask: mask16x16) -> i16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i16x16( + self, + values: i16x16, + mask: mask16x16, + merge: i16x16, + ) -> i16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i16x16(self, values: i16x16, mask: mask16x16) -> i16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i16x16( + self, + values: i16x16, + mask: mask16x16, + merge: i16x16, + ) -> i16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn combine_i16x16(self, a: i16x16, b: i16x16) -> i16x32 { i16x32 { val: crate::support::Aligned512(int16x8x4_t( @@ -6126,7 +6726,57 @@ impl Simd for Neon { Bytes::from_bytes(u8x32 { val: crate::support::Aligned256(result), simd: self, - }) + }) + } + #[inline(always)] + fn compress_u16x16(self, values: u16x16, mask: mask16x16) -> u16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u16x16( + self, + values: u16x16, + mask: mask16x16, + merge: u16x16, + ) -> u16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u16x16(self, values: u16x16, mask: mask16x16) -> u16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u16x16( + self, + values: u16x16, + mask: mask16x16, + merge: u16x16, + ) -> u16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) } #[inline(always)] fn combine_u16x16(self, a: u16x16, b: u16x16) -> u16x32 { @@ -6266,6 +6916,56 @@ impl Simd for Neon { }) } #[inline(always)] + fn compress_i32x8(self, values: i32x8, mask: mask32x8) -> i32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i32x8( + self, + values: i32x8, + mask: mask32x8, + merge: i32x8, + ) -> i32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i32x8(self, values: i32x8, mask: mask32x8) -> i32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i32x8( + self, + values: i32x8, + mask: mask32x8, + merge: i32x8, + ) -> i32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn combine_i32x8(self, a: i32x8, b: i32x8) -> i32x16 { i32x16 { val: crate::support::Aligned512(int32x4x4_t( @@ -6325,6 +7025,56 @@ impl Simd for Neon { }) } #[inline(always)] + fn compress_u32x8(self, values: u32x8, mask: mask32x8) -> u32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u32x8( + self, + values: u32x8, + mask: mask32x8, + merge: u32x8, + ) -> u32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u32x8(self, values: u32x8, mask: mask32x8) -> u32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u32x8( + self, + values: u32x8, + mask: mask32x8, + merge: u32x8, + ) -> u32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn combine_u32x8(self, a: u32x8, b: u32x8) -> u32x16 { u32x16 { val: crate::support::Aligned512(uint32x4x4_t( @@ -6462,6 +7212,56 @@ impl Simd for Neon { }) } #[inline(always)] + fn compress_f64x4(self, values: f64x4, mask: mask64x4) -> f64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_f64x4( + self, + values: f64x4, + mask: mask64x4, + merge: f64x4, + ) -> f64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_f64x4(self, values: f64x4, mask: mask64x4) -> f64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_f64x4( + self, + values: f64x4, + mask: mask64x4, + merge: f64x4, + ) -> f64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn is_nan_f64x4(self, a: f64x4) -> mask64x4 { !a.simd_eq(a) } @@ -6553,6 +7353,56 @@ impl Simd for Neon { }) } #[inline(always)] + fn compress_i64x4(self, values: i64x4, mask: mask64x4) -> i64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i64x4( + self, + values: i64x4, + mask: mask64x4, + merge: i64x4, + ) -> i64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i64x4(self, values: i64x4, mask: mask64x4) -> i64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i64x4( + self, + values: i64x4, + mask: mask64x4, + merge: i64x4, + ) -> i64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn combine_i64x4(self, a: i64x4, b: i64x4) -> i64x8 { i64x8 { val: crate::support::Aligned512(int64x2x4_t( @@ -6612,6 +7462,56 @@ impl Simd for Neon { }) } #[inline(always)] + fn compress_u64x4(self, values: u64x4, mask: mask64x4) -> u64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u64x4( + self, + values: u64x4, + mask: mask64x4, + merge: u64x4, + ) -> u64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u64x4(self, values: u64x4, mask: mask64x4) -> u64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u64x4( + self, + values: u64x4, + mask: mask64x4, + merge: u64x4, + ) -> u64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn combine_u64x4(self, a: u64x4, b: u64x4) -> u64x8 { u64x8 { val: crate::support::Aligned512(uint64x2x4_t( @@ -6767,6 +7667,56 @@ impl Simd for Neon { }) } #[inline(always)] + fn compress_f32x16(self, values: f32x16, mask: mask32x16) -> f32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_f32x16( + self, + values: f32x16, + mask: mask32x16, + merge: f32x16, + ) -> f32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_f32x16(self, values: f32x16, mask: mask32x16) -> f32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_f32x16( + self, + values: f32x16, + mask: mask32x16, + merge: f32x16, + ) -> f32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn is_nan_f32x16(self, a: f32x16) -> mask32x16 { !a.simd_eq(a) } @@ -6867,6 +7817,56 @@ impl Simd for Neon { }) } #[inline(always)] + fn compress_i8x64(self, values: i8x64, mask: mask8x64) -> i8x64 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i8x64( + self, + values: i8x64, + mask: mask8x64, + merge: i8x64, + ) -> i8x64 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i8x64(self, values: i8x64, mask: mask8x64) -> i8x64 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i8x64( + self, + values: i8x64, + mask: mask8x64, + merge: i8x64, + ) -> i8x64 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_i8x64(self, a: i8x64) -> (i8x32, i8x32) { ( i8x32 { @@ -7279,6 +8279,56 @@ impl Simd for Neon { }) } #[inline(always)] + fn compress_i16x32(self, values: i16x32, mask: mask16x32) -> i16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i16x32( + self, + values: i16x32, + mask: mask16x32, + merge: i16x32, + ) -> i16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i16x32(self, values: i16x32, mask: mask16x32) -> i16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i16x32( + self, + values: i16x32, + mask: mask16x32, + merge: i16x32, + ) -> i16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_i16x32(self, a: i16x32) -> (i16x16, i16x16) { ( i16x16 { @@ -7347,6 +8397,56 @@ impl Simd for Neon { }) } #[inline(always)] + fn compress_u16x32(self, values: u16x32, mask: mask16x32) -> u16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u16x32( + self, + values: u16x32, + mask: mask16x32, + merge: u16x32, + ) -> u16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u16x32(self, values: u16x32, mask: mask16x32) -> u16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u16x32( + self, + values: u16x32, + mask: mask16x32, + merge: u16x32, + ) -> u16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_u16x32(self, a: u16x32) -> (u16x16, u16x16) { ( u16x16 { @@ -7489,6 +8589,56 @@ impl Simd for Neon { }) } #[inline(always)] + fn compress_i32x16(self, values: i32x16, mask: mask32x16) -> i32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i32x16( + self, + values: i32x16, + mask: mask32x16, + merge: i32x16, + ) -> i32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i32x16(self, values: i32x16, mask: mask32x16) -> i32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i32x16( + self, + values: i32x16, + mask: mask32x16, + merge: i32x16, + ) -> i32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_i32x16(self, a: i32x16) -> (i32x8, i32x8) { ( i32x8 { @@ -7557,6 +8707,56 @@ impl Simd for Neon { }) } #[inline(always)] + fn compress_u32x16(self, values: u32x16, mask: mask32x16) -> u32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u32x16( + self, + values: u32x16, + mask: mask32x16, + merge: u32x16, + ) -> u32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u32x16(self, values: u32x16, mask: mask32x16) -> u32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u32x16( + self, + values: u32x16, + mask: mask32x16, + merge: u32x16, + ) -> u32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_u32x16(self, a: u32x16) -> (u32x8, u32x8) { ( u32x8 { @@ -7699,6 +8899,56 @@ impl Simd for Neon { }) } #[inline(always)] + fn compress_f64x8(self, values: f64x8, mask: mask64x8) -> f64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_f64x8( + self, + values: f64x8, + mask: mask64x8, + merge: f64x8, + ) -> f64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_f64x8(self, values: f64x8, mask: mask64x8) -> f64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_f64x8( + self, + values: f64x8, + mask: mask64x8, + merge: f64x8, + ) -> f64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn is_nan_f64x8(self, a: f64x8) -> mask64x8 { !a.simd_eq(a) } @@ -7799,6 +9049,56 @@ impl Simd for Neon { }) } #[inline(always)] + fn compress_i64x8(self, values: i64x8, mask: mask64x8) -> i64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i64x8( + self, + values: i64x8, + mask: mask64x8, + merge: i64x8, + ) -> i64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i64x8(self, values: i64x8, mask: mask64x8) -> i64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i64x8( + self, + values: i64x8, + mask: mask64x8, + merge: i64x8, + ) -> i64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_i64x8(self, a: i64x8) -> (i64x4, i64x4) { ( i64x4 { @@ -7867,6 +9167,56 @@ impl Simd for Neon { }) } #[inline(always)] + fn compress_u64x8(self, values: u64x8, mask: mask64x8) -> u64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u64x8( + self, + values: u64x8, + mask: mask64x8, + merge: u64x8, + ) -> u64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u64x8(self, values: u64x8, mask: mask64x8) -> u64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u64x8( + self, + values: u64x8, + mask: mask64x8, + merge: u64x8, + ) -> u64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_u64x8(self, a: u64x8) -> (u64x4, u64x4) { ( u64x4 { diff --git a/fearless_simd/src/generated/simd_trait.rs b/fearless_simd/src/generated/simd_trait.rs index b7cf8949e..5a936ef1e 100644 --- a/fearless_simd/src/generated/simd_trait.rs +++ b/fearless_simd/src/generated/simd_trait.rs @@ -294,6 +294,24 @@ pub trait Simd: indices, )) } + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_f32x4(self, values: f32x4, mask: mask32x4) -> f32x4; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_f32x4( + self, + values: f32x4, + mask: mask32x4, + merge: f32x4, + ) -> f32x4; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_f32x4(self, values: f32x4, mask: mask32x4) -> f32x4; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_f32x4( + self, + values: f32x4, + mask: mask32x4, + merge: f32x4, + ) -> f32x4; #[doc = "Negate each element of the vector."] fn neg_f32x4(self, a: f32x4) -> f32x4; #[doc = "Compute the square root of each element.\n\nNegative elements other than `-0.0` will become NaN."] @@ -474,6 +492,24 @@ pub trait Simd: indices, )) } + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_i8x16(self, values: i8x16, mask: mask8x16) -> i8x16; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_i8x16( + self, + values: i8x16, + mask: mask8x16, + merge: i8x16, + ) -> i8x16; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_i8x16(self, values: i8x16, mask: mask8x16) -> i8x16; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_i8x16( + self, + values: i8x16, + mask: mask8x16, + merge: i8x16, + ) -> i8x16; #[doc = "Return the number of ones in the binary representation of each element."] fn count_ones_i8x16(self, a: i8x16) -> i8x16; #[doc = "Return the number of zeros in the binary representation of each element."] @@ -602,6 +638,24 @@ pub trait Simd: b: u8x16, indices: u8x16, ) -> u8x16; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_u8x16( + self, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_u8x16( + self, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16; #[doc = "Return the number of ones in the binary representation of each element."] fn count_ones_u8x16(self, a: u8x16) -> u8x16; #[doc = "Return the number of zeros in the binary representation of each element."] @@ -674,24 +728,6 @@ pub trait Simd: fn deinterleave_u8x16(self, a: u8x16, b: u8x16) -> (u8x16, u8x16); #[doc = "Select elements from b and c based on the mask operand a.\n\nThis operation's behavior is unspecified if a was constructed from signed integer lanes that are neither all-zeroes (integer value 0) nor all-ones (integer value -1). See the [`Select`] trait's documentation for more information."] fn select_u8x16(self, a: mask8x16, b: u8x16, c: u8x16) -> u8x16; - #[doc = "Compact the bytes selected by `mask` into consecutive low lanes.\n\nLanes above the number of selected bytes are zero."] - fn compress_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16; - #[doc = "Compact the bytes selected by `mask` into consecutive low lanes.\n\nLanes above the number of selected bytes retain the corresponding values from `merge`."] - fn compress_merge_u8x16( - self, - values: u8x16, - mask: mask8x16, - merge: u8x16, - ) -> u8x16; - #[doc = "Expand consecutive low bytes from `values` into the lanes selected by `mask`.\n\nUnselected lanes are zero."] - fn expand_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16; - #[doc = "Expand consecutive low bytes from `values` into the lanes selected by `mask`.\n\nUnselected lanes retain the corresponding values from `merge`."] - fn expand_merge_u8x16( - self, - values: u8x16, - mask: mask8x16, - merge: u8x16, - ) -> u8x16; #[doc = "Combine two vectors into a single vector with twice the width.\n\n`a` provides the lower elements and `b` provides the upper elements."] fn combine_u8x16(self, a: u8x16, b: u8x16) -> u8x32; #[doc = "Load four 128-bit vectors from an array with 4-way interleaving.\n\nThis is useful e.g. in image processing to turn interleaved RGBA pixels into vectors of each color component.\n\nFor example, with 32-bit lanes, memory laid out as`[r0, g0, b0, a0, r1, g1, b1, a1, r2, g2, b2, a2, r3, g3, b3, a3]` loads as`[[r0, r1, r2, r3], [g0, g1, g2, g3], [b0, b1, b2, b3], [a0, a1, a2, a3]]`."] @@ -815,6 +851,24 @@ pub trait Simd: indices, )) } + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_i16x8(self, values: i16x8, mask: mask16x8) -> i16x8; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_i16x8( + self, + values: i16x8, + mask: mask16x8, + merge: i16x8, + ) -> i16x8; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_i16x8(self, values: i16x8, mask: mask16x8) -> i16x8; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_i16x8( + self, + values: i16x8, + mask: mask16x8, + merge: i16x8, + ) -> i16x8; #[doc = "Return the number of ones in the binary representation of each element."] fn count_ones_i16x8(self, a: i16x8) -> i16x8; #[doc = "Return the number of zeros in the binary representation of each element."] @@ -972,6 +1026,24 @@ pub trait Simd: indices, )) } + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_u16x8(self, values: u16x8, mask: mask16x8) -> u16x8; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_u16x8( + self, + values: u16x8, + mask: mask16x8, + merge: u16x8, + ) -> u16x8; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_u16x8(self, values: u16x8, mask: mask16x8) -> u16x8; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_u16x8( + self, + values: u16x8, + mask: mask16x8, + merge: u16x8, + ) -> u16x8; #[doc = "Return the number of ones in the binary representation of each element."] fn count_ones_u16x8(self, a: u16x8) -> u16x8; #[doc = "Return the number of zeros in the binary representation of each element."] @@ -1175,6 +1247,24 @@ pub trait Simd: indices, )) } + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_i32x4(self, values: i32x4, mask: mask32x4) -> i32x4; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_i32x4( + self, + values: i32x4, + mask: mask32x4, + merge: i32x4, + ) -> i32x4; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_i32x4(self, values: i32x4, mask: mask32x4) -> i32x4; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_i32x4( + self, + values: i32x4, + mask: mask32x4, + merge: i32x4, + ) -> i32x4; #[doc = "Return the number of ones in the binary representation of each element."] fn count_ones_i32x4(self, a: i32x4) -> i32x4; #[doc = "Return the number of zeros in the binary representation of each element."] @@ -1334,6 +1424,24 @@ pub trait Simd: indices, )) } + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_u32x4(self, values: u32x4, mask: mask32x4) -> u32x4; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_u32x4( + self, + values: u32x4, + mask: mask32x4, + merge: u32x4, + ) -> u32x4; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_u32x4(self, values: u32x4, mask: mask32x4) -> u32x4; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_u32x4( + self, + values: u32x4, + mask: mask32x4, + merge: u32x4, + ) -> u32x4; #[doc = "Return the number of ones in the binary representation of each element."] fn count_ones_u32x4(self, a: u32x4) -> u32x4; #[doc = "Return the number of zeros in the binary representation of each element."] @@ -1539,6 +1647,24 @@ pub trait Simd: indices, )) } + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_f64x2(self, values: f64x2, mask: mask64x2) -> f64x2; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_f64x2( + self, + values: f64x2, + mask: mask64x2, + merge: f64x2, + ) -> f64x2; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_f64x2(self, values: f64x2, mask: mask64x2) -> f64x2; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_f64x2( + self, + values: f64x2, + mask: mask64x2, + merge: f64x2, + ) -> f64x2; #[doc = "Negate each element of the vector."] fn neg_f64x2(self, a: f64x2) -> f64x2; #[doc = "Compute the square root of each element.\n\nNegative elements other than `-0.0` will become NaN."] @@ -1723,6 +1849,24 @@ pub trait Simd: indices, )) } + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_i64x2(self, values: i64x2, mask: mask64x2) -> i64x2; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_i64x2( + self, + values: i64x2, + mask: mask64x2, + merge: i64x2, + ) -> i64x2; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_i64x2(self, values: i64x2, mask: mask64x2) -> i64x2; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_i64x2( + self, + values: i64x2, + mask: mask64x2, + merge: i64x2, + ) -> i64x2; #[doc = "Return the number of ones in the binary representation of each element."] fn count_ones_i64x2(self, a: i64x2) -> i64x2; #[doc = "Return the number of zeros in the binary representation of each element."] @@ -1880,6 +2024,24 @@ pub trait Simd: indices, )) } + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_u64x2(self, values: u64x2, mask: mask64x2) -> u64x2; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_u64x2( + self, + values: u64x2, + mask: mask64x2, + merge: u64x2, + ) -> u64x2; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_u64x2(self, values: u64x2, mask: mask64x2) -> u64x2; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_u64x2( + self, + values: u64x2, + mask: mask64x2, + merge: u64x2, + ) -> u64x2; #[doc = "Return the number of ones in the binary representation of each element."] fn count_ones_u64x2(self, a: u64x2) -> u64x2; #[doc = "Return the number of zeros in the binary representation of each element."] @@ -2098,6 +2260,24 @@ pub trait Simd: indices, )) } + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_f32x8(self, values: f32x8, mask: mask32x8) -> f32x8; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_f32x8( + self, + values: f32x8, + mask: mask32x8, + merge: f32x8, + ) -> f32x8; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_f32x8(self, values: f32x8, mask: mask32x8) -> f32x8; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_f32x8( + self, + values: f32x8, + mask: mask32x8, + merge: f32x8, + ) -> f32x8; #[doc = "Negate each element of the vector."] #[inline(always)] fn neg_f32x8(self, a: f32x8) -> f32x8 { @@ -2529,6 +2709,24 @@ pub trait Simd: indices, )) } + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_i8x32(self, values: i8x32, mask: mask8x32) -> i8x32; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_i8x32( + self, + values: i8x32, + mask: mask8x32, + merge: i8x32, + ) -> i8x32; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_i8x32(self, values: i8x32, mask: mask8x32) -> i8x32; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_i8x32( + self, + values: i8x32, + mask: mask8x32, + merge: i8x32, + ) -> i8x32; #[doc = "Return the number of ones in the binary representation of each element."] #[inline(always)] fn count_ones_i8x32(self, a: i8x32) -> i8x32 { @@ -2847,6 +3045,24 @@ pub trait Simd: b: u8x32, indices: u8x32, ) -> u8x32; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_u8x32(self, values: u8x32, mask: mask8x32) -> u8x32; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_u8x32( + self, + values: u8x32, + mask: mask8x32, + merge: u8x32, + ) -> u8x32; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_u8x32(self, values: u8x32, mask: mask8x32) -> u8x32; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_u8x32( + self, + values: u8x32, + mask: mask8x32, + merge: u8x32, + ) -> u8x32; #[doc = "Return the number of ones in the binary representation of each element."] #[inline(always)] fn count_ones_u8x32(self, a: u8x32) -> u8x32 { @@ -3086,24 +3302,6 @@ pub trait Simd: let (c0, c1) = self.split_u8x32(c); self.combine_u8x16(self.select_u8x16(a0, b0, c0), self.select_u8x16(a1, b1, c1)) } - #[doc = "Compact the bytes selected by `mask` into consecutive low lanes.\n\nLanes above the number of selected bytes are zero."] - fn compress_u8x32(self, values: u8x32, mask: mask8x32) -> u8x32; - #[doc = "Compact the bytes selected by `mask` into consecutive low lanes.\n\nLanes above the number of selected bytes retain the corresponding values from `merge`."] - fn compress_merge_u8x32( - self, - values: u8x32, - mask: mask8x32, - merge: u8x32, - ) -> u8x32; - #[doc = "Expand consecutive low bytes from `values` into the lanes selected by `mask`.\n\nUnselected lanes are zero."] - fn expand_u8x32(self, values: u8x32, mask: mask8x32) -> u8x32; - #[doc = "Expand consecutive low bytes from `values` into the lanes selected by `mask`.\n\nUnselected lanes retain the corresponding values from `merge`."] - fn expand_merge_u8x32( - self, - values: u8x32, - mask: mask8x32, - merge: u8x32, - ) -> u8x32; #[doc = "Combine two vectors into a single vector with twice the width.\n\n`a` provides the lower elements and `b` provides the upper elements."] fn combine_u8x32(self, a: u8x32, b: u8x32) -> u8x64; #[doc = "Split a vector into two vectors of half the width.\n\nReturns a tuple of (lower half, upper half)."] @@ -3325,6 +3523,24 @@ pub trait Simd: indices, )) } + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_i16x16(self, values: i16x16, mask: mask16x16) -> i16x16; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_i16x16( + self, + values: i16x16, + mask: mask16x16, + merge: i16x16, + ) -> i16x16; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_i16x16(self, values: i16x16, mask: mask16x16) -> i16x16; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_i16x16( + self, + values: i16x16, + mask: mask16x16, + merge: i16x16, + ) -> i16x16; #[doc = "Return the number of ones in the binary representation of each element."] #[inline(always)] fn count_ones_i16x16(self, a: i16x16) -> i16x16 { @@ -3689,6 +3905,24 @@ pub trait Simd: indices, )) } + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_u16x16(self, values: u16x16, mask: mask16x16) -> u16x16; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_u16x16( + self, + values: u16x16, + mask: mask16x16, + merge: u16x16, + ) -> u16x16; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_u16x16(self, values: u16x16, mask: mask16x16) -> u16x16; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_u16x16( + self, + values: u16x16, + mask: mask16x16, + merge: u16x16, + ) -> u16x16; #[doc = "Return the number of ones in the binary representation of each element."] #[inline(always)] fn count_ones_u16x16(self, a: u16x16) -> u16x16 { @@ -4179,6 +4413,24 @@ pub trait Simd: indices, )) } + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_i32x8(self, values: i32x8, mask: mask32x8) -> i32x8; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_i32x8( + self, + values: i32x8, + mask: mask32x8, + merge: i32x8, + ) -> i32x8; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_i32x8(self, values: i32x8, mask: mask32x8) -> i32x8; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_i32x8( + self, + values: i32x8, + mask: mask32x8, + merge: i32x8, + ) -> i32x8; #[doc = "Return the number of ones in the binary representation of each element."] #[inline(always)] fn count_ones_i32x8(self, a: i32x8) -> i32x8 { @@ -4545,6 +4797,24 @@ pub trait Simd: indices, )) } + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_u32x8(self, values: u32x8, mask: mask32x8) -> u32x8; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_u32x8( + self, + values: u32x8, + mask: mask32x8, + merge: u32x8, + ) -> u32x8; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_u32x8(self, values: u32x8, mask: mask32x8) -> u32x8; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_u32x8( + self, + values: u32x8, + mask: mask32x8, + merge: u32x8, + ) -> u32x8; #[doc = "Return the number of ones in the binary representation of each element."] #[inline(always)] fn count_ones_u32x8(self, a: u32x8) -> u32x8 { @@ -5041,6 +5311,24 @@ pub trait Simd: indices, )) } + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_f64x4(self, values: f64x4, mask: mask64x4) -> f64x4; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_f64x4( + self, + values: f64x4, + mask: mask64x4, + merge: f64x4, + ) -> f64x4; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_f64x4(self, values: f64x4, mask: mask64x4) -> f64x4; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_f64x4( + self, + values: f64x4, + mask: mask64x4, + merge: f64x4, + ) -> f64x4; #[doc = "Negate each element of the vector."] #[inline(always)] fn neg_f64x4(self, a: f64x4) -> f64x4 { @@ -5491,6 +5779,24 @@ pub trait Simd: indices, )) } + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_i64x4(self, values: i64x4, mask: mask64x4) -> i64x4; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_i64x4( + self, + values: i64x4, + mask: mask64x4, + merge: i64x4, + ) -> i64x4; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_i64x4(self, values: i64x4, mask: mask64x4) -> i64x4; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_i64x4( + self, + values: i64x4, + mask: mask64x4, + merge: i64x4, + ) -> i64x4; #[doc = "Return the number of ones in the binary representation of each element."] #[inline(always)] fn count_ones_i64x4(self, a: i64x4) -> i64x4 { @@ -5849,6 +6155,24 @@ pub trait Simd: indices, )) } + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_u64x4(self, values: u64x4, mask: mask64x4) -> u64x4; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_u64x4( + self, + values: u64x4, + mask: mask64x4, + merge: u64x4, + ) -> u64x4; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_u64x4(self, values: u64x4, mask: mask64x4) -> u64x4; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_u64x4( + self, + values: u64x4, + mask: mask64x4, + merge: u64x4, + ) -> u64x4; #[doc = "Return the number of ones in the binary representation of each element."] #[inline(always)] fn count_ones_u64x4(self, a: u64x4) -> u64x4 { @@ -6330,6 +6654,24 @@ pub trait Simd: indices, )) } + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_f32x16(self, values: f32x16, mask: mask32x16) -> f32x16; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_f32x16( + self, + values: f32x16, + mask: mask32x16, + merge: f32x16, + ) -> f32x16; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_f32x16(self, values: f32x16, mask: mask32x16) -> f32x16; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_f32x16( + self, + values: f32x16, + mask: mask32x16, + merge: f32x16, + ) -> f32x16; #[doc = "Negate each element of the vector."] #[inline(always)] fn neg_f32x16(self, a: f32x16) -> f32x16 { @@ -6769,6 +7111,24 @@ pub trait Simd: indices, )) } + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_i8x64(self, values: i8x64, mask: mask8x64) -> i8x64; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_i8x64( + self, + values: i8x64, + mask: mask8x64, + merge: i8x64, + ) -> i8x64; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_i8x64(self, values: i8x64, mask: mask8x64) -> i8x64; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_i8x64( + self, + values: i8x64, + mask: mask8x64, + merge: i8x64, + ) -> i8x64; #[doc = "Return the number of ones in the binary representation of each element."] #[inline(always)] fn count_ones_i8x64(self, a: i8x64) -> i8x64 { @@ -7085,6 +7445,24 @@ pub trait Simd: b: u8x64, indices: u8x64, ) -> u8x64; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_u8x64(self, values: u8x64, mask: mask8x64) -> u8x64; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_u8x64( + self, + values: u8x64, + mask: mask8x64, + merge: u8x64, + ) -> u8x64; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_u8x64(self, values: u8x64, mask: mask8x64) -> u8x64; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_u8x64( + self, + values: u8x64, + mask: mask8x64, + merge: u8x64, + ) -> u8x64; #[doc = "Return the number of ones in the binary representation of each element."] #[inline(always)] fn count_ones_u8x64(self, a: u8x64) -> u8x64 { @@ -7324,24 +7702,6 @@ pub trait Simd: let (c0, c1) = self.split_u8x64(c); self.combine_u8x32(self.select_u8x32(a0, b0, c0), self.select_u8x32(a1, b1, c1)) } - #[doc = "Compact the bytes selected by `mask` into consecutive low lanes.\n\nLanes above the number of selected bytes are zero."] - fn compress_u8x64(self, values: u8x64, mask: mask8x64) -> u8x64; - #[doc = "Compact the bytes selected by `mask` into consecutive low lanes.\n\nLanes above the number of selected bytes retain the corresponding values from `merge`."] - fn compress_merge_u8x64( - self, - values: u8x64, - mask: mask8x64, - merge: u8x64, - ) -> u8x64; - #[doc = "Expand consecutive low bytes from `values` into the lanes selected by `mask`.\n\nUnselected lanes are zero."] - fn expand_u8x64(self, values: u8x64, mask: mask8x64) -> u8x64; - #[doc = "Expand consecutive low bytes from `values` into the lanes selected by `mask`.\n\nUnselected lanes retain the corresponding values from `merge`."] - fn expand_merge_u8x64( - self, - values: u8x64, - mask: mask8x64, - merge: u8x64, - ) -> u8x64; #[doc = "Split a vector into two vectors of half the width.\n\nReturns a tuple of (lower half, upper half)."] fn split_u8x64(self, a: u8x64) -> (u8x32, u8x32); #[doc = "Widen every lane into two same-width vectors.\n\nThe first result contains the widened lower lanes and the second contains the widened upper lanes."] @@ -7559,6 +7919,24 @@ pub trait Simd: indices, )) } + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_i16x32(self, values: i16x32, mask: mask16x32) -> i16x32; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_i16x32( + self, + values: i16x32, + mask: mask16x32, + merge: i16x32, + ) -> i16x32; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_i16x32(self, values: i16x32, mask: mask16x32) -> i16x32; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_i16x32( + self, + values: i16x32, + mask: mask16x32, + merge: i16x32, + ) -> i16x32; #[doc = "Return the number of ones in the binary representation of each element."] #[inline(always)] fn count_ones_i16x32(self, a: i16x32) -> i16x32 { @@ -7927,6 +8305,24 @@ pub trait Simd: indices, )) } + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_u16x32(self, values: u16x32, mask: mask16x32) -> u16x32; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_u16x32( + self, + values: u16x32, + mask: mask16x32, + merge: u16x32, + ) -> u16x32; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_u16x32(self, values: u16x32, mask: mask16x32) -> u16x32; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_u16x32( + self, + values: u16x32, + mask: mask16x32, + merge: u16x32, + ) -> u16x32; #[doc = "Return the number of ones in the binary representation of each element."] #[inline(always)] fn count_ones_u16x32(self, a: u16x32) -> u16x32 { @@ -8426,6 +8822,24 @@ pub trait Simd: indices, )) } + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_i32x16(self, values: i32x16, mask: mask32x16) -> i32x16; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_i32x16( + self, + values: i32x16, + mask: mask32x16, + merge: i32x16, + ) -> i32x16; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_i32x16(self, values: i32x16, mask: mask32x16) -> i32x16; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_i32x16( + self, + values: i32x16, + mask: mask32x16, + merge: i32x16, + ) -> i32x16; #[doc = "Return the number of ones in the binary representation of each element."] #[inline(always)] fn count_ones_i32x16(self, a: i32x16) -> i32x16 { @@ -8794,6 +9208,24 @@ pub trait Simd: indices, )) } + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_u32x16(self, values: u32x16, mask: mask32x16) -> u32x16; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_u32x16( + self, + values: u32x16, + mask: mask32x16, + merge: u32x16, + ) -> u32x16; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_u32x16(self, values: u32x16, mask: mask32x16) -> u32x16; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_u32x16( + self, + values: u32x16, + mask: mask32x16, + merge: u32x16, + ) -> u32x16; #[doc = "Return the number of ones in the binary representation of each element."] #[inline(always)] fn count_ones_u32x16(self, a: u32x16) -> u32x16 { @@ -9286,6 +9718,24 @@ pub trait Simd: indices, )) } + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_f64x8(self, values: f64x8, mask: mask64x8) -> f64x8; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_f64x8( + self, + values: f64x8, + mask: mask64x8, + merge: f64x8, + ) -> f64x8; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_f64x8(self, values: f64x8, mask: mask64x8) -> f64x8; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_f64x8( + self, + values: f64x8, + mask: mask64x8, + merge: f64x8, + ) -> f64x8; #[doc = "Negate each element of the vector."] #[inline(always)] fn neg_f64x8(self, a: f64x8) -> f64x8 { @@ -9734,6 +10184,24 @@ pub trait Simd: indices, )) } + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_i64x8(self, values: i64x8, mask: mask64x8) -> i64x8; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_i64x8( + self, + values: i64x8, + mask: mask64x8, + merge: i64x8, + ) -> i64x8; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_i64x8(self, values: i64x8, mask: mask64x8) -> i64x8; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_i64x8( + self, + values: i64x8, + mask: mask64x8, + merge: i64x8, + ) -> i64x8; #[doc = "Return the number of ones in the binary representation of each element."] #[inline(always)] fn count_ones_i64x8(self, a: i64x8) -> i64x8 { @@ -10090,6 +10558,24 @@ pub trait Simd: indices, )) } + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_u64x8(self, values: u64x8, mask: mask64x8) -> u64x8; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_u64x8( + self, + values: u64x8, + mask: mask64x8, + merge: u64x8, + ) -> u64x8; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_u64x8(self, values: u64x8, mask: mask64x8) -> u64x8; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_u64x8( + self, + values: u64x8, + mask: mask64x8, + merge: u64x8, + ) -> u64x8; #[doc = "Return the number of ones in the binary representation of each element."] #[inline(always)] fn count_ones_u64x8(self, a: u64x8) -> u64x8 { @@ -11065,6 +11551,14 @@ pub trait SimdBase: rhs: impl SimdInto, indices: impl SimdInto, ) -> Self; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress(self, mask: Self::Mask) -> Self; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self; + #[doc = "Expand consecutive low elements from `self` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand(self, mask: Self::Mask) -> Self; + #[doc = "Expand consecutive low elements from `self` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self; #[doc = "Return the maximum element in the vector. Integer vectors always return the exact maximum.\n\nFor floating-point vectors with no NaNs, this returns the true maximum. If any lane is NaN, the entire result is implementation-defined: it may be NaN or a numeric lane that is not the true maximum. See `reduce_max_precise` for a version that ignores quiet NaNs.\n\nIf the floating-point vector contains both positive zero and negative zero, either sign of zero may be returned."] fn reduce_max(self) -> Self::Element; #[doc = "Return the minimum element in the vector. Integer vectors always return the exact minimum.\n\nFor floating-point vectors with no NaNs, this returns the true minimum. If any lane is NaN, the entire result is implementation-defined: it may be NaN or a numeric lane that is not the true minimum. See `reduce_min_precise` for a version that ignores quiet NaNs.\n\nIf the floating-point vector contains both positive zero and negative zero, either sign of zero may be returned."] diff --git a/fearless_simd/src/generated/simd_types.rs b/fearless_simd/src/generated/simd_types.rs index c81456416..a5d20c2f0 100644 --- a/fearless_simd/src/generated/simd_types.rs +++ b/fearless_simd/src/generated/simd_types.rs @@ -191,6 +191,24 @@ impl SimdBase for f32x4 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_f32x4(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_f32x4(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_f32x4(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_f32x4(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_f32x4(self) } @@ -600,6 +618,24 @@ impl SimdBase for i8x16 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_i8x16(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_i8x16(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_i8x16(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_i8x16(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_i8x16(self) } @@ -933,6 +969,24 @@ impl SimdBase for u8x16 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_u8x16(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_u8x16(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_u8x16(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_u8x16(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_u8x16(self) } @@ -1374,6 +1428,24 @@ impl SimdBase for i16x8 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_i16x8(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_i16x8(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_i16x8(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_i16x8(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_i16x8(self) } @@ -1714,6 +1786,24 @@ impl SimdBase for u16x8 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_u16x8(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_u16x8(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_u16x8(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_u16x8(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_u16x8(self) } @@ -2165,6 +2255,24 @@ impl SimdBase for i32x4 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_i32x4(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_i32x4(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_i32x4(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_i32x4(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_i32x4(self) } @@ -2505,6 +2613,24 @@ impl SimdBase for u32x4 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_u32x4(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_u32x4(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_u32x4(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_u32x4(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_u32x4(self) } @@ -2968,6 +3094,24 @@ impl SimdBase for f64x2 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_f64x2(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_f64x2(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_f64x2(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_f64x2(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_f64x2(self) } @@ -3365,6 +3509,24 @@ impl SimdBase for i64x2 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_i64x2(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_i64x2(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_i64x2(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_i64x2(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_i64x2(self) } @@ -3698,6 +3860,24 @@ impl SimdBase for u64x2 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_u64x2(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_u64x2(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_u64x2(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_u64x2(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_u64x2(self) } @@ -4159,6 +4339,24 @@ impl SimdBase for f32x8 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_f32x8(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_f32x8(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_f32x8(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_f32x8(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_f32x8(self) } @@ -4579,6 +4777,24 @@ impl SimdBase for i8x32 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_i8x32(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_i8x32(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_i8x32(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_i8x32(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_i8x32(self) } @@ -4923,6 +5139,24 @@ impl SimdBase for u8x32 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_u8x32(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_u8x32(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_u8x32(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_u8x32(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_u8x32(self) } @@ -5367,6 +5601,24 @@ impl SimdBase for i16x16 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_i16x16(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_i16x16(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_i16x16(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_i16x16(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_i16x16(self) } @@ -5711,6 +5963,24 @@ impl SimdBase for u16x16 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_u16x16(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_u16x16(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_u16x16(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_u16x16(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_u16x16(self) } @@ -6170,6 +6440,24 @@ impl SimdBase for i32x8 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_i32x8(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_i32x8(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_i32x8(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_i32x8(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_i32x8(self) } @@ -6517,6 +6805,24 @@ impl SimdBase for u32x8 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_u32x8(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_u32x8(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_u32x8(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_u32x8(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_u32x8(self) } @@ -6975,6 +7281,24 @@ impl SimdBase for f64x4 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_f64x4(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_f64x4(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_f64x4(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_f64x4(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_f64x4(self) } @@ -7367,6 +7691,24 @@ impl SimdBase for i64x4 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_i64x4(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_i64x4(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_i64x4(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_i64x4(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_i64x4(self) } @@ -7695,6 +8037,24 @@ impl SimdBase for u64x4 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_u64x4(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_u64x4(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_u64x4(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_u64x4(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_u64x4(self) } @@ -8160,6 +8520,24 @@ impl SimdBase for f32x16 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_f32x16(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_f32x16(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_f32x16(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_f32x16(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_f32x16(self) } @@ -8607,6 +8985,24 @@ impl SimdBase for i8x64 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_i8x64(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_i8x64(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_i8x64(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_i8x64(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_i8x64(self) } @@ -8977,6 +9373,24 @@ impl SimdBase for u8x64 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_u8x64(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_u8x64(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_u8x64(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_u8x64(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_u8x64(self) } @@ -9431,6 +9845,24 @@ impl SimdBase for i16x32 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_i16x32(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_i16x32(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_i16x32(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_i16x32(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_i16x32(self) } @@ -9785,6 +10217,24 @@ impl SimdBase for u16x32 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_u16x32(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_u16x32(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_u16x32(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_u16x32(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_u16x32(self) } @@ -10246,6 +10696,24 @@ impl SimdBase for i32x16 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_i32x16(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_i32x16(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_i32x16(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_i32x16(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_i32x16(self) } @@ -10596,6 +11064,24 @@ impl SimdBase for u32x16 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_u32x16(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_u32x16(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_u32x16(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_u32x16(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_u32x16(self) } @@ -11061,6 +11547,24 @@ impl SimdBase for f64x8 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_f64x8(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_f64x8(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_f64x8(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_f64x8(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_f64x8(self) } @@ -11459,6 +11963,24 @@ impl SimdBase for i64x8 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_i64x8(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_i64x8(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_i64x8(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_i64x8(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_i64x8(self) } @@ -11793,6 +12315,24 @@ impl SimdBase for u64x8 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_u64x8(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_u64x8(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_u64x8(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_u64x8(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_u64x8(self) } diff --git a/fearless_simd/src/generated/sse2.rs b/fearless_simd/src/generated/sse2.rs index d747af81c..043b514c3 100644 --- a/fearless_simd/src/generated/sse2.rs +++ b/fearless_simd/src/generated/sse2.rs @@ -168,6 +168,56 @@ impl Simd for Sse2 { }) } #[inline(always)] + fn compress_f32x4(self, values: f32x4, mask: mask32x4) -> f32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_f32x4( + self, + values: f32x4, + mask: mask32x4, + merge: f32x4, + ) -> f32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_f32x4(self, values: f32x4, mask: mask32x4) -> f32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_f32x4( + self, + values: f32x4, + mask: mask32x4, + merge: f32x4, + ) -> f32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn neg_f32x4(self, a: f32x4) -> f32x4 { crate::kernel!( #[inline(always)] @@ -834,6 +884,56 @@ impl Simd for Sse2 { }) } #[inline(always)] + fn compress_i8x16(self, values: i8x16, mask: mask8x16) -> i8x16 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i8x16( + self, + values: i8x16, + mask: mask8x16, + merge: i8x16, + ) -> i8x16 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i8x16(self, values: i8x16, mask: mask8x16) -> i8x16 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i8x16( + self, + values: i8x16, + mask: mask8x16, + merge: i8x16, + ) -> i8x16 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_i8x16(self, a: i8x16) -> i8x16 { [ i8::try_from(a[0usize].count_ones()).unwrap(), @@ -1675,6 +1775,53 @@ impl Simd for Sse2 { Bytes::from_bytes(result) } #[inline(always)] + fn compress_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16 { + self.compress_merge_u8x16(values, mask, u8x16::splat(self, 0)) + } + #[inline(always)] + fn compress_merge_u8x16( + self, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + let mask = self.to_bitmask_mask8x16(mask); + let mut merge_padded = [0u8; 16 + 1]; + merge_padded[..16].copy_from_slice(&*merge); + let mut compacted = merge_padded; + let mut output_lane = 0; + for input_lane in 0..16 { + compacted[output_lane] = values[input_lane]; + output_lane += ((mask >> input_lane) & 1) as usize; + } + compacted[output_lane] = merge_padded[output_lane]; + let mut result = merge; + result.copy_from_slice(&compacted[..16]); + result + } + #[inline(always)] + fn expand_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16 { + self.expand_merge_u8x16(values, mask, u8x16::splat(self, 0)) + } + #[inline(always)] + fn expand_merge_u8x16( + self, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + let mask = self.to_bitmask_mask8x16(mask); + let mut result = merge; + let mut input_lane = 0; + for output_lane in 0..16 { + let bit = ((mask >> output_lane) & 1) as u8; + let keep = 0u8.wrapping_sub(bit); + result[output_lane] = (values[input_lane] & keep) | (result[output_lane] & !keep); + input_lane += usize::from(bit); + } + result + } + #[inline(always)] fn count_ones_u8x16(self, a: u8x16) -> u8x16 { [ u8::try_from(a[0usize].count_ones()).unwrap(), @@ -2096,53 +2243,6 @@ impl Simd for Sse2 { kernel(self, a, b, c) } #[inline(always)] - fn compress_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16 { - self.compress_merge_u8x16(values, mask, u8x16::splat(self, 0)) - } - #[inline(always)] - fn compress_merge_u8x16( - self, - values: u8x16, - mask: mask8x16, - merge: u8x16, - ) -> u8x16 { - let mask = self.to_bitmask_mask8x16(mask); - let mut merge_padded = [0u8; 16 + 1]; - merge_padded[..16].copy_from_slice(&*merge); - let mut compacted = merge_padded; - let mut output_lane = 0; - for input_lane in 0..16 { - compacted[output_lane] = values[input_lane]; - output_lane += ((mask >> input_lane) & 1) as usize; - } - compacted[output_lane] = merge_padded[output_lane]; - let mut result = merge; - result.copy_from_slice(&compacted[..16]); - result - } - #[inline(always)] - fn expand_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16 { - self.expand_merge_u8x16(values, mask, u8x16::splat(self, 0)) - } - #[inline(always)] - fn expand_merge_u8x16( - self, - values: u8x16, - mask: mask8x16, - merge: u8x16, - ) -> u8x16 { - let mask = self.to_bitmask_mask8x16(mask); - let mut result = merge; - let mut input_lane = 0; - for output_lane in 0..16 { - let bit = ((mask >> output_lane) & 1) as u8; - let keep = 0u8.wrapping_sub(bit); - result[output_lane] = (values[input_lane] & keep) | (result[output_lane] & !keep); - input_lane += usize::from(bit); - } - result - } - #[inline(always)] fn combine_u8x16(self, a: u8x16, b: u8x16) -> u8x32 { u8x32 { val: crate::support::Aligned256([a.val.0, b.val.0]), @@ -2591,6 +2691,56 @@ impl Simd for Sse2 { }) } #[inline(always)] + fn compress_i16x8(self, values: i16x8, mask: mask16x8) -> i16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i16x8( + self, + values: i16x8, + mask: mask16x8, + merge: i16x8, + ) -> i16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i16x8(self, values: i16x8, mask: mask16x8) -> i16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i16x8( + self, + values: i16x8, + mask: mask16x8, + merge: i16x8, + ) -> i16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_i16x8(self, a: i16x8) -> i16x8 { [ i16::try_from(a[0usize].count_ones()).unwrap(), @@ -3124,6 +3274,56 @@ impl Simd for Sse2 { }) } #[inline(always)] + fn compress_u16x8(self, values: u16x8, mask: mask16x8) -> u16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u16x8( + self, + values: u16x8, + mask: mask16x8, + merge: u16x8, + ) -> u16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u16x8(self, values: u16x8, mask: mask16x8) -> u16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u16x8( + self, + values: u16x8, + mask: mask16x8, + merge: u16x8, + ) -> u16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_u16x8(self, a: u16x8) -> u16x8 { [ u16::try_from(a[0usize].count_ones()).unwrap(), @@ -3998,6 +4198,56 @@ impl Simd for Sse2 { }) } #[inline(always)] + fn compress_i32x4(self, values: i32x4, mask: mask32x4) -> i32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i32x4( + self, + values: i32x4, + mask: mask32x4, + merge: i32x4, + ) -> i32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i32x4(self, values: i32x4, mask: mask32x4) -> i32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i32x4( + self, + values: i32x4, + mask: mask32x4, + merge: i32x4, + ) -> i32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_i32x4(self, a: i32x4) -> i32x4 { [ a[0usize].count_ones().cast_signed(), @@ -4556,6 +4806,56 @@ impl Simd for Sse2 { }) } #[inline(always)] + fn compress_u32x4(self, values: u32x4, mask: mask32x4) -> u32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u32x4( + self, + values: u32x4, + mask: mask32x4, + merge: u32x4, + ) -> u32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u32x4(self, values: u32x4, mask: mask32x4) -> u32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u32x4( + self, + values: u32x4, + mask: mask32x4, + merge: u32x4, + ) -> u32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_u32x4(self, a: u32x4) -> u32x4 { [ a[0usize].count_ones(), @@ -5390,6 +5690,56 @@ impl Simd for Sse2 { }) } #[inline(always)] + fn compress_f64x2(self, values: f64x2, mask: mask64x2) -> f64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_f64x2( + self, + values: f64x2, + mask: mask64x2, + merge: f64x2, + ) -> f64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_f64x2(self, values: f64x2, mask: mask64x2) -> f64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_f64x2( + self, + values: f64x2, + mask: mask64x2, + merge: f64x2, + ) -> f64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn neg_f64x2(self, a: f64x2) -> f64x2 { crate::kernel!( #[inline(always)] @@ -5941,6 +6291,56 @@ impl Simd for Sse2 { }) } #[inline(always)] + fn compress_i64x2(self, values: i64x2, mask: mask64x2) -> i64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i64x2( + self, + values: i64x2, + mask: mask64x2, + merge: i64x2, + ) -> i64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i64x2(self, values: i64x2, mask: mask64x2) -> i64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i64x2( + self, + values: i64x2, + mask: mask64x2, + merge: i64x2, + ) -> i64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_i64x2(self, a: i64x2) -> i64x2 { [ i64::from(a[0usize].count_ones()), @@ -6380,20 +6780,70 @@ impl Simd for Sse2 { }) } #[inline(always)] - fn count_ones_u64x2(self, a: u64x2) -> u64x2 { - [ - u64::from(a[0usize].count_ones()), - u64::from(a[1usize].count_ones()), - ] - .simd_into(self) - } - #[inline(always)] - fn count_zeros_u64x2(self, a: u64x2) -> u64x2 { - [ - u64::from(a[0usize].count_zeros()), - u64::from(a[1usize].count_zeros()), - ] - .simd_into(self) + fn compress_u64x2(self, values: u64x2, mask: mask64x2) -> u64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u64x2( + self, + values: u64x2, + mask: mask64x2, + merge: u64x2, + ) -> u64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u64x2(self, values: u64x2, mask: mask64x2) -> u64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u64x2( + self, + values: u64x2, + mask: mask64x2, + merge: u64x2, + ) -> u64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn count_ones_u64x2(self, a: u64x2) -> u64x2 { + [ + u64::from(a[0usize].count_ones()), + u64::from(a[1usize].count_ones()), + ] + .simd_into(self) + } + #[inline(always)] + fn count_zeros_u64x2(self, a: u64x2) -> u64x2 { + [ + u64::from(a[0usize].count_zeros()), + u64::from(a[1usize].count_zeros()), + ] + .simd_into(self) } #[inline(always)] fn add_u64x2(self, a: u64x2, b: u64x2) -> u64x2 { @@ -7013,6 +7463,56 @@ impl Simd for Sse2 { }) } #[inline(always)] + fn compress_f32x8(self, values: f32x8, mask: mask32x8) -> f32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_f32x8( + self, + values: f32x8, + mask: mask32x8, + merge: f32x8, + ) -> f32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_f32x8(self, values: f32x8, mask: mask32x8) -> f32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_f32x8( + self, + values: f32x8, + mask: mask32x8, + merge: f32x8, + ) -> f32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn is_nan_f32x8(self, a: f32x8) -> mask32x8 { !a.simd_eq(a) } @@ -7081,6 +7581,56 @@ impl Simd for Sse2 { }) } #[inline(always)] + fn compress_i8x32(self, values: i8x32, mask: mask8x32) -> i8x32 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i8x32( + self, + values: i8x32, + mask: mask8x32, + merge: i8x32, + ) -> i8x32 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i8x32(self, values: i8x32, mask: mask8x32) -> i8x32 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i8x32( + self, + values: i8x32, + mask: mask8x32, + merge: i8x32, + ) -> i8x32 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn combine_i8x32(self, a: i8x32, b: i8x32) -> i8x64 { i8x64 { val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), @@ -7328,6 +7878,56 @@ impl Simd for Sse2 { }) } #[inline(always)] + fn compress_i16x16(self, values: i16x16, mask: mask16x16) -> i16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i16x16( + self, + values: i16x16, + mask: mask16x16, + merge: i16x16, + ) -> i16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i16x16(self, values: i16x16, mask: mask16x16) -> i16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i16x16( + self, + values: i16x16, + mask: mask16x16, + merge: i16x16, + ) -> i16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn combine_i16x16(self, a: i16x16, b: i16x16) -> i16x32 { i16x32 { val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), @@ -7364,6 +7964,56 @@ impl Simd for Sse2 { }) } #[inline(always)] + fn compress_u16x16(self, values: u16x16, mask: mask16x16) -> u16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u16x16( + self, + values: u16x16, + mask: mask16x16, + merge: u16x16, + ) -> u16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u16x16(self, values: u16x16, mask: mask16x16) -> u16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u16x16( + self, + values: u16x16, + mask: mask16x16, + merge: u16x16, + ) -> u16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn combine_u16x16(self, a: u16x16, b: u16x16) -> u16x32 { u16x32 { val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), @@ -7474,6 +8124,56 @@ impl Simd for Sse2 { }) } #[inline(always)] + fn compress_i32x8(self, values: i32x8, mask: mask32x8) -> i32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i32x8( + self, + values: i32x8, + mask: mask32x8, + merge: i32x8, + ) -> i32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i32x8(self, values: i32x8, mask: mask32x8) -> i32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i32x8( + self, + values: i32x8, + mask: mask32x8, + merge: i32x8, + ) -> i32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn combine_i32x8(self, a: i32x8, b: i32x8) -> i32x16 { i32x16 { val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), @@ -7510,6 +8210,56 @@ impl Simd for Sse2 { }) } #[inline(always)] + fn compress_u32x8(self, values: u32x8, mask: mask32x8) -> u32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u32x8( + self, + values: u32x8, + mask: mask32x8, + merge: u32x8, + ) -> u32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u32x8(self, values: u32x8, mask: mask32x8) -> u32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u32x8( + self, + values: u32x8, + mask: mask32x8, + merge: u32x8, + ) -> u32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn combine_u32x8(self, a: u32x8, b: u32x8) -> u32x16 { u32x16 { val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), @@ -7607,6 +8357,56 @@ impl Simd for Sse2 { }) } #[inline(always)] + fn compress_f64x4(self, values: f64x4, mask: mask64x4) -> f64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_f64x4( + self, + values: f64x4, + mask: mask64x4, + merge: f64x4, + ) -> f64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_f64x4(self, values: f64x4, mask: mask64x4) -> f64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_f64x4( + self, + values: f64x4, + mask: mask64x4, + merge: f64x4, + ) -> f64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn is_nan_f64x4(self, a: f64x4) -> mask64x4 { !a.simd_eq(a) } @@ -7675,6 +8475,56 @@ impl Simd for Sse2 { }) } #[inline(always)] + fn compress_i64x4(self, values: i64x4, mask: mask64x4) -> i64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i64x4( + self, + values: i64x4, + mask: mask64x4, + merge: i64x4, + ) -> i64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i64x4(self, values: i64x4, mask: mask64x4) -> i64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i64x4( + self, + values: i64x4, + mask: mask64x4, + merge: i64x4, + ) -> i64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn combine_i64x4(self, a: i64x4, b: i64x4) -> i64x8 { i64x8 { val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), @@ -7711,6 +8561,56 @@ impl Simd for Sse2 { }) } #[inline(always)] + fn compress_u64x4(self, values: u64x4, mask: mask64x4) -> u64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u64x4( + self, + values: u64x4, + mask: mask64x4, + merge: u64x4, + ) -> u64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u64x4(self, values: u64x4, mask: mask64x4) -> u64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u64x4( + self, + values: u64x4, + mask: mask64x4, + merge: u64x4, + ) -> u64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn combine_u64x4(self, a: u64x4, b: u64x4) -> u64x8 { u64x8 { val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), @@ -7808,6 +8708,56 @@ impl Simd for Sse2 { }) } #[inline(always)] + fn compress_f32x16(self, values: f32x16, mask: mask32x16) -> f32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_f32x16( + self, + values: f32x16, + mask: mask32x16, + merge: f32x16, + ) -> f32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_f32x16(self, values: f32x16, mask: mask32x16) -> f32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_f32x16( + self, + values: f32x16, + mask: mask32x16, + merge: f32x16, + ) -> f32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn is_nan_f32x16(self, a: f32x16) -> mask32x16 { !a.simd_eq(a) } @@ -7869,6 +8819,56 @@ impl Simd for Sse2 { }) } #[inline(always)] + fn compress_i8x64(self, values: i8x64, mask: mask8x64) -> i8x64 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i8x64( + self, + values: i8x64, + mask: mask8x64, + merge: i8x64, + ) -> i8x64 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i8x64(self, values: i8x64, mask: mask8x64) -> i8x64 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i8x64( + self, + values: i8x64, + mask: mask8x64, + merge: i8x64, + ) -> i8x64 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_i8x64(self, a: i8x64) -> (i8x32, i8x32) { ( i8x32 { @@ -8095,6 +9095,56 @@ impl Simd for Sse2 { }) } #[inline(always)] + fn compress_i16x32(self, values: i16x32, mask: mask16x32) -> i16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i16x32( + self, + values: i16x32, + mask: mask16x32, + merge: i16x32, + ) -> i16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i16x32(self, values: i16x32, mask: mask16x32) -> i16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i16x32( + self, + values: i16x32, + mask: mask16x32, + merge: i16x32, + ) -> i16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_i16x32(self, a: i16x32) -> (i16x16, i16x16) { ( i16x16 { @@ -8124,6 +9174,56 @@ impl Simd for Sse2 { }) } #[inline(always)] + fn compress_u16x32(self, values: u16x32, mask: mask16x32) -> u16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u16x32( + self, + values: u16x32, + mask: mask16x32, + merge: u16x32, + ) -> u16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u16x32(self, values: u16x32, mask: mask16x32) -> u16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u16x32( + self, + values: u16x32, + mask: mask16x32, + merge: u16x32, + ) -> u16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_u16x32(self, a: u16x32) -> (u16x16, u16x16) { ( u16x16 { @@ -8223,6 +9323,56 @@ impl Simd for Sse2 { }) } #[inline(always)] + fn compress_i32x16(self, values: i32x16, mask: mask32x16) -> i32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i32x16( + self, + values: i32x16, + mask: mask32x16, + merge: i32x16, + ) -> i32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i32x16(self, values: i32x16, mask: mask32x16) -> i32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i32x16( + self, + values: i32x16, + mask: mask32x16, + merge: i32x16, + ) -> i32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_i32x16(self, a: i32x16) -> (i32x8, i32x8) { ( i32x8 { @@ -8252,6 +9402,56 @@ impl Simd for Sse2 { }) } #[inline(always)] + fn compress_u32x16(self, values: u32x16, mask: mask32x16) -> u32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u32x16( + self, + values: u32x16, + mask: mask32x16, + merge: u32x16, + ) -> u32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u32x16(self, values: u32x16, mask: mask32x16) -> u32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u32x16( + self, + values: u32x16, + mask: mask32x16, + merge: u32x16, + ) -> u32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_u32x16(self, a: u32x16) -> (u32x8, u32x8) { ( u32x8 { @@ -8335,6 +9535,56 @@ impl Simd for Sse2 { }) } #[inline(always)] + fn compress_f64x8(self, values: f64x8, mask: mask64x8) -> f64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_f64x8( + self, + values: f64x8, + mask: mask64x8, + merge: f64x8, + ) -> f64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_f64x8(self, values: f64x8, mask: mask64x8) -> f64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_f64x8( + self, + values: f64x8, + mask: mask64x8, + merge: f64x8, + ) -> f64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn is_nan_f64x8(self, a: f64x8) -> mask64x8 { !a.simd_eq(a) } @@ -8396,6 +9646,56 @@ impl Simd for Sse2 { }) } #[inline(always)] + fn compress_i64x8(self, values: i64x8, mask: mask64x8) -> i64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i64x8( + self, + values: i64x8, + mask: mask64x8, + merge: i64x8, + ) -> i64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i64x8(self, values: i64x8, mask: mask64x8) -> i64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i64x8( + self, + values: i64x8, + mask: mask64x8, + merge: i64x8, + ) -> i64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_i64x8(self, a: i64x8) -> (i64x4, i64x4) { ( i64x4 { @@ -8425,6 +9725,56 @@ impl Simd for Sse2 { }) } #[inline(always)] + fn compress_u64x8(self, values: u64x8, mask: mask64x8) -> u64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u64x8( + self, + values: u64x8, + mask: mask64x8, + merge: u64x8, + ) -> u64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u64x8(self, values: u64x8, mask: mask64x8) -> u64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u64x8( + self, + values: u64x8, + mask: mask64x8, + merge: u64x8, + ) -> u64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_u64x8(self, a: u64x8) -> (u64x4, u64x4) { ( u64x4 { diff --git a/fearless_simd/src/generated/sse4_2.rs b/fearless_simd/src/generated/sse4_2.rs index 8bca0c15f..8e10f46b6 100644 --- a/fearless_simd/src/generated/sse4_2.rs +++ b/fearless_simd/src/generated/sse4_2.rs @@ -170,6 +170,56 @@ impl Simd for Sse4_2 { }) } #[inline(always)] + fn compress_f32x4(self, values: f32x4, mask: mask32x4) -> f32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_f32x4( + self, + values: f32x4, + mask: mask32x4, + merge: f32x4, + ) -> f32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_f32x4(self, values: f32x4, mask: mask32x4) -> f32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_f32x4( + self, + values: f32x4, + mask: mask32x4, + merge: f32x4, + ) -> f32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn neg_f32x4(self, a: f32x4) -> f32x4 { crate::kernel!( #[inline(always)] @@ -797,6 +847,56 @@ impl Simd for Sse4_2 { }) } #[inline(always)] + fn compress_i8x16(self, values: i8x16, mask: mask8x16) -> i8x16 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i8x16( + self, + values: i8x16, + mask: mask8x16, + merge: i8x16, + ) -> i8x16 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i8x16(self, values: i8x16, mask: mask8x16) -> i8x16 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i8x16( + self, + values: i8x16, + mask: mask8x16, + merge: i8x16, + ) -> i8x16 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_i8x16(self, a: i8x16) -> i8x16 { crate::kernel!( #[inline(always)] @@ -1433,6 +1533,129 @@ impl Simd for Sse4_2 { Bytes::from_bytes(result_bytes) } #[inline(always)] + fn compress_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Sse4_2, + values: u8x16, + mask: mask8x16, + ) -> u8x16 { + let mask_bits = token.to_bitmask_mask8x16(mask) as u16; + let low_mask = (usize::from(mask_bits)) & 0xff; + let high_mask = (usize::from(mask_bits)) >> 8; + let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask]; + let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask]; + let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + 0x0808_0808_0808_0808) + | (high_control & 0x8080_8080_8080_8080); + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let low_count = low_mask.count_ones() as usize; + let compacted = _mm_shuffle_epi8(values.into(), control); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count]; + let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let compressed = _mm_shuffle_epi8(compacted, splice); + compressed.simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_u8x16( + self, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Sse4_2, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + let mask_bits = token.to_bitmask_mask8x16(mask) as u16; + let low_mask = (usize::from(mask_bits)) & 0xff; + let high_mask = (usize::from(mask_bits)) >> 8; + let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask]; + let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask]; + let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + 0x0808_0808_0808_0808) + | (high_control & 0x8080_8080_8080_8080); + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let low_count = low_mask.count_ones() as usize; + let compacted = _mm_shuffle_epi8(values.into(), control); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count]; + let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let compressed = _mm_shuffle_epi8(compacted, splice); + let count = mask_bits.count_ones() as usize; + let prefix = unsafe { + _mm_load_si128( + core::ptr::from_ref(&crate::support::COMPACT_PREFIX_MASKS[count]) + .cast::<__m128i>(), + ) + }; + _mm_blendv_epi8(merge.into(), compressed, prefix).simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Sse4_2, + values: u8x16, + mask: mask8x16, + ) -> u8x16 { + let mask_bits = token.to_bitmask_mask8x16(mask) as u16; + let low_mask = (usize::from(mask_bits)) & 0xff; + let high_mask = (usize::from(mask_bits)) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) + | (high_control & 0x8080_8080_8080_8080); + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let expanded = _mm_shuffle_epi8(values.into(), control); + expanded.simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_u8x16( + self, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Sse4_2, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + let mask_bits = token.to_bitmask_mask8x16(mask) as u16; + let low_mask = (usize::from(mask_bits)) & 0xff; + let high_mask = (usize::from(mask_bits)) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) + | (high_control & 0x8080_8080_8080_8080); + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let expanded = _mm_shuffle_epi8(values.into(), control); + _mm_blendv_epi8(merge.into(), expanded, mask.into()).simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn count_ones_u8x16(self, a: u8x16) -> u8x16 { crate::kernel!( #[inline(always)] @@ -1843,180 +2066,57 @@ impl Simd for Sse4_2 { kernel(self, a, b, c) } #[inline(always)] - fn compress_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16 { + fn combine_u8x16(self, a: u8x16, b: u8x16) -> u8x32 { + u8x32 { + val: crate::support::Aligned256([a.val.0, b.val.0]), + simd: self, + } + } + #[inline(always)] + fn load_four_interleaved_u8x16(self, src: &[u8; 64usize]) -> [u8x16; 4usize] { crate::kernel!( #[inline(always)] - fn kernel( - token: Sse4_2, - values: u8x16, - mask: mask8x16, - ) -> u8x16 { - let mask_bits = token.to_bitmask_mask8x16(mask) as u16; - let low_mask = (usize::from(mask_bits)) & 0xff; - let high_mask = (usize::from(mask_bits)) >> 8; - let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask]; - let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask]; - let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + 0x0808_0808_0808_0808) - | (high_control & 0x8080_8080_8080_8080); - let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); - let low_count = low_mask.count_ones() as usize; - let compacted = _mm_shuffle_epi8(values.into(), control); - let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count]; - let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64); - let compressed = _mm_shuffle_epi8(compacted, splice); - compressed.simd_into(token) + fn kernel(token: Sse4_2, src: &[u8; 64usize]) -> [u8x16; 4usize] { + let (chunks, []) = src.as_chunks::<16usize>() else { + unreachable!() + }; + let v0: __m128i = + crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[0]); + let v1: __m128i = + crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[1]); + let v2: __m128i = + crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[2]); + let v3: __m128i = + crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[3]); + let mask = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); + let v0 = _mm_shuffle_epi8(v0, mask); + let v1 = _mm_shuffle_epi8(v1, mask); + let v2 = _mm_shuffle_epi8(v2, mask); + let v3 = _mm_shuffle_epi8(v3, mask); + let tmp0 = _mm_unpacklo_epi32(v0, v1); + let tmp1 = _mm_unpackhi_epi32(v0, v1); + let tmp2 = _mm_unpacklo_epi32(v2, v3); + let tmp3 = _mm_unpackhi_epi32(v2, v3); + let out0 = _mm_unpacklo_epi64(tmp0, tmp2); + let out1 = _mm_unpackhi_epi64(tmp0, tmp2); + let out2 = _mm_unpacklo_epi64(tmp1, tmp3); + let out3 = _mm_unpackhi_epi64(tmp1, tmp3); + [ + out0.simd_into(token), + out1.simd_into(token), + out2.simd_into(token), + out3.simd_into(token), + ] } ); - kernel(self, values, mask) + kernel(self, src) } #[inline(always)] - fn compress_merge_u8x16( + fn store_four_interleaved_u8x16( self, - values: u8x16, - mask: mask8x16, - merge: u8x16, - ) -> u8x16 { - crate::kernel!( - #[inline(always)] - fn kernel( - token: Sse4_2, - values: u8x16, - mask: mask8x16, - merge: u8x16, - ) -> u8x16 { - let mask_bits = token.to_bitmask_mask8x16(mask) as u16; - let low_mask = (usize::from(mask_bits)) & 0xff; - let high_mask = (usize::from(mask_bits)) >> 8; - let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask]; - let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask]; - let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + 0x0808_0808_0808_0808) - | (high_control & 0x8080_8080_8080_8080); - let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); - let low_count = low_mask.count_ones() as usize; - let compacted = _mm_shuffle_epi8(values.into(), control); - let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count]; - let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64); - let compressed = _mm_shuffle_epi8(compacted, splice); - let count = mask_bits.count_ones() as usize; - let prefix = unsafe { - _mm_load_si128( - core::ptr::from_ref(&crate::support::COMPACT_PREFIX_MASKS[count]) - .cast::<__m128i>(), - ) - }; - _mm_blendv_epi8(merge.into(), compressed, prefix).simd_into(token) - } - ); - kernel(self, values, mask, merge) - } - #[inline(always)] - fn expand_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16 { - crate::kernel!( - #[inline(always)] - fn kernel( - token: Sse4_2, - values: u8x16, - mask: mask8x16, - ) -> u8x16 { - let mask_bits = token.to_bitmask_mask8x16(mask) as u16; - let low_mask = (usize::from(mask_bits)) & 0xff; - let high_mask = (usize::from(mask_bits)) >> 8; - let low_count = low_mask.count_ones() as u64; - let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; - let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; - let high_base = low_count * 0x0101_0101_0101_0101; - let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) - | (high_control & 0x8080_8080_8080_8080); - let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); - let expanded = _mm_shuffle_epi8(values.into(), control); - expanded.simd_into(token) - } - ); - kernel(self, values, mask) - } - #[inline(always)] - fn expand_merge_u8x16( - self, - values: u8x16, - mask: mask8x16, - merge: u8x16, - ) -> u8x16 { - crate::kernel!( - #[inline(always)] - fn kernel( - token: Sse4_2, - values: u8x16, - mask: mask8x16, - merge: u8x16, - ) -> u8x16 { - let mask_bits = token.to_bitmask_mask8x16(mask) as u16; - let low_mask = (usize::from(mask_bits)) & 0xff; - let high_mask = (usize::from(mask_bits)) >> 8; - let low_count = low_mask.count_ones() as u64; - let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; - let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; - let high_base = low_count * 0x0101_0101_0101_0101; - let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) - | (high_control & 0x8080_8080_8080_8080); - let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); - let expanded = _mm_shuffle_epi8(values.into(), control); - _mm_blendv_epi8(merge.into(), expanded, mask.into()).simd_into(token) - } - ); - kernel(self, values, mask, merge) - } - #[inline(always)] - fn combine_u8x16(self, a: u8x16, b: u8x16) -> u8x32 { - u8x32 { - val: crate::support::Aligned256([a.val.0, b.val.0]), - simd: self, - } - } - #[inline(always)] - fn load_four_interleaved_u8x16(self, src: &[u8; 64usize]) -> [u8x16; 4usize] { - crate::kernel!( - #[inline(always)] - fn kernel(token: Sse4_2, src: &[u8; 64usize]) -> [u8x16; 4usize] { - let (chunks, []) = src.as_chunks::<16usize>() else { - unreachable!() - }; - let v0: __m128i = - crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[0]); - let v1: __m128i = - crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[1]); - let v2: __m128i = - crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[2]); - let v3: __m128i = - crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[3]); - let mask = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); - let v0 = _mm_shuffle_epi8(v0, mask); - let v1 = _mm_shuffle_epi8(v1, mask); - let v2 = _mm_shuffle_epi8(v2, mask); - let v3 = _mm_shuffle_epi8(v3, mask); - let tmp0 = _mm_unpacklo_epi32(v0, v1); - let tmp1 = _mm_unpackhi_epi32(v0, v1); - let tmp2 = _mm_unpacklo_epi32(v2, v3); - let tmp3 = _mm_unpackhi_epi32(v2, v3); - let out0 = _mm_unpacklo_epi64(tmp0, tmp2); - let out1 = _mm_unpackhi_epi64(tmp0, tmp2); - let out2 = _mm_unpacklo_epi64(tmp1, tmp3); - let out3 = _mm_unpackhi_epi64(tmp1, tmp3); - [ - out0.simd_into(token), - out1.simd_into(token), - out2.simd_into(token), - out3.simd_into(token), - ] - } - ); - kernel(self, src) - } - #[inline(always)] - fn store_four_interleaved_u8x16( - self, - vectors: [u8x16; 4usize], - dest: &mut [u8; 64usize], - ) -> () { + vectors: [u8x16; 4usize], + dest: &mut [u8; 64usize], + ) -> () { crate::kernel!( #[inline(always)] fn kernel( @@ -2341,6 +2441,56 @@ impl Simd for Sse4_2 { }) } #[inline(always)] + fn compress_i16x8(self, values: i16x8, mask: mask16x8) -> i16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i16x8( + self, + values: i16x8, + mask: mask16x8, + merge: i16x8, + ) -> i16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i16x8(self, values: i16x8, mask: mask16x8) -> i16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i16x8( + self, + values: i16x8, + mask: mask16x8, + merge: i16x8, + ) -> i16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_i16x8(self, a: i16x8) -> i16x8 { crate::kernel!( #[inline(always)] @@ -2890,6 +3040,56 @@ impl Simd for Sse4_2 { }) } #[inline(always)] + fn compress_u16x8(self, values: u16x8, mask: mask16x8) -> u16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u16x8( + self, + values: u16x8, + mask: mask16x8, + merge: u16x8, + ) -> u16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u16x8(self, values: u16x8, mask: mask16x8) -> u16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u16x8( + self, + values: u16x8, + mask: mask16x8, + merge: u16x8, + ) -> u16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_u16x8(self, a: u16x8) -> u16x8 { crate::kernel!( #[inline(always)] @@ -3677,6 +3877,56 @@ impl Simd for Sse4_2 { }) } #[inline(always)] + fn compress_i32x4(self, values: i32x4, mask: mask32x4) -> i32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i32x4( + self, + values: i32x4, + mask: mask32x4, + merge: i32x4, + ) -> i32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i32x4(self, values: i32x4, mask: mask32x4) -> i32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i32x4( + self, + values: i32x4, + mask: mask32x4, + merge: i32x4, + ) -> i32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_i32x4(self, a: i32x4) -> i32x4 { crate::kernel!( #[inline(always)] @@ -4213,6 +4463,56 @@ impl Simd for Sse4_2 { }) } #[inline(always)] + fn compress_u32x4(self, values: u32x4, mask: mask32x4) -> u32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u32x4( + self, + values: u32x4, + mask: mask32x4, + merge: u32x4, + ) -> u32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u32x4(self, values: u32x4, mask: mask32x4) -> u32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u32x4( + self, + values: u32x4, + mask: mask32x4, + merge: u32x4, + ) -> u32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_u32x4(self, a: u32x4) -> u32x4 { crate::kernel!( #[inline(always)] @@ -4975,6 +5275,56 @@ impl Simd for Sse4_2 { }) } #[inline(always)] + fn compress_f64x2(self, values: f64x2, mask: mask64x2) -> f64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_f64x2( + self, + values: f64x2, + mask: mask64x2, + merge: f64x2, + ) -> f64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_f64x2(self, values: f64x2, mask: mask64x2) -> f64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_f64x2( + self, + values: f64x2, + mask: mask64x2, + merge: f64x2, + ) -> f64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn neg_f64x2(self, a: f64x2) -> f64x2 { crate::kernel!( #[inline(always)] @@ -5542,6 +5892,56 @@ impl Simd for Sse4_2 { }) } #[inline(always)] + fn compress_i64x2(self, values: i64x2, mask: mask64x2) -> i64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i64x2( + self, + values: i64x2, + mask: mask64x2, + merge: i64x2, + ) -> i64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i64x2(self, values: i64x2, mask: mask64x2) -> i64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i64x2( + self, + values: i64x2, + mask: mask64x2, + merge: i64x2, + ) -> i64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_i64x2(self, a: i64x2) -> i64x2 { crate::kernel!( #[inline(always)] @@ -6039,6 +6439,56 @@ impl Simd for Sse4_2 { }) } #[inline(always)] + fn compress_u64x2(self, values: u64x2, mask: mask64x2) -> u64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u64x2( + self, + values: u64x2, + mask: mask64x2, + merge: u64x2, + ) -> u64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u64x2(self, values: u64x2, mask: mask64x2) -> u64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u64x2( + self, + values: u64x2, + mask: mask64x2, + merge: u64x2, + ) -> u64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_u64x2(self, a: u64x2) -> u64x2 { crate::kernel!( #[inline(always)] @@ -6717,19 +7167,69 @@ impl Simd for Sse4_2 { }) } #[inline(always)] - fn is_nan_f32x8(self, a: f32x8) -> mask32x8 { - !a.simd_eq(a) + fn compress_f32x8(self, values: f32x8, mask: mask32x8) -> f32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) } #[inline(always)] - fn is_infinite_f32x8(self, a: f32x8) -> mask32x8 { - a.abs().simd_eq(f32::INFINITY) + fn compress_merge_f32x8( + self, + values: f32x8, + mask: mask32x8, + merge: f32x8, + ) -> f32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) } #[inline(always)] - fn is_finite_f32x8(self, a: f32x8) -> mask32x8 { - a.abs().simd_lt(f32::INFINITY) + fn expand_f32x8(self, values: f32x8, mask: mask32x8) -> f32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) } #[inline(always)] - fn is_subnormal_f32x8(self, a: f32x8) -> mask32x8 { + fn expand_merge_f32x8( + self, + values: f32x8, + mask: mask32x8, + merge: f32x8, + ) -> f32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn is_nan_f32x8(self, a: f32x8) -> mask32x8 { + !a.simd_eq(a) + } + #[inline(always)] + fn is_infinite_f32x8(self, a: f32x8) -> mask32x8 { + a.abs().simd_eq(f32::INFINITY) + } + #[inline(always)] + fn is_finite_f32x8(self, a: f32x8) -> mask32x8 { + a.abs().simd_lt(f32::INFINITY) + } + #[inline(always)] + fn is_subnormal_f32x8(self, a: f32x8) -> mask32x8 { let i: i32x8 = a.bitcast(); (i & 0x7F80_0000_i32).simd_eq(0) & !(i & 0x007F_FFFF_i32).simd_eq(0) } @@ -6785,6 +7285,56 @@ impl Simd for Sse4_2 { }) } #[inline(always)] + fn compress_i8x32(self, values: i8x32, mask: mask8x32) -> i8x32 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i8x32( + self, + values: i8x32, + mask: mask8x32, + merge: i8x32, + ) -> i8x32 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i8x32(self, values: i8x32, mask: mask8x32) -> i8x32 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i8x32( + self, + values: i8x32, + mask: mask8x32, + merge: i8x32, + ) -> i8x32 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn combine_i8x32(self, a: i8x32, b: i8x32) -> i8x64 { i8x64 { val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), @@ -7231,6 +7781,56 @@ impl Simd for Sse4_2 { }) } #[inline(always)] + fn compress_i16x16(self, values: i16x16, mask: mask16x16) -> i16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i16x16( + self, + values: i16x16, + mask: mask16x16, + merge: i16x16, + ) -> i16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i16x16(self, values: i16x16, mask: mask16x16) -> i16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i16x16( + self, + values: i16x16, + mask: mask16x16, + merge: i16x16, + ) -> i16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn combine_i16x16(self, a: i16x16, b: i16x16) -> i16x32 { i16x32 { val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), @@ -7267,6 +7867,56 @@ impl Simd for Sse4_2 { }) } #[inline(always)] + fn compress_u16x16(self, values: u16x16, mask: mask16x16) -> u16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u16x16( + self, + values: u16x16, + mask: mask16x16, + merge: u16x16, + ) -> u16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u16x16(self, values: u16x16, mask: mask16x16) -> u16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u16x16( + self, + values: u16x16, + mask: mask16x16, + merge: u16x16, + ) -> u16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn combine_u16x16(self, a: u16x16, b: u16x16) -> u16x32 { u16x32 { val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), @@ -7377,6 +8027,56 @@ impl Simd for Sse4_2 { }) } #[inline(always)] + fn compress_i32x8(self, values: i32x8, mask: mask32x8) -> i32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i32x8( + self, + values: i32x8, + mask: mask32x8, + merge: i32x8, + ) -> i32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i32x8(self, values: i32x8, mask: mask32x8) -> i32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i32x8( + self, + values: i32x8, + mask: mask32x8, + merge: i32x8, + ) -> i32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn combine_i32x8(self, a: i32x8, b: i32x8) -> i32x16 { i32x16 { val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), @@ -7413,6 +8113,56 @@ impl Simd for Sse4_2 { }) } #[inline(always)] + fn compress_u32x8(self, values: u32x8, mask: mask32x8) -> u32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u32x8( + self, + values: u32x8, + mask: mask32x8, + merge: u32x8, + ) -> u32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u32x8(self, values: u32x8, mask: mask32x8) -> u32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u32x8( + self, + values: u32x8, + mask: mask32x8, + merge: u32x8, + ) -> u32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn combine_u32x8(self, a: u32x8, b: u32x8) -> u32x16 { u32x16 { val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), @@ -7510,6 +8260,56 @@ impl Simd for Sse4_2 { }) } #[inline(always)] + fn compress_f64x4(self, values: f64x4, mask: mask64x4) -> f64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_f64x4( + self, + values: f64x4, + mask: mask64x4, + merge: f64x4, + ) -> f64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_f64x4(self, values: f64x4, mask: mask64x4) -> f64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_f64x4( + self, + values: f64x4, + mask: mask64x4, + merge: f64x4, + ) -> f64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn is_nan_f64x4(self, a: f64x4) -> mask64x4 { !a.simd_eq(a) } @@ -7578,6 +8378,56 @@ impl Simd for Sse4_2 { }) } #[inline(always)] + fn compress_i64x4(self, values: i64x4, mask: mask64x4) -> i64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i64x4( + self, + values: i64x4, + mask: mask64x4, + merge: i64x4, + ) -> i64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i64x4(self, values: i64x4, mask: mask64x4) -> i64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i64x4( + self, + values: i64x4, + mask: mask64x4, + merge: i64x4, + ) -> i64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn combine_i64x4(self, a: i64x4, b: i64x4) -> i64x8 { i64x8 { val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), @@ -7614,6 +8464,56 @@ impl Simd for Sse4_2 { }) } #[inline(always)] + fn compress_u64x4(self, values: u64x4, mask: mask64x4) -> u64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u64x4( + self, + values: u64x4, + mask: mask64x4, + merge: u64x4, + ) -> u64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u64x4(self, values: u64x4, mask: mask64x4) -> u64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u64x4( + self, + values: u64x4, + mask: mask64x4, + merge: u64x4, + ) -> u64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn combine_u64x4(self, a: u64x4, b: u64x4) -> u64x8 { u64x8 { val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), @@ -7711,6 +8611,56 @@ impl Simd for Sse4_2 { }) } #[inline(always)] + fn compress_f32x16(self, values: f32x16, mask: mask32x16) -> f32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_f32x16( + self, + values: f32x16, + mask: mask32x16, + merge: f32x16, + ) -> f32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_f32x16(self, values: f32x16, mask: mask32x16) -> f32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_f32x16( + self, + values: f32x16, + mask: mask32x16, + merge: f32x16, + ) -> f32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn is_nan_f32x16(self, a: f32x16) -> mask32x16 { !a.simd_eq(a) } @@ -7772,6 +8722,56 @@ impl Simd for Sse4_2 { }) } #[inline(always)] + fn compress_i8x64(self, values: i8x64, mask: mask8x64) -> i8x64 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i8x64( + self, + values: i8x64, + mask: mask8x64, + merge: i8x64, + ) -> i8x64 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i8x64(self, values: i8x64, mask: mask8x64) -> i8x64 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i8x64( + self, + values: i8x64, + mask: mask8x64, + merge: i8x64, + ) -> i8x64 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_i8x64(self, a: i8x64) -> (i8x32, i8x32) { ( i8x32 { @@ -8422,6 +9422,56 @@ impl Simd for Sse4_2 { }) } #[inline(always)] + fn compress_i16x32(self, values: i16x32, mask: mask16x32) -> i16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i16x32( + self, + values: i16x32, + mask: mask16x32, + merge: i16x32, + ) -> i16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i16x32(self, values: i16x32, mask: mask16x32) -> i16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i16x32( + self, + values: i16x32, + mask: mask16x32, + merge: i16x32, + ) -> i16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_i16x32(self, a: i16x32) -> (i16x16, i16x16) { ( i16x16 { @@ -8451,6 +9501,56 @@ impl Simd for Sse4_2 { }) } #[inline(always)] + fn compress_u16x32(self, values: u16x32, mask: mask16x32) -> u16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u16x32( + self, + values: u16x32, + mask: mask16x32, + merge: u16x32, + ) -> u16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u16x32(self, values: u16x32, mask: mask16x32) -> u16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u16x32( + self, + values: u16x32, + mask: mask16x32, + merge: u16x32, + ) -> u16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_u16x32(self, a: u16x32) -> (u16x16, u16x16) { ( u16x16 { @@ -8550,6 +9650,56 @@ impl Simd for Sse4_2 { }) } #[inline(always)] + fn compress_i32x16(self, values: i32x16, mask: mask32x16) -> i32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i32x16( + self, + values: i32x16, + mask: mask32x16, + merge: i32x16, + ) -> i32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i32x16(self, values: i32x16, mask: mask32x16) -> i32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i32x16( + self, + values: i32x16, + mask: mask32x16, + merge: i32x16, + ) -> i32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_i32x16(self, a: i32x16) -> (i32x8, i32x8) { ( i32x8 { @@ -8579,6 +9729,56 @@ impl Simd for Sse4_2 { }) } #[inline(always)] + fn compress_u32x16(self, values: u32x16, mask: mask32x16) -> u32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u32x16( + self, + values: u32x16, + mask: mask32x16, + merge: u32x16, + ) -> u32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u32x16(self, values: u32x16, mask: mask32x16) -> u32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u32x16( + self, + values: u32x16, + mask: mask32x16, + merge: u32x16, + ) -> u32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_u32x16(self, a: u32x16) -> (u32x8, u32x8) { ( u32x8 { @@ -8662,6 +9862,56 @@ impl Simd for Sse4_2 { }) } #[inline(always)] + fn compress_f64x8(self, values: f64x8, mask: mask64x8) -> f64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_f64x8( + self, + values: f64x8, + mask: mask64x8, + merge: f64x8, + ) -> f64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_f64x8(self, values: f64x8, mask: mask64x8) -> f64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_f64x8( + self, + values: f64x8, + mask: mask64x8, + merge: f64x8, + ) -> f64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn is_nan_f64x8(self, a: f64x8) -> mask64x8 { !a.simd_eq(a) } @@ -8723,6 +9973,56 @@ impl Simd for Sse4_2 { }) } #[inline(always)] + fn compress_i64x8(self, values: i64x8, mask: mask64x8) -> i64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i64x8( + self, + values: i64x8, + mask: mask64x8, + merge: i64x8, + ) -> i64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i64x8(self, values: i64x8, mask: mask64x8) -> i64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i64x8( + self, + values: i64x8, + mask: mask64x8, + merge: i64x8, + ) -> i64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_i64x8(self, a: i64x8) -> (i64x4, i64x4) { ( i64x4 { @@ -8752,6 +10052,56 @@ impl Simd for Sse4_2 { }) } #[inline(always)] + fn compress_u64x8(self, values: u64x8, mask: mask64x8) -> u64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u64x8( + self, + values: u64x8, + mask: mask64x8, + merge: u64x8, + ) -> u64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u64x8(self, values: u64x8, mask: mask64x8) -> u64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u64x8( + self, + values: u64x8, + mask: mask64x8, + merge: u64x8, + ) -> u64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_u64x8(self, a: u64x8) -> (u64x4, u64x4) { ( u64x4 { diff --git a/fearless_simd/src/generated/wasm.rs b/fearless_simd/src/generated/wasm.rs index f981c553e..f73a8dfb2 100644 --- a/fearless_simd/src/generated/wasm.rs +++ b/fearless_simd/src/generated/wasm.rs @@ -147,6 +147,56 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn compress_f32x4(self, values: f32x4, mask: mask32x4) -> f32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_f32x4( + self, + values: f32x4, + mask: mask32x4, + merge: f32x4, + ) -> f32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_f32x4(self, values: f32x4, mask: mask32x4) -> f32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_f32x4( + self, + values: f32x4, + mask: mask32x4, + merge: f32x4, + ) -> f32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn neg_f32x4(self, a: f32x4) -> f32x4 { f32x4_neg(a.into()).simd_into(self) } @@ -514,6 +564,56 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn compress_i8x16(self, values: i8x16, mask: mask8x16) -> i8x16 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i8x16( + self, + values: i8x16, + mask: mask8x16, + merge: i8x16, + ) -> i8x16 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i8x16(self, values: i8x16, mask: mask8x16) -> i8x16 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i8x16( + self, + values: i8x16, + mask: mask8x16, + merge: i8x16, + ) -> i8x16 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_i8x16(self, a: i8x16) -> i8x16 { i8x16_popcnt(a.into()).simd_into(self) } @@ -941,6 +1041,98 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn compress_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16 { + let mask_bits = self.to_bitmask_mask8x16(mask); + let mut control = [u8::MAX; 16]; + let mut output_lane = 0; + for block in 0..16 / 8 { + let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; + let packed = crate::support::COMPRESS_8_CONTROLS[block_mask]; + let base = (block * 8) as u64 * 0x0101_0101_0101_0101; + let adjusted = + ((packed & 0x7f7f_7f7f_7f7f_7f7f) + base) | (packed & 0x8080_8080_8080_8080); + let adjusted = adjusted.to_le_bytes(); + let write_len = core::cmp::min(8, 16 - output_lane); + control[output_lane..output_lane + write_len].copy_from_slice(&adjusted[..write_len]); + output_lane += block_mask.count_ones() as usize; + } + let control = u8x16::simd_from(self, control); + self.swizzle_dyn_precise_u8x16(values, control) + } + #[inline(always)] + fn compress_merge_u8x16( + self, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + let mask_bits = self.to_bitmask_mask8x16(mask); + let mut control = [u8::MAX; 16]; + let mut output_lane = 0; + for block in 0..16 / 8 { + let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; + let packed = crate::support::COMPRESS_8_CONTROLS[block_mask]; + let base = (block * 8) as u64 * 0x0101_0101_0101_0101; + let adjusted = + ((packed & 0x7f7f_7f7f_7f7f_7f7f) + base) | (packed & 0x8080_8080_8080_8080); + let adjusted = adjusted.to_le_bytes(); + let write_len = core::cmp::min(8, 16 - output_lane); + control[output_lane..output_lane + write_len].copy_from_slice(&adjusted[..write_len]); + output_lane += block_mask.count_ones() as usize; + } + let control = u8x16::simd_from(self, control); + let compressed = self.swizzle_dyn_precise_u8x16(values, control); + let prefix_bits = if output_lane == 64 { + u64::MAX + } else { + (1u64 << output_lane) - 1 + }; + let prefix_mask = self.from_bitmask_mask8x16(prefix_bits); + self.select_u8x16(prefix_mask, compressed, merge) + } + #[inline(always)] + fn expand_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16 { + let mask_bits = self.to_bitmask_mask8x16(mask); + let mut control = [u8::MAX; 16]; + let mut input_lane = 0; + for block in 0..16 / 8 { + let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; + let packed = crate::support::EXPAND_8_CONTROLS[block_mask]; + let base = input_lane as u64 * 0x0101_0101_0101_0101; + let adjusted = + ((packed & 0x7f7f_7f7f_7f7f_7f7f) + base) | (packed & 0x8080_8080_8080_8080); + let output_lane = block * 8; + control[output_lane..output_lane + 8].copy_from_slice(&adjusted.to_le_bytes()); + input_lane += block_mask.count_ones() as usize; + } + let control = u8x16::simd_from(self, control); + self.swizzle_dyn_precise_u8x16(values, control) + } + #[inline(always)] + fn expand_merge_u8x16( + self, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + let mask_bits = self.to_bitmask_mask8x16(mask); + let mut control = [u8::MAX; 16]; + let mut input_lane = 0; + for block in 0..16 / 8 { + let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; + let packed = crate::support::EXPAND_8_CONTROLS[block_mask]; + let base = input_lane as u64 * 0x0101_0101_0101_0101; + let adjusted = + ((packed & 0x7f7f_7f7f_7f7f_7f7f) + base) | (packed & 0x8080_8080_8080_8080); + let output_lane = block * 8; + control[output_lane..output_lane + 8].copy_from_slice(&adjusted.to_le_bytes()); + input_lane += block_mask.count_ones() as usize; + } + let control = u8x16::simd_from(self, control); + let expanded = self.swizzle_dyn_precise_u8x16(values, control); + self.select_u8x16(mask, expanded, merge) + } + #[inline(always)] fn count_ones_u8x16(self, a: u8x16) -> u8x16 { i8x16_popcnt(a.into()).simd_into(self) } @@ -1195,98 +1387,6 @@ impl Simd for WasmSimd128 { } } #[inline(always)] - fn compress_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16 { - let mask_bits = self.to_bitmask_mask8x16(mask); - let mut control = [u8::MAX; 16]; - let mut output_lane = 0; - for block in 0..16 / 8 { - let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; - let packed = crate::support::COMPRESS_8_CONTROLS[block_mask]; - let base = (block * 8) as u64 * 0x0101_0101_0101_0101; - let adjusted = - ((packed & 0x7f7f_7f7f_7f7f_7f7f) + base) | (packed & 0x8080_8080_8080_8080); - let adjusted = adjusted.to_le_bytes(); - let write_len = core::cmp::min(8, 16 - output_lane); - control[output_lane..output_lane + write_len].copy_from_slice(&adjusted[..write_len]); - output_lane += block_mask.count_ones() as usize; - } - let control = u8x16::simd_from(self, control); - self.swizzle_dyn_precise_u8x16(values, control) - } - #[inline(always)] - fn compress_merge_u8x16( - self, - values: u8x16, - mask: mask8x16, - merge: u8x16, - ) -> u8x16 { - let mask_bits = self.to_bitmask_mask8x16(mask); - let mut control = [u8::MAX; 16]; - let mut output_lane = 0; - for block in 0..16 / 8 { - let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; - let packed = crate::support::COMPRESS_8_CONTROLS[block_mask]; - let base = (block * 8) as u64 * 0x0101_0101_0101_0101; - let adjusted = - ((packed & 0x7f7f_7f7f_7f7f_7f7f) + base) | (packed & 0x8080_8080_8080_8080); - let adjusted = adjusted.to_le_bytes(); - let write_len = core::cmp::min(8, 16 - output_lane); - control[output_lane..output_lane + write_len].copy_from_slice(&adjusted[..write_len]); - output_lane += block_mask.count_ones() as usize; - } - let control = u8x16::simd_from(self, control); - let compressed = self.swizzle_dyn_precise_u8x16(values, control); - let prefix_bits = if output_lane == 64 { - u64::MAX - } else { - (1u64 << output_lane) - 1 - }; - let prefix_mask = self.from_bitmask_mask8x16(prefix_bits); - self.select_u8x16(prefix_mask, compressed, merge) - } - #[inline(always)] - fn expand_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16 { - let mask_bits = self.to_bitmask_mask8x16(mask); - let mut control = [u8::MAX; 16]; - let mut input_lane = 0; - for block in 0..16 / 8 { - let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; - let packed = crate::support::EXPAND_8_CONTROLS[block_mask]; - let base = input_lane as u64 * 0x0101_0101_0101_0101; - let adjusted = - ((packed & 0x7f7f_7f7f_7f7f_7f7f) + base) | (packed & 0x8080_8080_8080_8080); - let output_lane = block * 8; - control[output_lane..output_lane + 8].copy_from_slice(&adjusted.to_le_bytes()); - input_lane += block_mask.count_ones() as usize; - } - let control = u8x16::simd_from(self, control); - self.swizzle_dyn_precise_u8x16(values, control) - } - #[inline(always)] - fn expand_merge_u8x16( - self, - values: u8x16, - mask: mask8x16, - merge: u8x16, - ) -> u8x16 { - let mask_bits = self.to_bitmask_mask8x16(mask); - let mut control = [u8::MAX; 16]; - let mut input_lane = 0; - for block in 0..16 / 8 { - let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; - let packed = crate::support::EXPAND_8_CONTROLS[block_mask]; - let base = input_lane as u64 * 0x0101_0101_0101_0101; - let adjusted = - ((packed & 0x7f7f_7f7f_7f7f_7f7f) + base) | (packed & 0x8080_8080_8080_8080); - let output_lane = block * 8; - control[output_lane..output_lane + 8].copy_from_slice(&adjusted.to_le_bytes()); - input_lane += block_mask.count_ones() as usize; - } - let control = u8x16::simd_from(self, control); - let expanded = self.swizzle_dyn_precise_u8x16(values, control); - self.select_u8x16(mask, expanded, merge) - } - #[inline(always)] fn combine_u8x16(self, a: u8x16, b: u8x16) -> u8x32 { u8x32 { val: crate::support::Aligned256([a.val.0, b.val.0]), @@ -1541,6 +1641,56 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn compress_i16x8(self, values: i16x8, mask: mask16x8) -> i16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i16x8( + self, + values: i16x8, + mask: mask16x8, + merge: i16x8, + ) -> i16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i16x8(self, values: i16x8, mask: mask16x8) -> i16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i16x8( + self, + values: i16x8, + mask: mask16x8, + merge: i16x8, + ) -> i16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_i16x8(self, a: i16x8) -> i16x8 { u16x8_extadd_pairwise_u8x16(i8x16_popcnt(a.into())).simd_into(self) } @@ -1832,21 +1982,71 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] - fn count_ones_u16x8(self, a: u16x8) -> u16x8 { - u16x8_extadd_pairwise_u8x16(i8x16_popcnt(a.into())).simd_into(self) + fn compress_u16x8(self, values: u16x8, mask: mask16x8) -> u16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) } #[inline(always)] - fn count_zeros_u16x8(self, a: u16x8) -> u16x8 { - self.count_ones_u16x8(self.not_u16x8(a)) + fn compress_merge_u16x8( + self, + values: u16x8, + mask: mask16x8, + merge: u16x8, + ) -> u16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) } #[inline(always)] - fn add_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { - u16x8_add(a.into(), b.into()).simd_into(self) + fn expand_u16x8(self, values: u16x8, mask: mask16x8) -> u16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) } #[inline(always)] - fn saturating_add_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { - u16x8_add_sat(a.into(), b.into()).simd_into(self) - } + fn expand_merge_u16x8( + self, + values: u16x8, + mask: mask16x8, + merge: u16x8, + ) -> u16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn count_ones_u16x8(self, a: u16x8) -> u16x8 { + u16x8_extadd_pairwise_u8x16(i8x16_popcnt(a.into())).simd_into(self) + } + #[inline(always)] + fn count_zeros_u16x8(self, a: u16x8) -> u16x8 { + self.count_ones_u16x8(self.not_u16x8(a)) + } + #[inline(always)] + fn add_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { + u16x8_add(a.into(), b.into()).simd_into(self) + } + #[inline(always)] + fn saturating_add_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { + u16x8_add_sat(a.into(), b.into()).simd_into(self) + } #[inline(always)] fn sub_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { u16x8_sub(a.into(), b.into()).simd_into(self) @@ -2263,6 +2463,56 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn compress_i32x4(self, values: i32x4, mask: mask32x4) -> i32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i32x4( + self, + values: i32x4, + mask: mask32x4, + merge: i32x4, + ) -> i32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i32x4(self, values: i32x4, mask: mask32x4) -> i32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i32x4( + self, + values: i32x4, + mask: mask32x4, + merge: i32x4, + ) -> i32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_i32x4(self, a: i32x4) -> i32x4 { u32x4_extadd_pairwise_u16x8(u16x8_extadd_pairwise_u8x16(i8x16_popcnt(a.into()))) .simd_into(self) @@ -2549,6 +2799,56 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn compress_u32x4(self, values: u32x4, mask: mask32x4) -> u32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u32x4( + self, + values: u32x4, + mask: mask32x4, + merge: u32x4, + ) -> u32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u32x4(self, values: u32x4, mask: mask32x4) -> u32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u32x4( + self, + values: u32x4, + mask: mask32x4, + merge: u32x4, + ) -> u32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_u32x4(self, a: u32x4) -> u32x4 { u32x4_extadd_pairwise_u16x8(u16x8_extadd_pairwise_u8x16(i8x16_popcnt(a.into()))) .simd_into(self) @@ -2969,6 +3269,56 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn compress_f64x2(self, values: f64x2, mask: mask64x2) -> f64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_f64x2( + self, + values: f64x2, + mask: mask64x2, + merge: f64x2, + ) -> f64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_f64x2(self, values: f64x2, mask: mask64x2) -> f64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_f64x2( + self, + values: f64x2, + mask: mask64x2, + merge: f64x2, + ) -> f64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn neg_f64x2(self, a: f64x2) -> f64x2 { f64x2_neg(a.into()).simd_into(self) } @@ -3314,6 +3664,56 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn compress_i64x2(self, values: i64x2, mask: mask64x2) -> i64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i64x2( + self, + values: i64x2, + mask: mask64x2, + merge: i64x2, + ) -> i64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i64x2(self, values: i64x2, mask: mask64x2) -> i64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i64x2( + self, + values: i64x2, + mask: mask64x2, + merge: i64x2, + ) -> i64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_i64x2(self, a: i64x2) -> i64x2 { { let counts = @@ -3594,6 +3994,56 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn compress_u64x2(self, values: u64x2, mask: mask64x2) -> u64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u64x2( + self, + values: u64x2, + mask: mask64x2, + merge: u64x2, + ) -> u64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u64x2(self, values: u64x2, mask: mask64x2) -> u64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u64x2( + self, + values: u64x2, + mask: mask64x2, + merge: u64x2, + ) -> u64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_u64x2(self, a: u64x2) -> u64x2 { { let counts = @@ -4008,6 +4458,56 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn compress_f32x8(self, values: f32x8, mask: mask32x8) -> f32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_f32x8( + self, + values: f32x8, + mask: mask32x8, + merge: f32x8, + ) -> f32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_f32x8(self, values: f32x8, mask: mask32x8) -> f32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_f32x8( + self, + values: f32x8, + mask: mask32x8, + merge: f32x8, + ) -> f32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn is_nan_f32x8(self, a: f32x8) -> mask32x8 { !a.simd_eq(a) } @@ -4072,6 +4572,56 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn compress_i8x32(self, values: i8x32, mask: mask8x32) -> i8x32 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i8x32( + self, + values: i8x32, + mask: mask8x32, + merge: i8x32, + ) -> i8x32 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i8x32(self, values: i8x32, mask: mask8x32) -> i8x32 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i8x32( + self, + values: i8x32, + mask: mask8x32, + merge: i8x32, + ) -> i8x32 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn combine_i8x32(self, a: i8x32, b: i8x32) -> i8x64 { i8x64 { val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), @@ -4403,6 +4953,56 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn compress_i16x16(self, values: i16x16, mask: mask16x16) -> i16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i16x16( + self, + values: i16x16, + mask: mask16x16, + merge: i16x16, + ) -> i16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i16x16(self, values: i16x16, mask: mask16x16) -> i16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i16x16( + self, + values: i16x16, + mask: mask16x16, + merge: i16x16, + ) -> i16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn combine_i16x16(self, a: i16x16, b: i16x16) -> i16x32 { i16x32 { val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), @@ -4438,6 +5038,56 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn compress_u16x16(self, values: u16x16, mask: mask16x16) -> u16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u16x16( + self, + values: u16x16, + mask: mask16x16, + merge: u16x16, + ) -> u16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u16x16(self, values: u16x16, mask: mask16x16) -> u16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u16x16( + self, + values: u16x16, + mask: mask16x16, + merge: u16x16, + ) -> u16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn combine_u16x16(self, a: u16x16, b: u16x16) -> u16x32 { u16x32 { val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), @@ -4534,6 +5184,56 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn compress_i32x8(self, values: i32x8, mask: mask32x8) -> i32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i32x8( + self, + values: i32x8, + mask: mask32x8, + merge: i32x8, + ) -> i32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i32x8(self, values: i32x8, mask: mask32x8) -> i32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i32x8( + self, + values: i32x8, + mask: mask32x8, + merge: i32x8, + ) -> i32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn combine_i32x8(self, a: i32x8, b: i32x8) -> i32x16 { i32x16 { val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), @@ -4569,6 +5269,56 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn compress_u32x8(self, values: u32x8, mask: mask32x8) -> u32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u32x8( + self, + values: u32x8, + mask: mask32x8, + merge: u32x8, + ) -> u32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u32x8(self, values: u32x8, mask: mask32x8) -> u32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u32x8( + self, + values: u32x8, + mask: mask32x8, + merge: u32x8, + ) -> u32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn combine_u32x8(self, a: u32x8, b: u32x8) -> u32x16 { u32x16 { val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), @@ -4665,6 +5415,56 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn compress_f64x4(self, values: f64x4, mask: mask64x4) -> f64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_f64x4( + self, + values: f64x4, + mask: mask64x4, + merge: f64x4, + ) -> f64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_f64x4(self, values: f64x4, mask: mask64x4) -> f64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_f64x4( + self, + values: f64x4, + mask: mask64x4, + merge: f64x4, + ) -> f64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn is_nan_f64x4(self, a: f64x4) -> mask64x4 { !a.simd_eq(a) } @@ -4732,6 +5532,56 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn compress_i64x4(self, values: i64x4, mask: mask64x4) -> i64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i64x4( + self, + values: i64x4, + mask: mask64x4, + merge: i64x4, + ) -> i64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i64x4(self, values: i64x4, mask: mask64x4) -> i64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i64x4( + self, + values: i64x4, + mask: mask64x4, + merge: i64x4, + ) -> i64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn combine_i64x4(self, a: i64x4, b: i64x4) -> i64x8 { i64x8 { val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), @@ -4767,6 +5617,56 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn compress_u64x4(self, values: u64x4, mask: mask64x4) -> u64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u64x4( + self, + values: u64x4, + mask: mask64x4, + merge: u64x4, + ) -> u64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u64x4(self, values: u64x4, mask: mask64x4) -> u64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u64x4( + self, + values: u64x4, + mask: mask64x4, + merge: u64x4, + ) -> u64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn combine_u64x4(self, a: u64x4, b: u64x4) -> u64x8 { u64x8 { val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), @@ -4863,6 +5763,56 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn compress_f32x16(self, values: f32x16, mask: mask32x16) -> f32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_f32x16( + self, + values: f32x16, + mask: mask32x16, + merge: f32x16, + ) -> f32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_f32x16(self, values: f32x16, mask: mask32x16) -> f32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_f32x16( + self, + values: f32x16, + mask: mask32x16, + merge: f32x16, + ) -> f32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn is_nan_f32x16(self, a: f32x16) -> mask32x16 { !a.simd_eq(a) } @@ -4920,6 +5870,56 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn compress_i8x64(self, values: i8x64, mask: mask8x64) -> i8x64 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i8x64( + self, + values: i8x64, + mask: mask8x64, + merge: i8x64, + ) -> i8x64 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i8x64(self, values: i8x64, mask: mask8x64) -> i8x64 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i8x64( + self, + values: i8x64, + mask: mask8x64, + merge: i8x64, + ) -> i8x64 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_i8x64(self, a: i8x64) -> (i8x32, i8x32) { ( i8x32 { @@ -5203,6 +6203,56 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn compress_i16x32(self, values: i16x32, mask: mask16x32) -> i16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i16x32( + self, + values: i16x32, + mask: mask16x32, + merge: i16x32, + ) -> i16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i16x32(self, values: i16x32, mask: mask16x32) -> i16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i16x32( + self, + values: i16x32, + mask: mask16x32, + merge: i16x32, + ) -> i16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_i16x32(self, a: i16x32) -> (i16x16, i16x16) { ( i16x16 { @@ -5231,6 +6281,56 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn compress_u16x32(self, values: u16x32, mask: mask16x32) -> u16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u16x32( + self, + values: u16x32, + mask: mask16x32, + merge: u16x32, + ) -> u16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u16x32(self, values: u16x32, mask: mask16x32) -> u16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u16x32( + self, + values: u16x32, + mask: mask16x32, + merge: u16x32, + ) -> u16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_u16x32(self, a: u16x32) -> (u16x16, u16x16) { ( u16x16 { @@ -5313,6 +6413,56 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn compress_i32x16(self, values: i32x16, mask: mask32x16) -> i32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i32x16( + self, + values: i32x16, + mask: mask32x16, + merge: i32x16, + ) -> i32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i32x16(self, values: i32x16, mask: mask32x16) -> i32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i32x16( + self, + values: i32x16, + mask: mask32x16, + merge: i32x16, + ) -> i32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_i32x16(self, a: i32x16) -> (i32x8, i32x8) { ( i32x8 { @@ -5341,6 +6491,56 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn compress_u32x16(self, values: u32x16, mask: mask32x16) -> u32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u32x16( + self, + values: u32x16, + mask: mask32x16, + merge: u32x16, + ) -> u32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u32x16(self, values: u32x16, mask: mask32x16) -> u32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u32x16( + self, + values: u32x16, + mask: mask32x16, + merge: u32x16, + ) -> u32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_u32x16(self, a: u32x16) -> (u32x8, u32x8) { ( u32x8 { @@ -5423,6 +6623,56 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn compress_f64x8(self, values: f64x8, mask: mask64x8) -> f64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_f64x8( + self, + values: f64x8, + mask: mask64x8, + merge: f64x8, + ) -> f64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_f64x8(self, values: f64x8, mask: mask64x8) -> f64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_f64x8( + self, + values: f64x8, + mask: mask64x8, + merge: f64x8, + ) -> f64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn is_nan_f64x8(self, a: f64x8) -> mask64x8 { !a.simd_eq(a) } @@ -5483,6 +6733,56 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn compress_i64x8(self, values: i64x8, mask: mask64x8) -> i64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i64x8( + self, + values: i64x8, + mask: mask64x8, + merge: i64x8, + ) -> i64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i64x8(self, values: i64x8, mask: mask64x8) -> i64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i64x8( + self, + values: i64x8, + mask: mask64x8, + merge: i64x8, + ) -> i64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_i64x8(self, a: i64x8) -> (i64x4, i64x4) { ( i64x4 { @@ -5511,6 +6811,56 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn compress_u64x8(self, values: u64x8, mask: mask64x8) -> u64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u64x8( + self, + values: u64x8, + mask: mask64x8, + merge: u64x8, + ) -> u64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u64x8(self, values: u64x8, mask: mask64x8) -> u64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u64x8( + self, + values: u64x8, + mask: mask64x8, + merge: u64x8, + ) -> u64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_u64x8(self, a: u64x8) -> (u64x4, u64x4) { ( u64x4 { diff --git a/fearless_simd_gen/src/generic.rs b/fearless_simd_gen/src/generic.rs index 3c5079c47..801d6e584 100644 --- a/fearless_simd_gen/src/generic.rs +++ b/fearless_simd_gen/src/generic.rs @@ -67,6 +67,34 @@ pub(crate) fn byte_swizzle_op(op: &Op, vec_ty: &VecType) -> TokenStream { } } +/// Forward typed compression/expansion through bytes on vector-backed mask backends. +/// +/// Each all-zero/all-one mask lane becomes a group of identical byte lanes, so entire +/// elements move together. This must not be used with compact predicate masks (AVX-512). +pub(crate) fn byte_compact_op(op: Op, vec_ty: &VecType) -> TokenStream { + assert_ne!(*vec_ty, vec_ty.bytes_ty()); + assert_ne!(vec_ty.scalar, ScalarType::Mask); + let method_sig = op.simd_trait_method_sig(vec_ty); + let byte_method = generic_op_name(op.method, &vec_ty.bytes_ty()); + let byte_mask = vec_ty.bytes_ty().mask_ty().rust(); + let merge_arg = match op.sig { + OpSig::Compress { merge: true } | OpSig::Expand { merge: true } => { + quote! { , Bytes::to_bytes(merge) } + } + OpSig::Compress { merge: false } | OpSig::Expand { merge: false } => TokenStream::new(), + _ => unreachable!("only compact operations can be forwarded through bytes"), + }; + quote! { + #method_sig { + let mask = #byte_mask { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.#byte_method(Bytes::to_bytes(values), mask #merge_arg)) + } + } +} + /// Implement a greater-than comparison by reversing the corresponding less-than comparison. pub(crate) fn reversed_compare_op(op: &Op, vec_ty: &VecType) -> Option { let reversed_method = generic_op_name(op.reversed_compare_method()?, vec_ty); diff --git a/fearless_simd_gen/src/mk_fallback.rs b/fearless_simd_gen/src/mk_fallback.rs index 1a86c1bed..5d3ad9322 100644 --- a/fearless_simd_gen/src/mk_fallback.rs +++ b/fearless_simd_gen/src/mk_fallback.rs @@ -3,8 +3,9 @@ use crate::arch::fallback; use crate::generic::{ - generic_classify, generic_mask_from_bitmask, generic_mask_set, generic_mask_to_bitmask, - generic_op, generic_op_name, integer_lane_mask_rotate, integer_lane_mask_splat_arg, + byte_compact_op, generic_classify, generic_mask_from_bitmask, generic_mask_set, + generic_mask_to_bitmask, generic_op, generic_op_name, integer_lane_mask_rotate, + integer_lane_mask_splat_arg, }; use crate::level::Level; use crate::ops::{NarrowingMode, Op, OpSig, relaxed_narrow_method}; @@ -591,6 +592,9 @@ impl Level for Fallback { } } } + OpSig::Compress { .. } | OpSig::Expand { .. } if *vec_ty != vec_ty.bytes_ty() => { + byte_compact_op(op, vec_ty) + } OpSig::Compress { .. } | OpSig::Expand { .. } => generic_op(&op, vec_ty), OpSig::Cvt { target_ty, diff --git a/fearless_simd_gen/src/mk_neon.rs b/fearless_simd_gen/src/mk_neon.rs index 59a44c772..53369243a 100644 --- a/fearless_simd_gen/src/mk_neon.rs +++ b/fearless_simd_gen/src/mk_neon.rs @@ -5,9 +5,9 @@ use proc_macro2::{Ident, Literal, Span, TokenStream}; use quote::{ToTokens as _, format_ident, quote}; use crate::generic::{ - CompactOptions, composed_compact_op, count_zeros_method, fallback_method, generic_classify, - generic_mask_set, generic_op_name, integer_lane_mask_rotate, integer_lane_mask_splat_arg, - reverse_method, reverse_vector_mask_method, + CompactOptions, byte_compact_op, composed_compact_op, count_zeros_method, fallback_method, + generic_classify, generic_mask_set, generic_op_name, integer_lane_mask_rotate, + integer_lane_mask_splat_arg, reverse_method, reverse_vector_mask_method, }; use crate::level::Level; use crate::ops::{NarrowingMode, Op, SlideGranularity, relaxed_narrow_method}; @@ -850,6 +850,9 @@ impl Level for Neon { } }) } + OpSig::Compress { .. } | OpSig::Expand { .. } if *vec_ty != vec_ty.bytes_ty() => { + byte_compact_op(op, vec_ty) + } OpSig::Compress { .. } | OpSig::Expand { .. } => composed_compact_op( op, vec_ty, diff --git a/fearless_simd_gen/src/mk_wasm.rs b/fearless_simd_gen/src/mk_wasm.rs index 2487c7f12..6c54314e6 100644 --- a/fearless_simd_gen/src/mk_wasm.rs +++ b/fearless_simd_gen/src/mk_wasm.rs @@ -6,10 +6,11 @@ use quote::{format_ident, quote}; use crate::arch::wasm::{arch_prefix, v128_intrinsic}; use crate::generic::{ - CompactOptions, composed_compact_op, concat_swizzle_dyn_precise_body, count_zeros_method, - fallback_method, generic_block_combine, generic_block_split, generic_classify, - generic_mask_set, generic_op_name, integer_lane_mask_rotate, integer_lane_mask_splat_arg, - recursive_swizzle_dyn_precise_body, reverse_method, reverse_vector_mask_method, + CompactOptions, byte_compact_op, composed_compact_op, concat_swizzle_dyn_precise_body, + count_zeros_method, fallback_method, generic_block_combine, generic_block_split, + generic_classify, generic_mask_set, generic_op_name, integer_lane_mask_rotate, + integer_lane_mask_splat_arg, recursive_swizzle_dyn_precise_body, reverse_method, + reverse_vector_mask_method, }; use crate::level::Level; use crate::ops::{ @@ -1166,6 +1167,9 @@ impl Level for WasmSimd128 { } _ => unreachable!(), }, + OpSig::Compress { .. } | OpSig::Expand { .. } if *vec_ty != vec_ty.bytes_ty() => { + byte_compact_op(op, vec_ty) + } OpSig::Compress { .. } | OpSig::Expand { .. } => composed_compact_op( op, vec_ty, diff --git a/fearless_simd_gen/src/mk_x86.rs b/fearless_simd_gen/src/mk_x86.rs index 11325763f..b20651fca 100644 --- a/fearless_simd_gen/src/mk_x86.rs +++ b/fearless_simd_gen/src/mk_x86.rs @@ -7,10 +7,11 @@ use crate::arch::x86::{ unpack_intrinsic, }; use crate::generic::{ - concat_swizzle_dyn_precise_body, count_zeros_method, fallback_method, generic_block_combine, - generic_block_split, generic_classify, generic_mask_from_bitmask, generic_mask_set, generic_op, - generic_op_name, integer_lane_mask_rotate, integer_lane_mask_splat_arg, - recursive_swizzle_dyn_precise_body, reverse_method, reverse_vector_mask_method, + byte_compact_op, concat_swizzle_dyn_precise_body, count_zeros_method, fallback_method, + generic_block_combine, generic_block_split, generic_classify, generic_mask_from_bitmask, + generic_mask_set, generic_op, generic_op_name, integer_lane_mask_rotate, + integer_lane_mask_splat_arg, recursive_swizzle_dyn_precise_body, reverse_method, + reverse_vector_mask_method, }; use crate::level::Level; use crate::ops::{ @@ -331,6 +332,8 @@ impl Level for X86 { OpSig::Compress { .. } | OpSig::Expand { .. } => { if *self == Self::Avx512 { self.handle_avx512_compact_op(op, vec_ty) + } else if *vec_ty != vec_ty.bytes_ty() { + byte_compact_op(op, vec_ty) } else if matches!(*self, Self::Sse4_2 | Self::Avx2) && vec_ty.len == 16 { self.handle_x86_compact_16(op, vec_ty) } else if *self == Self::Avx2 && matches!(sig, OpSig::Compress { .. }) { @@ -1838,55 +1841,43 @@ impl X86 { } fn handle_avx512_compact_op(&self, op: Op, vec_ty: &VecType) -> TokenStream { - assert!( - *self == Self::Avx512, - "compact byte intrinsics require AVX-512" - ); - assert_eq!( - vec_ty.scalar, - ScalarType::Unsigned, - "compact byte intrinsics require unsigned vectors" - ); - assert_eq!( - vec_ty.scalar_bits, 8, - "compact byte intrinsics require byte lanes" - ); - - let prefix = match vec_ty.len { - 16 => "_mm", - 32 => "_mm256", - 64 => "_mm512", + assert!(*self == Self::Avx512, "compact intrinsics require AVX-512"); + let prefix = match vec_ty.n_bits() { + 128 => "_mm", + 256 => "_mm256", + 512 => "_mm512", _ => unreachable!(), }; - let mask_ty = match vec_ty.len { - 16 => quote! { u16 }, - 32 => quote! { u32 }, - 64 => quote! { u64 }, - _ => unreachable!(), + let suffix = match (vec_ty.scalar, vec_ty.scalar_bits) { + (ScalarType::Float, 32) => "ps".to_owned(), + (ScalarType::Float, 64) => "pd".to_owned(), + (ScalarType::Int | ScalarType::Unsigned, bits) => format!("epi{bits}"), + _ => unreachable!("compact intrinsics require numeric vectors"), }; + let mask_ty = ScalarType::Unsigned.rust(avx512_mask_register_bits(vec_ty)); let mask_bits = avx512_mask_bits_expr(quote! { mask }); self.kernel_method(op, vec_ty, |token| match op.sig { OpSig::Compress { merge: false } => { - let intrinsic = format_ident!("{prefix}_maskz_compress_epi8"); + let intrinsic = format_ident!("{prefix}_maskz_compress_{suffix}"); quote! { #intrinsic(#mask_bits as #mask_ty, values.into()).simd_into(#token) } } OpSig::Compress { merge: true } => { - let intrinsic = format_ident!("{prefix}_mask_compress_epi8"); + let intrinsic = format_ident!("{prefix}_mask_compress_{suffix}"); quote! { #intrinsic(merge.into(), #mask_bits as #mask_ty, values.into()).simd_into(#token) } } OpSig::Expand { merge: false } => { - let intrinsic = format_ident!("{prefix}_maskz_expand_epi8"); + let intrinsic = format_ident!("{prefix}_maskz_expand_{suffix}"); quote! { #intrinsic(#mask_bits as #mask_ty, values.into()).simd_into(#token) } } OpSig::Expand { merge: true } => { - let intrinsic = format_ident!("{prefix}_mask_expand_epi8"); + let intrinsic = format_ident!("{prefix}_mask_expand_{suffix}"); quote! { #intrinsic(merge.into(), #mask_bits as #mask_ty, values.into()).simd_into(#token) } diff --git a/fearless_simd_gen/src/ops.rs b/fearless_simd_gen/src/ops.rs index 7da96d815..5cec40ba5 100644 --- a/fearless_simd_gen/src/ops.rs +++ b/fearless_simd_gen/src/ops.rs @@ -467,12 +467,15 @@ impl Op { let arg1 = &arg_names[1]; quote! { (#arg0: S, #arg1: Self::Element) -> Self } } - OpSig::LoadInterleaved { .. } - | OpSig::StoreInterleaved { .. } - | OpSig::Compress { .. } - | OpSig::Expand { .. } => { + OpSig::LoadInterleaved { .. } | OpSig::StoreInterleaved { .. } => { return None; } + OpSig::Compress { merge: false } | OpSig::Expand { merge: false } => { + quote! { (self, mask: Self::Mask) -> Self } + } + OpSig::Compress { merge: true } | OpSig::Expand { merge: true } => { + quote! { (self, mask: Self::Mask, merge: impl SimdInto) -> Self } + } OpSig::MaskFromBitmask | OpSig::MaskToBitmask | OpSig::MaskSet => return None, OpSig::Unary | OpSig::Cvt { .. } => { let arg0 = &arg_names[0]; @@ -729,36 +732,37 @@ const BASE_OPS: &[Op] = &[ "Dynamically select bytes from the concatenation of this vector and `rhs`.\n\n\ The `indices` operand is a same-width byte vector. For each output byte, index values within the concatenated vectors' byte length select the corresponding byte: the first vector comes first, followed by `rhs`. Out-of-range indices produce zero.", ), -]; - -const U8_ONLY_OPS: &[Op] = &[ Op::new( "compress", - OpKind::AssociatedOnly, + OpKind::BaseTraitMethod, OpSig::Compress { merge: false }, - "Compact the bytes selected by `{arg1}` into consecutive low lanes.\n\n\ - Lanes above the number of selected bytes are zero.", + "Compact the elements selected by `{arg1}` into consecutive low lanes, preserving their order.\n\n\ + Lanes above the number of selected elements are zero (positive zero for floats).\n\n\ + Elements are moved without changing their bits, including floating-point NaN payloads and signed zeros.", ), Op::new( "compress_merge", - OpKind::AssociatedOnly, + OpKind::BaseTraitMethod, OpSig::Compress { merge: true }, - "Compact the bytes selected by `{arg1}` into consecutive low lanes.\n\n\ - Lanes above the number of selected bytes retain the corresponding values from `{arg2}`.", + "Compact the elements selected by `{arg1}` into consecutive low lanes, preserving their order.\n\n\ + Lanes above the number of selected elements retain the corresponding values from `{arg2}`.\n\n\ + Elements are moved without changing their bits, including floating-point NaN payloads and signed zeros.", ), Op::new( "expand", - OpKind::AssociatedOnly, + OpKind::BaseTraitMethod, OpSig::Expand { merge: false }, - "Expand consecutive low bytes from `{arg0}` into the lanes selected by `{arg1}`.\n\n\ - Unselected lanes are zero.", + "Expand consecutive low elements from `{arg0}` into the lanes selected by `{arg1}`, preserving their order.\n\n\ + Unselected lanes are zero (positive zero for floats).\n\n\ + Elements are moved without changing their bits, including floating-point NaN payloads and signed zeros.", ), Op::new( "expand_merge", - OpKind::AssociatedOnly, + OpKind::BaseTraitMethod, OpSig::Expand { merge: true }, - "Expand consecutive low bytes from `{arg0}` into the lanes selected by `{arg1}`.\n\n\ - Unselected lanes retain the corresponding values from `{arg2}`.", + "Expand consecutive low elements from `{arg0}` into the lanes selected by `{arg1}`, preserving their order.\n\n\ + Unselected lanes retain the corresponding values from `{arg2}`.\n\n\ + Elements are moved without changing their bits, including floating-point NaN payloads and signed zeros.", ), ]; @@ -1630,10 +1634,6 @@ pub(crate) fn ops_for_type(ty: &VecType) -> Vec { } } - if ty.scalar == ScalarType::Unsigned && ty.scalar_bits == 8 { - ops.extend_from_slice(U8_ONLY_OPS); - } - if let Some(combined_ty) = ty.combine_operand() { ops.push(Op::new( "combine", @@ -2029,9 +2029,11 @@ impl OpSig { | Self::StoreInterleaved { .. } | Self::MaskFromBitmask | Self::MaskToBitmask - | Self::MaskSet - | Self::Compress { .. } - | Self::Expand { .. } => &[], + | Self::MaskSet => &[], + Self::Compress { merge: false } | Self::Expand { merge: false } => &["self", "mask"], + Self::Compress { merge: true } | Self::Expand { merge: true } => { + &["self", "mask", "merge"] + } Self::Unary | Self::Reduce { .. } | Self::RotateElements { .. } @@ -2110,9 +2112,13 @@ impl OpSig { | Self::SwizzleDynWithinBlocks | Self::SwizzleDyn | Self::SwizzleDynPrecise - | Self::Compress { .. } - | Self::Expand { .. } | Self::Slide { .. } => return None, + Self::Compress { merge: false } | Self::Expand { merge: false } => { + quote! { self, mask } + } + Self::Compress { merge: true } | Self::Expand { merge: true } => { + quote! { self, mask, merge.simd_into(self.simd) } + } }; Some(args) } diff --git a/fearless_simd_tests/tests/harness/ops/compress.rs b/fearless_simd_tests/tests/harness/ops/compress.rs index c47a5caed..460ce4a4f 100644 --- a/fearless_simd_tests/tests/harness/ops/compress.rs +++ b/fearless_simd_tests/tests/harness/ops/compress.rs @@ -4,6 +4,2086 @@ use fearless_simd::*; use fearless_simd_dev_macros::simd_test; +#[simd_test] +fn compact_u8x16(simd: S) { + let values: [u8; 16] = core::array::from_fn(|lane| { + (0x91e3_a5c7_d9fb_2d4f_u64.wrapping_mul(lane as u64 + 1)) as u8 + }); + let merge: [u8; 16] = core::array::from_fn(|lane| !values[lane]); + let values_vec = u8x16::simd_from(simd, values); + let merge_vec = u8x16::simd_from(simd, merge); + let all_lanes = u64::MAX >> (64 - 16); + let single_lanes = (0..16).map(|lane| 1_u64 << lane); + let prefixes = (1..=16).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..16).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + { + let mask = mask8x16::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 16]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 16]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..16 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!(*values_vec.compress(mask), expected_compress); + assert_eq!(*simd.compress_u8x16(values_vec, mask), expected_compress); + assert_eq!( + *values_vec.compress_merge(mask, merge_vec), + expected_compress_merge + ); + assert_eq!( + *simd.compress_merge_u8x16(values_vec, mask, merge_vec), + expected_compress_merge + ); + assert_eq!(*values_vec.expand(mask), expected_expand); + assert_eq!(*simd.expand_u8x16(values_vec, mask), expected_expand); + assert_eq!( + *values_vec.expand_merge(mask, merge_vec), + expected_expand_merge + ); + assert_eq!( + *simd.expand_merge_u8x16(values_vec, mask, merge_vec), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_u8x32(simd: S) { + let values: [u8; 32] = core::array::from_fn(|lane| { + (0x91e3_a5c7_d9fb_2d4f_u64.wrapping_mul(lane as u64 + 1)) as u8 + }); + let merge: [u8; 32] = core::array::from_fn(|lane| !values[lane]); + let values_vec = u8x32::simd_from(simd, values); + let merge_vec = u8x32::simd_from(simd, merge); + let all_lanes = u64::MAX >> (64 - 32); + let single_lanes = (0..32).map(|lane| 1_u64 << lane); + let prefixes = (1..=32).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..32).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + { + let mask = mask8x32::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 32]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 32]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..32 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!(*values_vec.compress(mask), expected_compress); + assert_eq!(*simd.compress_u8x32(values_vec, mask), expected_compress); + assert_eq!( + *values_vec.compress_merge(mask, merge_vec), + expected_compress_merge + ); + assert_eq!( + *simd.compress_merge_u8x32(values_vec, mask, merge_vec), + expected_compress_merge + ); + assert_eq!(*values_vec.expand(mask), expected_expand); + assert_eq!(*simd.expand_u8x32(values_vec, mask), expected_expand); + assert_eq!( + *values_vec.expand_merge(mask, merge_vec), + expected_expand_merge + ); + assert_eq!( + *simd.expand_merge_u8x32(values_vec, mask, merge_vec), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_u8x64(simd: S) { + let values: [u8; 64] = core::array::from_fn(|lane| { + (0x91e3_a5c7_d9fb_2d4f_u64.wrapping_mul(lane as u64 + 1)) as u8 + }); + let merge: [u8; 64] = core::array::from_fn(|lane| !values[lane]); + let values_vec = u8x64::simd_from(simd, values); + let merge_vec = u8x64::simd_from(simd, merge); + let all_lanes = u64::MAX; + let single_lanes = (0..64).map(|lane| 1_u64 << lane); + let prefixes = (1..=64).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..64).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + { + let mask = mask8x64::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 64]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 64]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..64 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!(*values_vec.compress(mask), expected_compress); + assert_eq!(*simd.compress_u8x64(values_vec, mask), expected_compress); + assert_eq!( + *values_vec.compress_merge(mask, merge_vec), + expected_compress_merge + ); + assert_eq!( + *simd.compress_merge_u8x64(values_vec, mask, merge_vec), + expected_compress_merge + ); + assert_eq!(*values_vec.expand(mask), expected_expand); + assert_eq!(*simd.expand_u8x64(values_vec, mask), expected_expand); + assert_eq!( + *values_vec.expand_merge(mask, merge_vec), + expected_expand_merge + ); + assert_eq!( + *simd.expand_merge_u8x64(values_vec, mask, merge_vec), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_i8x16(simd: S) { + let values: [i8; 16] = core::array::from_fn(|lane| { + (0x91e3_a5c7_d9fb_2d4f_u64.wrapping_mul(lane as u64 + 1)) as i8 + }); + let merge: [i8; 16] = core::array::from_fn(|lane| !values[lane]); + let values_vec = i8x16::simd_from(simd, values); + let merge_vec = i8x16::simd_from(simd, merge); + let all_lanes = u64::MAX >> (64 - 16); + let single_lanes = (0..16).map(|lane| 1_u64 << lane); + let prefixes = (1..=16).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..16).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + { + let mask = mask8x16::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 16]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 16]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..16 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!(*values_vec.compress(mask), expected_compress); + assert_eq!(*simd.compress_i8x16(values_vec, mask), expected_compress); + assert_eq!( + *values_vec.compress_merge(mask, merge_vec), + expected_compress_merge + ); + assert_eq!( + *simd.compress_merge_i8x16(values_vec, mask, merge_vec), + expected_compress_merge + ); + assert_eq!(*values_vec.expand(mask), expected_expand); + assert_eq!(*simd.expand_i8x16(values_vec, mask), expected_expand); + assert_eq!( + *values_vec.expand_merge(mask, merge_vec), + expected_expand_merge + ); + assert_eq!( + *simd.expand_merge_i8x16(values_vec, mask, merge_vec), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_i8x32(simd: S) { + let values: [i8; 32] = core::array::from_fn(|lane| { + (0x91e3_a5c7_d9fb_2d4f_u64.wrapping_mul(lane as u64 + 1)) as i8 + }); + let merge: [i8; 32] = core::array::from_fn(|lane| !values[lane]); + let values_vec = i8x32::simd_from(simd, values); + let merge_vec = i8x32::simd_from(simd, merge); + let all_lanes = u64::MAX >> (64 - 32); + let single_lanes = (0..32).map(|lane| 1_u64 << lane); + let prefixes = (1..=32).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..32).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + { + let mask = mask8x32::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 32]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 32]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..32 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!(*values_vec.compress(mask), expected_compress); + assert_eq!(*simd.compress_i8x32(values_vec, mask), expected_compress); + assert_eq!( + *values_vec.compress_merge(mask, merge_vec), + expected_compress_merge + ); + assert_eq!( + *simd.compress_merge_i8x32(values_vec, mask, merge_vec), + expected_compress_merge + ); + assert_eq!(*values_vec.expand(mask), expected_expand); + assert_eq!(*simd.expand_i8x32(values_vec, mask), expected_expand); + assert_eq!( + *values_vec.expand_merge(mask, merge_vec), + expected_expand_merge + ); + assert_eq!( + *simd.expand_merge_i8x32(values_vec, mask, merge_vec), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_i8x64(simd: S) { + let values: [i8; 64] = core::array::from_fn(|lane| { + (0x91e3_a5c7_d9fb_2d4f_u64.wrapping_mul(lane as u64 + 1)) as i8 + }); + let merge: [i8; 64] = core::array::from_fn(|lane| !values[lane]); + let values_vec = i8x64::simd_from(simd, values); + let merge_vec = i8x64::simd_from(simd, merge); + let all_lanes = u64::MAX; + let single_lanes = (0..64).map(|lane| 1_u64 << lane); + let prefixes = (1..=64).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..64).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + { + let mask = mask8x64::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 64]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 64]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..64 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!(*values_vec.compress(mask), expected_compress); + assert_eq!(*simd.compress_i8x64(values_vec, mask), expected_compress); + assert_eq!( + *values_vec.compress_merge(mask, merge_vec), + expected_compress_merge + ); + assert_eq!( + *simd.compress_merge_i8x64(values_vec, mask, merge_vec), + expected_compress_merge + ); + assert_eq!(*values_vec.expand(mask), expected_expand); + assert_eq!(*simd.expand_i8x64(values_vec, mask), expected_expand); + assert_eq!( + *values_vec.expand_merge(mask, merge_vec), + expected_expand_merge + ); + assert_eq!( + *simd.expand_merge_i8x64(values_vec, mask, merge_vec), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_u16x8(simd: S) { + let values: [u16; 8] = core::array::from_fn(|lane| { + (0x91e3_a5c7_d9fb_2d4f_u64.wrapping_mul(lane as u64 + 1)) as u16 + }); + let merge: [u16; 8] = core::array::from_fn(|lane| !values[lane]); + let values_vec = u16x8::simd_from(simd, values); + let merge_vec = u16x8::simd_from(simd, merge); + let all_lanes = u64::MAX >> (64 - 8); + let single_lanes = (0..8).map(|lane| 1_u64 << lane); + let prefixes = (1..=8).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..8).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + { + let mask = mask16x8::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 8]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 8]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..8 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!(*values_vec.compress(mask), expected_compress); + assert_eq!(*simd.compress_u16x8(values_vec, mask), expected_compress); + assert_eq!( + *values_vec.compress_merge(mask, merge_vec), + expected_compress_merge + ); + assert_eq!( + *simd.compress_merge_u16x8(values_vec, mask, merge_vec), + expected_compress_merge + ); + assert_eq!(*values_vec.expand(mask), expected_expand); + assert_eq!(*simd.expand_u16x8(values_vec, mask), expected_expand); + assert_eq!( + *values_vec.expand_merge(mask, merge_vec), + expected_expand_merge + ); + assert_eq!( + *simd.expand_merge_u16x8(values_vec, mask, merge_vec), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_u16x16(simd: S) { + let values: [u16; 16] = core::array::from_fn(|lane| { + (0x91e3_a5c7_d9fb_2d4f_u64.wrapping_mul(lane as u64 + 1)) as u16 + }); + let merge: [u16; 16] = core::array::from_fn(|lane| !values[lane]); + let values_vec = u16x16::simd_from(simd, values); + let merge_vec = u16x16::simd_from(simd, merge); + let all_lanes = u64::MAX >> (64 - 16); + let single_lanes = (0..16).map(|lane| 1_u64 << lane); + let prefixes = (1..=16).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..16).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + { + let mask = mask16x16::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 16]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 16]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..16 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!(*values_vec.compress(mask), expected_compress); + assert_eq!(*simd.compress_u16x16(values_vec, mask), expected_compress); + assert_eq!( + *values_vec.compress_merge(mask, merge_vec), + expected_compress_merge + ); + assert_eq!( + *simd.compress_merge_u16x16(values_vec, mask, merge_vec), + expected_compress_merge + ); + assert_eq!(*values_vec.expand(mask), expected_expand); + assert_eq!(*simd.expand_u16x16(values_vec, mask), expected_expand); + assert_eq!( + *values_vec.expand_merge(mask, merge_vec), + expected_expand_merge + ); + assert_eq!( + *simd.expand_merge_u16x16(values_vec, mask, merge_vec), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_u16x32(simd: S) { + let values: [u16; 32] = core::array::from_fn(|lane| { + (0x91e3_a5c7_d9fb_2d4f_u64.wrapping_mul(lane as u64 + 1)) as u16 + }); + let merge: [u16; 32] = core::array::from_fn(|lane| !values[lane]); + let values_vec = u16x32::simd_from(simd, values); + let merge_vec = u16x32::simd_from(simd, merge); + let all_lanes = u64::MAX >> (64 - 32); + let single_lanes = (0..32).map(|lane| 1_u64 << lane); + let prefixes = (1..=32).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..32).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + { + let mask = mask16x32::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 32]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 32]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..32 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!(*values_vec.compress(mask), expected_compress); + assert_eq!(*simd.compress_u16x32(values_vec, mask), expected_compress); + assert_eq!( + *values_vec.compress_merge(mask, merge_vec), + expected_compress_merge + ); + assert_eq!( + *simd.compress_merge_u16x32(values_vec, mask, merge_vec), + expected_compress_merge + ); + assert_eq!(*values_vec.expand(mask), expected_expand); + assert_eq!(*simd.expand_u16x32(values_vec, mask), expected_expand); + assert_eq!( + *values_vec.expand_merge(mask, merge_vec), + expected_expand_merge + ); + assert_eq!( + *simd.expand_merge_u16x32(values_vec, mask, merge_vec), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_i16x8(simd: S) { + let values: [i16; 8] = core::array::from_fn(|lane| { + (0x91e3_a5c7_d9fb_2d4f_u64.wrapping_mul(lane as u64 + 1)) as i16 + }); + let merge: [i16; 8] = core::array::from_fn(|lane| !values[lane]); + let values_vec = i16x8::simd_from(simd, values); + let merge_vec = i16x8::simd_from(simd, merge); + let all_lanes = u64::MAX >> (64 - 8); + let single_lanes = (0..8).map(|lane| 1_u64 << lane); + let prefixes = (1..=8).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..8).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + { + let mask = mask16x8::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 8]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 8]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..8 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!(*values_vec.compress(mask), expected_compress); + assert_eq!(*simd.compress_i16x8(values_vec, mask), expected_compress); + assert_eq!( + *values_vec.compress_merge(mask, merge_vec), + expected_compress_merge + ); + assert_eq!( + *simd.compress_merge_i16x8(values_vec, mask, merge_vec), + expected_compress_merge + ); + assert_eq!(*values_vec.expand(mask), expected_expand); + assert_eq!(*simd.expand_i16x8(values_vec, mask), expected_expand); + assert_eq!( + *values_vec.expand_merge(mask, merge_vec), + expected_expand_merge + ); + assert_eq!( + *simd.expand_merge_i16x8(values_vec, mask, merge_vec), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_i16x16(simd: S) { + let values: [i16; 16] = core::array::from_fn(|lane| { + (0x91e3_a5c7_d9fb_2d4f_u64.wrapping_mul(lane as u64 + 1)) as i16 + }); + let merge: [i16; 16] = core::array::from_fn(|lane| !values[lane]); + let values_vec = i16x16::simd_from(simd, values); + let merge_vec = i16x16::simd_from(simd, merge); + let all_lanes = u64::MAX >> (64 - 16); + let single_lanes = (0..16).map(|lane| 1_u64 << lane); + let prefixes = (1..=16).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..16).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + { + let mask = mask16x16::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 16]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 16]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..16 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!(*values_vec.compress(mask), expected_compress); + assert_eq!(*simd.compress_i16x16(values_vec, mask), expected_compress); + assert_eq!( + *values_vec.compress_merge(mask, merge_vec), + expected_compress_merge + ); + assert_eq!( + *simd.compress_merge_i16x16(values_vec, mask, merge_vec), + expected_compress_merge + ); + assert_eq!(*values_vec.expand(mask), expected_expand); + assert_eq!(*simd.expand_i16x16(values_vec, mask), expected_expand); + assert_eq!( + *values_vec.expand_merge(mask, merge_vec), + expected_expand_merge + ); + assert_eq!( + *simd.expand_merge_i16x16(values_vec, mask, merge_vec), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_i16x32(simd: S) { + let values: [i16; 32] = core::array::from_fn(|lane| { + (0x91e3_a5c7_d9fb_2d4f_u64.wrapping_mul(lane as u64 + 1)) as i16 + }); + let merge: [i16; 32] = core::array::from_fn(|lane| !values[lane]); + let values_vec = i16x32::simd_from(simd, values); + let merge_vec = i16x32::simd_from(simd, merge); + let all_lanes = u64::MAX >> (64 - 32); + let single_lanes = (0..32).map(|lane| 1_u64 << lane); + let prefixes = (1..=32).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..32).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + { + let mask = mask16x32::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 32]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 32]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..32 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!(*values_vec.compress(mask), expected_compress); + assert_eq!(*simd.compress_i16x32(values_vec, mask), expected_compress); + assert_eq!( + *values_vec.compress_merge(mask, merge_vec), + expected_compress_merge + ); + assert_eq!( + *simd.compress_merge_i16x32(values_vec, mask, merge_vec), + expected_compress_merge + ); + assert_eq!(*values_vec.expand(mask), expected_expand); + assert_eq!(*simd.expand_i16x32(values_vec, mask), expected_expand); + assert_eq!( + *values_vec.expand_merge(mask, merge_vec), + expected_expand_merge + ); + assert_eq!( + *simd.expand_merge_i16x32(values_vec, mask, merge_vec), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_u32x4(simd: S) { + let values: [u32; 4] = core::array::from_fn(|lane| { + (0x91e3_a5c7_d9fb_2d4f_u64.wrapping_mul(lane as u64 + 1)) as u32 + }); + let merge: [u32; 4] = core::array::from_fn(|lane| !values[lane]); + let values_vec = u32x4::simd_from(simd, values); + let merge_vec = u32x4::simd_from(simd, merge); + let all_lanes = u64::MAX >> (64 - 4); + let single_lanes = (0..4).map(|lane| 1_u64 << lane); + let prefixes = (1..=4).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..4).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + { + let mask = mask32x4::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 4]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 4]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..4 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!(*values_vec.compress(mask), expected_compress); + assert_eq!(*simd.compress_u32x4(values_vec, mask), expected_compress); + assert_eq!( + *values_vec.compress_merge(mask, merge_vec), + expected_compress_merge + ); + assert_eq!( + *simd.compress_merge_u32x4(values_vec, mask, merge_vec), + expected_compress_merge + ); + assert_eq!(*values_vec.expand(mask), expected_expand); + assert_eq!(*simd.expand_u32x4(values_vec, mask), expected_expand); + assert_eq!( + *values_vec.expand_merge(mask, merge_vec), + expected_expand_merge + ); + assert_eq!( + *simd.expand_merge_u32x4(values_vec, mask, merge_vec), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_u32x8(simd: S) { + let values: [u32; 8] = core::array::from_fn(|lane| { + (0x91e3_a5c7_d9fb_2d4f_u64.wrapping_mul(lane as u64 + 1)) as u32 + }); + let merge: [u32; 8] = core::array::from_fn(|lane| !values[lane]); + let values_vec = u32x8::simd_from(simd, values); + let merge_vec = u32x8::simd_from(simd, merge); + let all_lanes = u64::MAX >> (64 - 8); + let single_lanes = (0..8).map(|lane| 1_u64 << lane); + let prefixes = (1..=8).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..8).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + { + let mask = mask32x8::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 8]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 8]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..8 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!(*values_vec.compress(mask), expected_compress); + assert_eq!(*simd.compress_u32x8(values_vec, mask), expected_compress); + assert_eq!( + *values_vec.compress_merge(mask, merge_vec), + expected_compress_merge + ); + assert_eq!( + *simd.compress_merge_u32x8(values_vec, mask, merge_vec), + expected_compress_merge + ); + assert_eq!(*values_vec.expand(mask), expected_expand); + assert_eq!(*simd.expand_u32x8(values_vec, mask), expected_expand); + assert_eq!( + *values_vec.expand_merge(mask, merge_vec), + expected_expand_merge + ); + assert_eq!( + *simd.expand_merge_u32x8(values_vec, mask, merge_vec), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_u32x16(simd: S) { + let values: [u32; 16] = core::array::from_fn(|lane| { + (0x91e3_a5c7_d9fb_2d4f_u64.wrapping_mul(lane as u64 + 1)) as u32 + }); + let merge: [u32; 16] = core::array::from_fn(|lane| !values[lane]); + let values_vec = u32x16::simd_from(simd, values); + let merge_vec = u32x16::simd_from(simd, merge); + let all_lanes = u64::MAX >> (64 - 16); + let single_lanes = (0..16).map(|lane| 1_u64 << lane); + let prefixes = (1..=16).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..16).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + { + let mask = mask32x16::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 16]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 16]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..16 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!(*values_vec.compress(mask), expected_compress); + assert_eq!(*simd.compress_u32x16(values_vec, mask), expected_compress); + assert_eq!( + *values_vec.compress_merge(mask, merge_vec), + expected_compress_merge + ); + assert_eq!( + *simd.compress_merge_u32x16(values_vec, mask, merge_vec), + expected_compress_merge + ); + assert_eq!(*values_vec.expand(mask), expected_expand); + assert_eq!(*simd.expand_u32x16(values_vec, mask), expected_expand); + assert_eq!( + *values_vec.expand_merge(mask, merge_vec), + expected_expand_merge + ); + assert_eq!( + *simd.expand_merge_u32x16(values_vec, mask, merge_vec), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_i32x4(simd: S) { + let values: [i32; 4] = core::array::from_fn(|lane| { + (0x91e3_a5c7_d9fb_2d4f_u64.wrapping_mul(lane as u64 + 1)) as i32 + }); + let merge: [i32; 4] = core::array::from_fn(|lane| !values[lane]); + let values_vec = i32x4::simd_from(simd, values); + let merge_vec = i32x4::simd_from(simd, merge); + let all_lanes = u64::MAX >> (64 - 4); + let single_lanes = (0..4).map(|lane| 1_u64 << lane); + let prefixes = (1..=4).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..4).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + { + let mask = mask32x4::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 4]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 4]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..4 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!(*values_vec.compress(mask), expected_compress); + assert_eq!(*simd.compress_i32x4(values_vec, mask), expected_compress); + assert_eq!( + *values_vec.compress_merge(mask, merge_vec), + expected_compress_merge + ); + assert_eq!( + *simd.compress_merge_i32x4(values_vec, mask, merge_vec), + expected_compress_merge + ); + assert_eq!(*values_vec.expand(mask), expected_expand); + assert_eq!(*simd.expand_i32x4(values_vec, mask), expected_expand); + assert_eq!( + *values_vec.expand_merge(mask, merge_vec), + expected_expand_merge + ); + assert_eq!( + *simd.expand_merge_i32x4(values_vec, mask, merge_vec), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_i32x8(simd: S) { + let values: [i32; 8] = core::array::from_fn(|lane| { + (0x91e3_a5c7_d9fb_2d4f_u64.wrapping_mul(lane as u64 + 1)) as i32 + }); + let merge: [i32; 8] = core::array::from_fn(|lane| !values[lane]); + let values_vec = i32x8::simd_from(simd, values); + let merge_vec = i32x8::simd_from(simd, merge); + let all_lanes = u64::MAX >> (64 - 8); + let single_lanes = (0..8).map(|lane| 1_u64 << lane); + let prefixes = (1..=8).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..8).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + { + let mask = mask32x8::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 8]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 8]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..8 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!(*values_vec.compress(mask), expected_compress); + assert_eq!(*simd.compress_i32x8(values_vec, mask), expected_compress); + assert_eq!( + *values_vec.compress_merge(mask, merge_vec), + expected_compress_merge + ); + assert_eq!( + *simd.compress_merge_i32x8(values_vec, mask, merge_vec), + expected_compress_merge + ); + assert_eq!(*values_vec.expand(mask), expected_expand); + assert_eq!(*simd.expand_i32x8(values_vec, mask), expected_expand); + assert_eq!( + *values_vec.expand_merge(mask, merge_vec), + expected_expand_merge + ); + assert_eq!( + *simd.expand_merge_i32x8(values_vec, mask, merge_vec), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_i32x16(simd: S) { + let values: [i32; 16] = core::array::from_fn(|lane| { + (0x91e3_a5c7_d9fb_2d4f_u64.wrapping_mul(lane as u64 + 1)) as i32 + }); + let merge: [i32; 16] = core::array::from_fn(|lane| !values[lane]); + let values_vec = i32x16::simd_from(simd, values); + let merge_vec = i32x16::simd_from(simd, merge); + let all_lanes = u64::MAX >> (64 - 16); + let single_lanes = (0..16).map(|lane| 1_u64 << lane); + let prefixes = (1..=16).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..16).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + { + let mask = mask32x16::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 16]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 16]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..16 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!(*values_vec.compress(mask), expected_compress); + assert_eq!(*simd.compress_i32x16(values_vec, mask), expected_compress); + assert_eq!( + *values_vec.compress_merge(mask, merge_vec), + expected_compress_merge + ); + assert_eq!( + *simd.compress_merge_i32x16(values_vec, mask, merge_vec), + expected_compress_merge + ); + assert_eq!(*values_vec.expand(mask), expected_expand); + assert_eq!(*simd.expand_i32x16(values_vec, mask), expected_expand); + assert_eq!( + *values_vec.expand_merge(mask, merge_vec), + expected_expand_merge + ); + assert_eq!( + *simd.expand_merge_i32x16(values_vec, mask, merge_vec), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_u64x2(simd: S) { + let values: [u64; 2] = + core::array::from_fn(|lane| 0x91e3_a5c7_d9fb_2d4f_u64.wrapping_mul(lane as u64 + 1)); + let merge: [u64; 2] = core::array::from_fn(|lane| !values[lane]); + let values_vec = u64x2::simd_from(simd, values); + let merge_vec = u64x2::simd_from(simd, merge); + let all_lanes = u64::MAX >> (64 - 2); + let single_lanes = (0..2).map(|lane| 1_u64 << lane); + let prefixes = (1..=2).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..2).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + { + let mask = mask64x2::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 2]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 2]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..2 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!(*values_vec.compress(mask), expected_compress); + assert_eq!(*simd.compress_u64x2(values_vec, mask), expected_compress); + assert_eq!( + *values_vec.compress_merge(mask, merge_vec), + expected_compress_merge + ); + assert_eq!( + *simd.compress_merge_u64x2(values_vec, mask, merge_vec), + expected_compress_merge + ); + assert_eq!(*values_vec.expand(mask), expected_expand); + assert_eq!(*simd.expand_u64x2(values_vec, mask), expected_expand); + assert_eq!( + *values_vec.expand_merge(mask, merge_vec), + expected_expand_merge + ); + assert_eq!( + *simd.expand_merge_u64x2(values_vec, mask, merge_vec), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_u64x4(simd: S) { + let values: [u64; 4] = + core::array::from_fn(|lane| 0x91e3_a5c7_d9fb_2d4f_u64.wrapping_mul(lane as u64 + 1)); + let merge: [u64; 4] = core::array::from_fn(|lane| !values[lane]); + let values_vec = u64x4::simd_from(simd, values); + let merge_vec = u64x4::simd_from(simd, merge); + let all_lanes = u64::MAX >> (64 - 4); + let single_lanes = (0..4).map(|lane| 1_u64 << lane); + let prefixes = (1..=4).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..4).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + { + let mask = mask64x4::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 4]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 4]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..4 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!(*values_vec.compress(mask), expected_compress); + assert_eq!(*simd.compress_u64x4(values_vec, mask), expected_compress); + assert_eq!( + *values_vec.compress_merge(mask, merge_vec), + expected_compress_merge + ); + assert_eq!( + *simd.compress_merge_u64x4(values_vec, mask, merge_vec), + expected_compress_merge + ); + assert_eq!(*values_vec.expand(mask), expected_expand); + assert_eq!(*simd.expand_u64x4(values_vec, mask), expected_expand); + assert_eq!( + *values_vec.expand_merge(mask, merge_vec), + expected_expand_merge + ); + assert_eq!( + *simd.expand_merge_u64x4(values_vec, mask, merge_vec), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_u64x8(simd: S) { + let values: [u64; 8] = + core::array::from_fn(|lane| 0x91e3_a5c7_d9fb_2d4f_u64.wrapping_mul(lane as u64 + 1)); + let merge: [u64; 8] = core::array::from_fn(|lane| !values[lane]); + let values_vec = u64x8::simd_from(simd, values); + let merge_vec = u64x8::simd_from(simd, merge); + let all_lanes = u64::MAX >> (64 - 8); + let single_lanes = (0..8).map(|lane| 1_u64 << lane); + let prefixes = (1..=8).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..8).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + { + let mask = mask64x8::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 8]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 8]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..8 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!(*values_vec.compress(mask), expected_compress); + assert_eq!(*simd.compress_u64x8(values_vec, mask), expected_compress); + assert_eq!( + *values_vec.compress_merge(mask, merge_vec), + expected_compress_merge + ); + assert_eq!( + *simd.compress_merge_u64x8(values_vec, mask, merge_vec), + expected_compress_merge + ); + assert_eq!(*values_vec.expand(mask), expected_expand); + assert_eq!(*simd.expand_u64x8(values_vec, mask), expected_expand); + assert_eq!( + *values_vec.expand_merge(mask, merge_vec), + expected_expand_merge + ); + assert_eq!( + *simd.expand_merge_u64x8(values_vec, mask, merge_vec), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_i64x2(simd: S) { + let values: [i64; 2] = core::array::from_fn(|lane| { + (0x91e3_a5c7_d9fb_2d4f_u64.wrapping_mul(lane as u64 + 1)) as i64 + }); + let merge: [i64; 2] = core::array::from_fn(|lane| !values[lane]); + let values_vec = i64x2::simd_from(simd, values); + let merge_vec = i64x2::simd_from(simd, merge); + let all_lanes = u64::MAX >> (64 - 2); + let single_lanes = (0..2).map(|lane| 1_u64 << lane); + let prefixes = (1..=2).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..2).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + { + let mask = mask64x2::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 2]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 2]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..2 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!(*values_vec.compress(mask), expected_compress); + assert_eq!(*simd.compress_i64x2(values_vec, mask), expected_compress); + assert_eq!( + *values_vec.compress_merge(mask, merge_vec), + expected_compress_merge + ); + assert_eq!( + *simd.compress_merge_i64x2(values_vec, mask, merge_vec), + expected_compress_merge + ); + assert_eq!(*values_vec.expand(mask), expected_expand); + assert_eq!(*simd.expand_i64x2(values_vec, mask), expected_expand); + assert_eq!( + *values_vec.expand_merge(mask, merge_vec), + expected_expand_merge + ); + assert_eq!( + *simd.expand_merge_i64x2(values_vec, mask, merge_vec), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_i64x4(simd: S) { + let values: [i64; 4] = core::array::from_fn(|lane| { + (0x91e3_a5c7_d9fb_2d4f_u64.wrapping_mul(lane as u64 + 1)) as i64 + }); + let merge: [i64; 4] = core::array::from_fn(|lane| !values[lane]); + let values_vec = i64x4::simd_from(simd, values); + let merge_vec = i64x4::simd_from(simd, merge); + let all_lanes = u64::MAX >> (64 - 4); + let single_lanes = (0..4).map(|lane| 1_u64 << lane); + let prefixes = (1..=4).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..4).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + { + let mask = mask64x4::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 4]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 4]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..4 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!(*values_vec.compress(mask), expected_compress); + assert_eq!(*simd.compress_i64x4(values_vec, mask), expected_compress); + assert_eq!( + *values_vec.compress_merge(mask, merge_vec), + expected_compress_merge + ); + assert_eq!( + *simd.compress_merge_i64x4(values_vec, mask, merge_vec), + expected_compress_merge + ); + assert_eq!(*values_vec.expand(mask), expected_expand); + assert_eq!(*simd.expand_i64x4(values_vec, mask), expected_expand); + assert_eq!( + *values_vec.expand_merge(mask, merge_vec), + expected_expand_merge + ); + assert_eq!( + *simd.expand_merge_i64x4(values_vec, mask, merge_vec), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_i64x8(simd: S) { + let values: [i64; 8] = core::array::from_fn(|lane| { + (0x91e3_a5c7_d9fb_2d4f_u64.wrapping_mul(lane as u64 + 1)) as i64 + }); + let merge: [i64; 8] = core::array::from_fn(|lane| !values[lane]); + let values_vec = i64x8::simd_from(simd, values); + let merge_vec = i64x8::simd_from(simd, merge); + let all_lanes = u64::MAX >> (64 - 8); + let single_lanes = (0..8).map(|lane| 1_u64 << lane); + let prefixes = (1..=8).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..8).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + { + let mask = mask64x8::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 8]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 8]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..8 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!(*values_vec.compress(mask), expected_compress); + assert_eq!(*simd.compress_i64x8(values_vec, mask), expected_compress); + assert_eq!( + *values_vec.compress_merge(mask, merge_vec), + expected_compress_merge + ); + assert_eq!( + *simd.compress_merge_i64x8(values_vec, mask, merge_vec), + expected_compress_merge + ); + assert_eq!(*values_vec.expand(mask), expected_expand); + assert_eq!(*simd.expand_i64x8(values_vec, mask), expected_expand); + assert_eq!( + *values_vec.expand_merge(mask, merge_vec), + expected_expand_merge + ); + assert_eq!( + *simd.expand_merge_i64x8(values_vec, mask, merge_vec), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_f32x4(simd: S) { + let patterns: [u32; 8] = [ + 0x8000_0000, + 0x7fc1_2345, + 0x7f80_0000, + 0x7f80_0001, + 0, + 0xff80_0000, + 0xffc5_4321, + 0x3f80_0000, + ]; + let values: [u32; 4] = core::array::from_fn(|lane| patterns[lane % 8]); + let merge: [u32; 4] = core::array::from_fn(|lane| patterns[(lane + 3) % 8]); + let values_vec = f32x4::simd_from(simd, values.map(f32::from_bits)); + let merge_vec = f32x4::simd_from(simd, merge.map(f32::from_bits)); + let all_lanes = u64::MAX >> (64 - 4); + let single_lanes = (0..4).map(|lane| 1_u64 << lane); + let prefixes = (1..=4).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..4).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + { + let mask = mask32x4::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 4]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 4]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..4 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!( + values_vec.compress(mask).to_array().map(f32::to_bits), + expected_compress + ); + assert_eq!( + simd.compress_f32x4(values_vec, mask) + .to_array() + .map(f32::to_bits), + expected_compress + ); + assert_eq!( + values_vec + .compress_merge(mask, merge_vec) + .to_array() + .map(f32::to_bits), + expected_compress_merge + ); + assert_eq!( + simd.compress_merge_f32x4(values_vec, mask, merge_vec) + .to_array() + .map(f32::to_bits), + expected_compress_merge + ); + assert_eq!( + values_vec.expand(mask).to_array().map(f32::to_bits), + expected_expand + ); + assert_eq!( + simd.expand_f32x4(values_vec, mask) + .to_array() + .map(f32::to_bits), + expected_expand + ); + assert_eq!( + values_vec + .expand_merge(mask, merge_vec) + .to_array() + .map(f32::to_bits), + expected_expand_merge + ); + assert_eq!( + simd.expand_merge_f32x4(values_vec, mask, merge_vec) + .to_array() + .map(f32::to_bits), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_f32x8(simd: S) { + let patterns: [u32; 8] = [ + 0x8000_0000, + 0x7fc1_2345, + 0x7f80_0000, + 0x7f80_0001, + 0, + 0xff80_0000, + 0xffc5_4321, + 0x3f80_0000, + ]; + let values: [u32; 8] = core::array::from_fn(|lane| patterns[lane % 8]); + let merge: [u32; 8] = core::array::from_fn(|lane| patterns[(lane + 3) % 8]); + let values_vec = f32x8::simd_from(simd, values.map(f32::from_bits)); + let merge_vec = f32x8::simd_from(simd, merge.map(f32::from_bits)); + let all_lanes = u64::MAX >> (64 - 8); + let single_lanes = (0..8).map(|lane| 1_u64 << lane); + let prefixes = (1..=8).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..8).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + { + let mask = mask32x8::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 8]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 8]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..8 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!( + values_vec.compress(mask).to_array().map(f32::to_bits), + expected_compress + ); + assert_eq!( + simd.compress_f32x8(values_vec, mask) + .to_array() + .map(f32::to_bits), + expected_compress + ); + assert_eq!( + values_vec + .compress_merge(mask, merge_vec) + .to_array() + .map(f32::to_bits), + expected_compress_merge + ); + assert_eq!( + simd.compress_merge_f32x8(values_vec, mask, merge_vec) + .to_array() + .map(f32::to_bits), + expected_compress_merge + ); + assert_eq!( + values_vec.expand(mask).to_array().map(f32::to_bits), + expected_expand + ); + assert_eq!( + simd.expand_f32x8(values_vec, mask) + .to_array() + .map(f32::to_bits), + expected_expand + ); + assert_eq!( + values_vec + .expand_merge(mask, merge_vec) + .to_array() + .map(f32::to_bits), + expected_expand_merge + ); + assert_eq!( + simd.expand_merge_f32x8(values_vec, mask, merge_vec) + .to_array() + .map(f32::to_bits), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_f32x16(simd: S) { + let patterns: [u32; 8] = [ + 0x8000_0000, + 0x7fc1_2345, + 0x7f80_0000, + 0x7f80_0001, + 0, + 0xff80_0000, + 0xffc5_4321, + 0x3f80_0000, + ]; + let values: [u32; 16] = core::array::from_fn(|lane| patterns[lane % 8]); + let merge: [u32; 16] = core::array::from_fn(|lane| patterns[(lane + 3) % 8]); + let values_vec = f32x16::simd_from(simd, values.map(f32::from_bits)); + let merge_vec = f32x16::simd_from(simd, merge.map(f32::from_bits)); + let all_lanes = u64::MAX >> (64 - 16); + let single_lanes = (0..16).map(|lane| 1_u64 << lane); + let prefixes = (1..=16).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..16).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + { + let mask = mask32x16::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 16]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 16]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..16 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!( + values_vec.compress(mask).to_array().map(f32::to_bits), + expected_compress + ); + assert_eq!( + simd.compress_f32x16(values_vec, mask) + .to_array() + .map(f32::to_bits), + expected_compress + ); + assert_eq!( + values_vec + .compress_merge(mask, merge_vec) + .to_array() + .map(f32::to_bits), + expected_compress_merge + ); + assert_eq!( + simd.compress_merge_f32x16(values_vec, mask, merge_vec) + .to_array() + .map(f32::to_bits), + expected_compress_merge + ); + assert_eq!( + values_vec.expand(mask).to_array().map(f32::to_bits), + expected_expand + ); + assert_eq!( + simd.expand_f32x16(values_vec, mask) + .to_array() + .map(f32::to_bits), + expected_expand + ); + assert_eq!( + values_vec + .expand_merge(mask, merge_vec) + .to_array() + .map(f32::to_bits), + expected_expand_merge + ); + assert_eq!( + simd.expand_merge_f32x16(values_vec, mask, merge_vec) + .to_array() + .map(f32::to_bits), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_f64x2(simd: S) { + let patterns: [u64; 8] = [ + 0x8000_0000_0000_0000, + 0x7ff8_1234_5678_9abc, + 0x7ff0_0000_0000_0000, + 0x7ff0_0000_0000_0001, + 0, + 0xfff0_0000_0000_0000, + 0xfff8_abcd_1234_5678, + 0x3ff0_0000_0000_0000, + ]; + let values: [u64; 2] = core::array::from_fn(|lane| patterns[lane % 8]); + let merge: [u64; 2] = core::array::from_fn(|lane| patterns[(lane + 3) % 8]); + let values_vec = f64x2::simd_from(simd, values.map(f64::from_bits)); + let merge_vec = f64x2::simd_from(simd, merge.map(f64::from_bits)); + let all_lanes = u64::MAX >> (64 - 2); + let single_lanes = (0..2).map(|lane| 1_u64 << lane); + let prefixes = (1..=2).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..2).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + { + let mask = mask64x2::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 2]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 2]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..2 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!( + values_vec.compress(mask).to_array().map(f64::to_bits), + expected_compress + ); + assert_eq!( + simd.compress_f64x2(values_vec, mask) + .to_array() + .map(f64::to_bits), + expected_compress + ); + assert_eq!( + values_vec + .compress_merge(mask, merge_vec) + .to_array() + .map(f64::to_bits), + expected_compress_merge + ); + assert_eq!( + simd.compress_merge_f64x2(values_vec, mask, merge_vec) + .to_array() + .map(f64::to_bits), + expected_compress_merge + ); + assert_eq!( + values_vec.expand(mask).to_array().map(f64::to_bits), + expected_expand + ); + assert_eq!( + simd.expand_f64x2(values_vec, mask) + .to_array() + .map(f64::to_bits), + expected_expand + ); + assert_eq!( + values_vec + .expand_merge(mask, merge_vec) + .to_array() + .map(f64::to_bits), + expected_expand_merge + ); + assert_eq!( + simd.expand_merge_f64x2(values_vec, mask, merge_vec) + .to_array() + .map(f64::to_bits), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_f64x4(simd: S) { + let patterns: [u64; 8] = [ + 0x8000_0000_0000_0000, + 0x7ff8_1234_5678_9abc, + 0x7ff0_0000_0000_0000, + 0x7ff0_0000_0000_0001, + 0, + 0xfff0_0000_0000_0000, + 0xfff8_abcd_1234_5678, + 0x3ff0_0000_0000_0000, + ]; + let values: [u64; 4] = core::array::from_fn(|lane| patterns[lane % 8]); + let merge: [u64; 4] = core::array::from_fn(|lane| patterns[(lane + 3) % 8]); + let values_vec = f64x4::simd_from(simd, values.map(f64::from_bits)); + let merge_vec = f64x4::simd_from(simd, merge.map(f64::from_bits)); + let all_lanes = u64::MAX >> (64 - 4); + let single_lanes = (0..4).map(|lane| 1_u64 << lane); + let prefixes = (1..=4).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..4).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + { + let mask = mask64x4::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 4]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 4]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..4 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!( + values_vec.compress(mask).to_array().map(f64::to_bits), + expected_compress + ); + assert_eq!( + simd.compress_f64x4(values_vec, mask) + .to_array() + .map(f64::to_bits), + expected_compress + ); + assert_eq!( + values_vec + .compress_merge(mask, merge_vec) + .to_array() + .map(f64::to_bits), + expected_compress_merge + ); + assert_eq!( + simd.compress_merge_f64x4(values_vec, mask, merge_vec) + .to_array() + .map(f64::to_bits), + expected_compress_merge + ); + assert_eq!( + values_vec.expand(mask).to_array().map(f64::to_bits), + expected_expand + ); + assert_eq!( + simd.expand_f64x4(values_vec, mask) + .to_array() + .map(f64::to_bits), + expected_expand + ); + assert_eq!( + values_vec + .expand_merge(mask, merge_vec) + .to_array() + .map(f64::to_bits), + expected_expand_merge + ); + assert_eq!( + simd.expand_merge_f64x4(values_vec, mask, merge_vec) + .to_array() + .map(f64::to_bits), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_f64x8(simd: S) { + let patterns: [u64; 8] = [ + 0x8000_0000_0000_0000, + 0x7ff8_1234_5678_9abc, + 0x7ff0_0000_0000_0000, + 0x7ff0_0000_0000_0001, + 0, + 0xfff0_0000_0000_0000, + 0xfff8_abcd_1234_5678, + 0x3ff0_0000_0000_0000, + ]; + let values: [u64; 8] = core::array::from_fn(|lane| patterns[lane % 8]); + let merge: [u64; 8] = core::array::from_fn(|lane| patterns[(lane + 3) % 8]); + let values_vec = f64x8::simd_from(simd, values.map(f64::from_bits)); + let merge_vec = f64x8::simd_from(simd, merge.map(f64::from_bits)); + let all_lanes = u64::MAX >> (64 - 8); + let single_lanes = (0..8).map(|lane| 1_u64 << lane); + let prefixes = (1..=8).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..8).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + { + let mask = mask64x8::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 8]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 8]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..8 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!( + values_vec.compress(mask).to_array().map(f64::to_bits), + expected_compress + ); + assert_eq!( + simd.compress_f64x8(values_vec, mask) + .to_array() + .map(f64::to_bits), + expected_compress + ); + assert_eq!( + values_vec + .compress_merge(mask, merge_vec) + .to_array() + .map(f64::to_bits), + expected_compress_merge + ); + assert_eq!( + simd.compress_merge_f64x8(values_vec, mask, merge_vec) + .to_array() + .map(f64::to_bits), + expected_compress_merge + ); + assert_eq!( + values_vec.expand(mask).to_array().map(f64::to_bits), + expected_expand + ); + assert_eq!( + simd.expand_f64x8(values_vec, mask) + .to_array() + .map(f64::to_bits), + expected_expand + ); + assert_eq!( + values_vec + .expand_merge(mask, merge_vec) + .to_array() + .map(f64::to_bits), + expected_expand_merge + ); + assert_eq!( + simd.expand_merge_f64x8(values_vec, mask, merge_vec) + .to_array() + .map(f64::to_bits), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_generic_base(simd: S) { + fn compact>(value: V, mask: V::Mask, merge: V::Element) -> [V; 4] { + [ + value.compress(mask), + value.compress_merge(mask, merge), + value.expand(mask), + value.expand_merge(mask, merge), + ] + } + let values = u32x4::simd_from(simd, [10, 20, 30, 40]); + let mask = mask32x4::from_bitmask(simd, 0b1010); + let results = compact(values, mask, 99); + assert_eq!(*results[0], [20, 40, 0, 0]); + assert_eq!(*results[1], [20, 40, 99, 99]); + assert_eq!(*results[2], [0, 10, 0, 20]); + assert_eq!(*results[3], [99, 10, 99, 20]); +} + #[simd_test] fn compress_all_widths(simd: S) { macro_rules! check_width { From 0b4639af059d72e0ce8bbfa3ed9a52f583dbf338 Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Wed, 30 Sep 2026 11:18:49 +0100 Subject: [PATCH 3/8] Simplify mask control for a 13-28% measured speedup on zen5 in AVX2 mode --- fearless_simd/src/generated/avx2.rs | 96 ++++++------------- fearless_simd/src/generated/sse4_2.rs | 84 ++++++---------- fearless_simd/src/support.rs | 12 ++- fearless_simd_gen/src/mk_x86.rs | 11 ++- .../tests/harness/ops/compress.rs | 31 ++++++ .../tests/harness/ops/expand.rs | 31 ++++++ 6 files changed, 134 insertions(+), 131 deletions(-) diff --git a/fearless_simd/src/generated/avx2.rs b/fearless_simd/src/generated/avx2.rs index 23aa97bed..6c7051954 100644 --- a/fearless_simd/src/generated/avx2.rs +++ b/fearless_simd/src/generated/avx2.rs @@ -1548,8 +1548,7 @@ impl Simd for Avx2 { let high_mask = (usize::from(mask_bits)) >> 8; let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask]; let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask]; - let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + 0x0808_0808_0808_0808) - | (high_control & 0x8080_8080_8080_8080); + let high_control = high_control | 0x0808_0808_0808_0808; let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); let low_count = low_mask.count_ones() as usize; let compacted = _mm_shuffle_epi8(values.into(), control); @@ -1581,8 +1580,7 @@ impl Simd for Avx2 { let high_mask = (usize::from(mask_bits)) >> 8; let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask]; let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask]; - let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + 0x0808_0808_0808_0808) - | (high_control & 0x8080_8080_8080_8080); + let high_control = high_control | 0x0808_0808_0808_0808; let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); let low_count = low_mask.count_ones() as usize; let compacted = _mm_shuffle_epi8(values.into(), control); @@ -1613,8 +1611,7 @@ impl Simd for Avx2 { let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; let high_base = low_count * 0x0101_0101_0101_0101; - let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) - | (high_control & 0x8080_8080_8080_8080); + let high_control = high_control + high_base; let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); let expanded = _mm_shuffle_epi8(values.into(), control); expanded.simd_into(token) @@ -1644,8 +1641,7 @@ impl Simd for Avx2 { let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; let high_base = low_count * 0x0101_0101_0101_0101; - let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) - | (high_control & 0x8080_8080_8080_8080); + let high_control = high_control + high_base; let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); let expanded = _mm_shuffle_epi8(values.into(), control); _mm_blendv_epi8(merge.into(), expanded, mask.into()).simd_into(token) @@ -8558,9 +8554,7 @@ impl Simd for Avx2 { let high_mask_0 = (block_bits_0) >> 8; let low_control_0 = crate::support::COMPRESS_8_CONTROLS[low_mask_0]; let high_control_0 = crate::support::COMPRESS_8_CONTROLS[high_mask_0]; - let high_control_0 = ((high_control_0 & 0x7f7f_7f7f_7f7f_7f7f) - + 0x0808_0808_0808_0808) - | (high_control_0 & 0x8080_8080_8080_8080); + let high_control_0 = high_control_0 | 0x0808_0808_0808_0808; let control_0 = _mm_set_epi64x(high_control_0.cast_signed(), low_control_0.cast_signed()); let low_count_0 = low_mask_0.count_ones() as usize; @@ -8569,9 +8563,7 @@ impl Simd for Avx2 { let high_mask_1 = (block_bits_1) >> 8; let low_control_1 = crate::support::COMPRESS_8_CONTROLS[low_mask_1]; let high_control_1 = crate::support::COMPRESS_8_CONTROLS[high_mask_1]; - let high_control_1 = ((high_control_1 & 0x7f7f_7f7f_7f7f_7f7f) - + 0x0808_0808_0808_0808) - | (high_control_1 & 0x8080_8080_8080_8080); + let high_control_1 = high_control_1 | 0x0808_0808_0808_0808; let control_1 = _mm_set_epi64x(high_control_1.cast_signed(), low_control_1.cast_signed()); let low_count_1 = low_mask_1.count_ones() as usize; @@ -8623,9 +8615,7 @@ impl Simd for Avx2 { let high_mask_0 = (block_bits_0) >> 8; let low_control_0 = crate::support::COMPRESS_8_CONTROLS[low_mask_0]; let high_control_0 = crate::support::COMPRESS_8_CONTROLS[high_mask_0]; - let high_control_0 = ((high_control_0 & 0x7f7f_7f7f_7f7f_7f7f) - + 0x0808_0808_0808_0808) - | (high_control_0 & 0x8080_8080_8080_8080); + let high_control_0 = high_control_0 | 0x0808_0808_0808_0808; let control_0 = _mm_set_epi64x(high_control_0.cast_signed(), low_control_0.cast_signed()); let low_count_0 = low_mask_0.count_ones() as usize; @@ -8634,9 +8624,7 @@ impl Simd for Avx2 { let high_mask_1 = (block_bits_1) >> 8; let low_control_1 = crate::support::COMPRESS_8_CONTROLS[low_mask_1]; let high_control_1 = crate::support::COMPRESS_8_CONTROLS[high_mask_1]; - let high_control_1 = ((high_control_1 & 0x7f7f_7f7f_7f7f_7f7f) - + 0x0808_0808_0808_0808) - | (high_control_1 & 0x8080_8080_8080_8080); + let high_control_1 = high_control_1 | 0x0808_0808_0808_0808; let control_1 = _mm_set_epi64x(high_control_1.cast_signed(), low_control_1.cast_signed()); let low_count_1 = low_mask_1.count_ones() as usize; @@ -8699,8 +8687,7 @@ impl Simd for Avx2 { let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; let high_base = low_count * 0x0101_0101_0101_0101; - let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) - | (high_control & 0x8080_8080_8080_8080); + let high_control = high_control + high_base; let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); let packed = unsafe { _mm_loadu_si128(input.add(input_lane).cast::<__m128i>()) }; let expanded = _mm_shuffle_epi8(packed, control); @@ -8716,8 +8703,7 @@ impl Simd for Avx2 { let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; let high_base = low_count * 0x0101_0101_0101_0101; - let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) - | (high_control & 0x8080_8080_8080_8080); + let high_control = high_control + high_base; let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); let packed = unsafe { _mm_loadu_si128(input.add(input_lane).cast::<__m128i>()) }; let expanded = _mm_shuffle_epi8(packed, control); @@ -8762,8 +8748,7 @@ impl Simd for Avx2 { let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; let high_base = low_count * 0x0101_0101_0101_0101; - let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) - | (high_control & 0x8080_8080_8080_8080); + let high_control = high_control + high_base; let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); let packed = unsafe { _mm_loadu_si128(input.add(input_lane).cast::<__m128i>()) }; let expanded = _mm_shuffle_epi8(packed, control); @@ -8785,8 +8770,7 @@ impl Simd for Avx2 { let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; let high_base = low_count * 0x0101_0101_0101_0101; - let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) - | (high_control & 0x8080_8080_8080_8080); + let high_control = high_control + high_base; let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); let packed = unsafe { _mm_loadu_si128(input.add(input_lane).cast::<__m128i>()) }; let expanded = _mm_shuffle_epi8(packed, control); @@ -14407,9 +14391,7 @@ impl Simd for Avx2 { let high_mask_0 = (block_bits_0) >> 8; let low_control_0 = crate::support::COMPRESS_8_CONTROLS[low_mask_0]; let high_control_0 = crate::support::COMPRESS_8_CONTROLS[high_mask_0]; - let high_control_0 = ((high_control_0 & 0x7f7f_7f7f_7f7f_7f7f) - + 0x0808_0808_0808_0808) - | (high_control_0 & 0x8080_8080_8080_8080); + let high_control_0 = high_control_0 | 0x0808_0808_0808_0808; let control_0 = _mm_set_epi64x(high_control_0.cast_signed(), low_control_0.cast_signed()); let low_count_0 = low_mask_0.count_ones() as usize; @@ -14418,9 +14400,7 @@ impl Simd for Avx2 { let high_mask_1 = (block_bits_1) >> 8; let low_control_1 = crate::support::COMPRESS_8_CONTROLS[low_mask_1]; let high_control_1 = crate::support::COMPRESS_8_CONTROLS[high_mask_1]; - let high_control_1 = ((high_control_1 & 0x7f7f_7f7f_7f7f_7f7f) - + 0x0808_0808_0808_0808) - | (high_control_1 & 0x8080_8080_8080_8080); + let high_control_1 = high_control_1 | 0x0808_0808_0808_0808; let control_1 = _mm_set_epi64x(high_control_1.cast_signed(), low_control_1.cast_signed()); let low_count_1 = low_mask_1.count_ones() as usize; @@ -14450,9 +14430,7 @@ impl Simd for Avx2 { let high_mask_2 = (block_bits_2) >> 8; let low_control_2 = crate::support::COMPRESS_8_CONTROLS[low_mask_2]; let high_control_2 = crate::support::COMPRESS_8_CONTROLS[high_mask_2]; - let high_control_2 = ((high_control_2 & 0x7f7f_7f7f_7f7f_7f7f) - + 0x0808_0808_0808_0808) - | (high_control_2 & 0x8080_8080_8080_8080); + let high_control_2 = high_control_2 | 0x0808_0808_0808_0808; let control_2 = _mm_set_epi64x(high_control_2.cast_signed(), low_control_2.cast_signed()); let low_count_2 = low_mask_2.count_ones() as usize; @@ -14461,9 +14439,7 @@ impl Simd for Avx2 { let high_mask_3 = (block_bits_3) >> 8; let low_control_3 = crate::support::COMPRESS_8_CONTROLS[low_mask_3]; let high_control_3 = crate::support::COMPRESS_8_CONTROLS[high_mask_3]; - let high_control_3 = ((high_control_3 & 0x7f7f_7f7f_7f7f_7f7f) - + 0x0808_0808_0808_0808) - | (high_control_3 & 0x8080_8080_8080_8080); + let high_control_3 = high_control_3 | 0x0808_0808_0808_0808; let control_3 = _mm_set_epi64x(high_control_3.cast_signed(), low_control_3.cast_signed()); let low_count_3 = low_mask_3.count_ones() as usize; @@ -14561,9 +14537,7 @@ impl Simd for Avx2 { let high_mask_0 = (block_bits_0) >> 8; let low_control_0 = crate::support::COMPRESS_8_CONTROLS[low_mask_0]; let high_control_0 = crate::support::COMPRESS_8_CONTROLS[high_mask_0]; - let high_control_0 = ((high_control_0 & 0x7f7f_7f7f_7f7f_7f7f) - + 0x0808_0808_0808_0808) - | (high_control_0 & 0x8080_8080_8080_8080); + let high_control_0 = high_control_0 | 0x0808_0808_0808_0808; let control_0 = _mm_set_epi64x(high_control_0.cast_signed(), low_control_0.cast_signed()); let low_count_0 = low_mask_0.count_ones() as usize; @@ -14572,9 +14546,7 @@ impl Simd for Avx2 { let high_mask_1 = (block_bits_1) >> 8; let low_control_1 = crate::support::COMPRESS_8_CONTROLS[low_mask_1]; let high_control_1 = crate::support::COMPRESS_8_CONTROLS[high_mask_1]; - let high_control_1 = ((high_control_1 & 0x7f7f_7f7f_7f7f_7f7f) - + 0x0808_0808_0808_0808) - | (high_control_1 & 0x8080_8080_8080_8080); + let high_control_1 = high_control_1 | 0x0808_0808_0808_0808; let control_1 = _mm_set_epi64x(high_control_1.cast_signed(), low_control_1.cast_signed()); let low_count_1 = low_mask_1.count_ones() as usize; @@ -14604,9 +14576,7 @@ impl Simd for Avx2 { let high_mask_2 = (block_bits_2) >> 8; let low_control_2 = crate::support::COMPRESS_8_CONTROLS[low_mask_2]; let high_control_2 = crate::support::COMPRESS_8_CONTROLS[high_mask_2]; - let high_control_2 = ((high_control_2 & 0x7f7f_7f7f_7f7f_7f7f) - + 0x0808_0808_0808_0808) - | (high_control_2 & 0x8080_8080_8080_8080); + let high_control_2 = high_control_2 | 0x0808_0808_0808_0808; let control_2 = _mm_set_epi64x(high_control_2.cast_signed(), low_control_2.cast_signed()); let low_count_2 = low_mask_2.count_ones() as usize; @@ -14615,9 +14585,7 @@ impl Simd for Avx2 { let high_mask_3 = (block_bits_3) >> 8; let low_control_3 = crate::support::COMPRESS_8_CONTROLS[low_mask_3]; let high_control_3 = crate::support::COMPRESS_8_CONTROLS[high_mask_3]; - let high_control_3 = ((high_control_3 & 0x7f7f_7f7f_7f7f_7f7f) - + 0x0808_0808_0808_0808) - | (high_control_3 & 0x8080_8080_8080_8080); + let high_control_3 = high_control_3 | 0x0808_0808_0808_0808; let control_3 = _mm_set_epi64x(high_control_3.cast_signed(), low_control_3.cast_signed()); let low_count_3 = low_mask_3.count_ones() as usize; @@ -14726,8 +14694,7 @@ impl Simd for Avx2 { let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; let high_base = low_count * 0x0101_0101_0101_0101; - let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) - | (high_control & 0x8080_8080_8080_8080); + let high_control = high_control + high_base; let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); let packed = unsafe { _mm_loadu_si128(input.add(input_lane).cast::<__m128i>()) }; let expanded = _mm_shuffle_epi8(packed, control); @@ -14743,8 +14710,7 @@ impl Simd for Avx2 { let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; let high_base = low_count * 0x0101_0101_0101_0101; - let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) - | (high_control & 0x8080_8080_8080_8080); + let high_control = high_control + high_base; let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); let packed = unsafe { _mm_loadu_si128(input.add(input_lane).cast::<__m128i>()) }; let expanded = _mm_shuffle_epi8(packed, control); @@ -14760,8 +14726,7 @@ impl Simd for Avx2 { let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; let high_base = low_count * 0x0101_0101_0101_0101; - let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) - | (high_control & 0x8080_8080_8080_8080); + let high_control = high_control + high_base; let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); let packed = unsafe { _mm_loadu_si128(input.add(input_lane).cast::<__m128i>()) }; let expanded = _mm_shuffle_epi8(packed, control); @@ -14777,8 +14742,7 @@ impl Simd for Avx2 { let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; let high_base = low_count * 0x0101_0101_0101_0101; - let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) - | (high_control & 0x8080_8080_8080_8080); + let high_control = high_control + high_base; let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); let packed = unsafe { _mm_loadu_si128(input.add(input_lane).cast::<__m128i>()) }; let expanded = _mm_shuffle_epi8(packed, control); @@ -14831,8 +14795,7 @@ impl Simd for Avx2 { let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; let high_base = low_count * 0x0101_0101_0101_0101; - let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) - | (high_control & 0x8080_8080_8080_8080); + let high_control = high_control + high_base; let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); let packed = unsafe { _mm_loadu_si128(input.add(input_lane).cast::<__m128i>()) }; let expanded = _mm_shuffle_epi8(packed, control); @@ -14854,8 +14817,7 @@ impl Simd for Avx2 { let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; let high_base = low_count * 0x0101_0101_0101_0101; - let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) - | (high_control & 0x8080_8080_8080_8080); + let high_control = high_control + high_base; let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); let packed = unsafe { _mm_loadu_si128(input.add(input_lane).cast::<__m128i>()) }; let expanded = _mm_shuffle_epi8(packed, control); @@ -14877,8 +14839,7 @@ impl Simd for Avx2 { let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; let high_base = low_count * 0x0101_0101_0101_0101; - let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) - | (high_control & 0x8080_8080_8080_8080); + let high_control = high_control + high_base; let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); let packed = unsafe { _mm_loadu_si128(input.add(input_lane).cast::<__m128i>()) }; let expanded = _mm_shuffle_epi8(packed, control); @@ -14900,8 +14861,7 @@ impl Simd for Avx2 { let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; let high_base = low_count * 0x0101_0101_0101_0101; - let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) - | (high_control & 0x8080_8080_8080_8080); + let high_control = high_control + high_base; let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); let packed = unsafe { _mm_loadu_si128(input.add(input_lane).cast::<__m128i>()) }; let expanded = _mm_shuffle_epi8(packed, control); diff --git a/fearless_simd/src/generated/sse4_2.rs b/fearless_simd/src/generated/sse4_2.rs index 8e10f46b6..a8cd33f54 100644 --- a/fearless_simd/src/generated/sse4_2.rs +++ b/fearless_simd/src/generated/sse4_2.rs @@ -1546,8 +1546,7 @@ impl Simd for Sse4_2 { let high_mask = (usize::from(mask_bits)) >> 8; let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask]; let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask]; - let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + 0x0808_0808_0808_0808) - | (high_control & 0x8080_8080_8080_8080); + let high_control = high_control | 0x0808_0808_0808_0808; let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); let low_count = low_mask.count_ones() as usize; let compacted = _mm_shuffle_epi8(values.into(), control); @@ -1579,8 +1578,7 @@ impl Simd for Sse4_2 { let high_mask = (usize::from(mask_bits)) >> 8; let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask]; let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask]; - let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + 0x0808_0808_0808_0808) - | (high_control & 0x8080_8080_8080_8080); + let high_control = high_control | 0x0808_0808_0808_0808; let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); let low_count = low_mask.count_ones() as usize; let compacted = _mm_shuffle_epi8(values.into(), control); @@ -1615,8 +1613,7 @@ impl Simd for Sse4_2 { let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; let high_base = low_count * 0x0101_0101_0101_0101; - let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) - | (high_control & 0x8080_8080_8080_8080); + let high_control = high_control + high_base; let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); let expanded = _mm_shuffle_epi8(values.into(), control); expanded.simd_into(token) @@ -1646,8 +1643,7 @@ impl Simd for Sse4_2 { let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; let high_base = low_count * 0x0101_0101_0101_0101; - let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) - | (high_control & 0x8080_8080_8080_8080); + let high_control = high_control + high_base; let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); let expanded = _mm_shuffle_epi8(values.into(), control); _mm_blendv_epi8(merge.into(), expanded, mask.into()).simd_into(token) @@ -7443,8 +7439,7 @@ impl Simd for Sse4_2 { let high_mask = (block_bits) >> 8; let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask]; let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask]; - let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + 0x0808_0808_0808_0808) - | (high_control & 0x8080_8080_8080_8080); + let high_control = high_control | 0x0808_0808_0808_0808; let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); let low_count = low_mask.count_ones() as usize; let compacted = _mm_shuffle_epi8(values.val.0[0], control); @@ -7463,8 +7458,7 @@ impl Simd for Sse4_2 { let high_mask = (block_bits) >> 8; let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask]; let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask]; - let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + 0x0808_0808_0808_0808) - | (high_control & 0x8080_8080_8080_8080); + let high_control = high_control | 0x0808_0808_0808_0808; let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); let low_count = low_mask.count_ones() as usize; let compacted = _mm_shuffle_epi8(values.val.0[1], control); @@ -7505,8 +7499,7 @@ impl Simd for Sse4_2 { let high_mask = (block_bits) >> 8; let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask]; let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask]; - let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + 0x0808_0808_0808_0808) - | (high_control & 0x8080_8080_8080_8080); + let high_control = high_control | 0x0808_0808_0808_0808; let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); let low_count = low_mask.count_ones() as usize; let compacted = _mm_shuffle_epi8(values.val.0[0], control); @@ -7525,8 +7518,7 @@ impl Simd for Sse4_2 { let high_mask = (block_bits) >> 8; let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask]; let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask]; - let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + 0x0808_0808_0808_0808) - | (high_control & 0x8080_8080_8080_8080); + let high_control = high_control | 0x0808_0808_0808_0808; let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); let low_count = low_mask.count_ones() as usize; let compacted = _mm_shuffle_epi8(values.val.0[1], control); @@ -7591,8 +7583,7 @@ impl Simd for Sse4_2 { let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; let high_base = low_count * 0x0101_0101_0101_0101; - let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) - | (high_control & 0x8080_8080_8080_8080); + let high_control = high_control + high_base; let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); let packed = unsafe { _mm_loadu_si128(input.as_ptr().add(input_lane).cast::<__m128i>()) }; @@ -7609,8 +7600,7 @@ impl Simd for Sse4_2 { let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; let high_base = low_count * 0x0101_0101_0101_0101; - let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) - | (high_control & 0x8080_8080_8080_8080); + let high_control = high_control + high_base; let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); let packed = unsafe { _mm_loadu_si128(input.as_ptr().add(input_lane).cast::<__m128i>()) }; @@ -7650,8 +7640,7 @@ impl Simd for Sse4_2 { let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; let high_base = low_count * 0x0101_0101_0101_0101; - let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) - | (high_control & 0x8080_8080_8080_8080); + let high_control = high_control + high_base; let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); let packed = unsafe { _mm_loadu_si128(input.as_ptr().add(input_lane).cast::<__m128i>()) }; @@ -7668,8 +7657,7 @@ impl Simd for Sse4_2 { let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; let high_base = low_count * 0x0101_0101_0101_0101; - let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) - | (high_control & 0x8080_8080_8080_8080); + let high_control = high_control + high_base; let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); let packed = unsafe { _mm_loadu_si128(input.as_ptr().add(input_lane).cast::<__m128i>()) }; @@ -8873,8 +8861,7 @@ impl Simd for Sse4_2 { let high_mask = (block_bits) >> 8; let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask]; let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask]; - let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + 0x0808_0808_0808_0808) - | (high_control & 0x8080_8080_8080_8080); + let high_control = high_control | 0x0808_0808_0808_0808; let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); let low_count = low_mask.count_ones() as usize; let compacted = _mm_shuffle_epi8(values.val.0[0], control); @@ -8893,8 +8880,7 @@ impl Simd for Sse4_2 { let high_mask = (block_bits) >> 8; let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask]; let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask]; - let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + 0x0808_0808_0808_0808) - | (high_control & 0x8080_8080_8080_8080); + let high_control = high_control | 0x0808_0808_0808_0808; let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); let low_count = low_mask.count_ones() as usize; let compacted = _mm_shuffle_epi8(values.val.0[1], control); @@ -8913,8 +8899,7 @@ impl Simd for Sse4_2 { let high_mask = (block_bits) >> 8; let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask]; let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask]; - let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + 0x0808_0808_0808_0808) - | (high_control & 0x8080_8080_8080_8080); + let high_control = high_control | 0x0808_0808_0808_0808; let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); let low_count = low_mask.count_ones() as usize; let compacted = _mm_shuffle_epi8(values.val.0[2], control); @@ -8933,8 +8918,7 @@ impl Simd for Sse4_2 { let high_mask = (block_bits) >> 8; let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask]; let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask]; - let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + 0x0808_0808_0808_0808) - | (high_control & 0x8080_8080_8080_8080); + let high_control = high_control | 0x0808_0808_0808_0808; let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); let low_count = low_mask.count_ones() as usize; let compacted = _mm_shuffle_epi8(values.val.0[3], control); @@ -8975,8 +8959,7 @@ impl Simd for Sse4_2 { let high_mask = (block_bits) >> 8; let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask]; let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask]; - let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + 0x0808_0808_0808_0808) - | (high_control & 0x8080_8080_8080_8080); + let high_control = high_control | 0x0808_0808_0808_0808; let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); let low_count = low_mask.count_ones() as usize; let compacted = _mm_shuffle_epi8(values.val.0[0], control); @@ -8995,8 +8978,7 @@ impl Simd for Sse4_2 { let high_mask = (block_bits) >> 8; let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask]; let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask]; - let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + 0x0808_0808_0808_0808) - | (high_control & 0x8080_8080_8080_8080); + let high_control = high_control | 0x0808_0808_0808_0808; let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); let low_count = low_mask.count_ones() as usize; let compacted = _mm_shuffle_epi8(values.val.0[1], control); @@ -9015,8 +8997,7 @@ impl Simd for Sse4_2 { let high_mask = (block_bits) >> 8; let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask]; let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask]; - let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + 0x0808_0808_0808_0808) - | (high_control & 0x8080_8080_8080_8080); + let high_control = high_control | 0x0808_0808_0808_0808; let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); let low_count = low_mask.count_ones() as usize; let compacted = _mm_shuffle_epi8(values.val.0[2], control); @@ -9035,8 +9016,7 @@ impl Simd for Sse4_2 { let high_mask = (block_bits) >> 8; let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask]; let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask]; - let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + 0x0808_0808_0808_0808) - | (high_control & 0x8080_8080_8080_8080); + let high_control = high_control | 0x0808_0808_0808_0808; let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); let low_count = low_mask.count_ones() as usize; let compacted = _mm_shuffle_epi8(values.val.0[3], control); @@ -9127,8 +9107,7 @@ impl Simd for Sse4_2 { let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; let high_base = low_count * 0x0101_0101_0101_0101; - let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) - | (high_control & 0x8080_8080_8080_8080); + let high_control = high_control + high_base; let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); let packed = unsafe { _mm_loadu_si128(input.as_ptr().add(input_lane).cast::<__m128i>()) }; @@ -9145,8 +9124,7 @@ impl Simd for Sse4_2 { let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; let high_base = low_count * 0x0101_0101_0101_0101; - let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) - | (high_control & 0x8080_8080_8080_8080); + let high_control = high_control + high_base; let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); let packed = unsafe { _mm_loadu_si128(input.as_ptr().add(input_lane).cast::<__m128i>()) }; @@ -9163,8 +9141,7 @@ impl Simd for Sse4_2 { let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; let high_base = low_count * 0x0101_0101_0101_0101; - let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) - | (high_control & 0x8080_8080_8080_8080); + let high_control = high_control + high_base; let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); let packed = unsafe { _mm_loadu_si128(input.as_ptr().add(input_lane).cast::<__m128i>()) }; @@ -9181,8 +9158,7 @@ impl Simd for Sse4_2 { let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; let high_base = low_count * 0x0101_0101_0101_0101; - let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) - | (high_control & 0x8080_8080_8080_8080); + let high_control = high_control + high_base; let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); let packed = unsafe { _mm_loadu_si128(input.as_ptr().add(input_lane).cast::<__m128i>()) }; @@ -9222,8 +9198,7 @@ impl Simd for Sse4_2 { let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; let high_base = low_count * 0x0101_0101_0101_0101; - let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) - | (high_control & 0x8080_8080_8080_8080); + let high_control = high_control + high_base; let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); let packed = unsafe { _mm_loadu_si128(input.as_ptr().add(input_lane).cast::<__m128i>()) }; @@ -9240,8 +9215,7 @@ impl Simd for Sse4_2 { let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; let high_base = low_count * 0x0101_0101_0101_0101; - let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) - | (high_control & 0x8080_8080_8080_8080); + let high_control = high_control + high_base; let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); let packed = unsafe { _mm_loadu_si128(input.as_ptr().add(input_lane).cast::<__m128i>()) }; @@ -9258,8 +9232,7 @@ impl Simd for Sse4_2 { let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; let high_base = low_count * 0x0101_0101_0101_0101; - let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) - | (high_control & 0x8080_8080_8080_8080); + let high_control = high_control + high_base; let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); let packed = unsafe { _mm_loadu_si128(input.as_ptr().add(input_lane).cast::<__m128i>()) }; @@ -9276,8 +9249,7 @@ impl Simd for Sse4_2 { let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; let high_base = low_count * 0x0101_0101_0101_0101; - let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) - | (high_control & 0x8080_8080_8080_8080); + let high_control = high_control + high_base; let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); let packed = unsafe { _mm_loadu_si128(input.as_ptr().add(input_lane).cast::<__m128i>()) }; diff --git a/fearless_simd/src/support.rs b/fearless_simd/src/support.rs index ae7ea63da..fb86edb30 100644 --- a/fearless_simd/src/support.rs +++ b/fearless_simd/src/support.rs @@ -65,10 +65,18 @@ pub(crate) fn cross_block_slide_blocks_at( } const fn compact_control_table(expand: bool) -> [u64; 256] { - let mut table = [u64::MAX; 256]; + // Expansion adds a lane offset to each byte of the packed control. Use 0x80 + // for inactive lanes so these additions preserve the zeroing bit without + // carrying into adjacent bytes. Every supported shuffle treats it as zero. + let empty = if expand { + 0x8080_8080_8080_8080 + } else { + u64::MAX + }; + let mut table = [empty; 256]; let mut mask = 0_usize; while mask < table.len() { - let mut control = u64::MAX; + let mut control = empty; let mut selected = 0_usize; let mut lane = 0_usize; while lane < 8 { diff --git a/fearless_simd_gen/src/mk_x86.rs b/fearless_simd_gen/src/mk_x86.rs index b20651fca..65c92a215 100644 --- a/fearless_simd_gen/src/mk_x86.rs +++ b/fearless_simd_gen/src/mk_x86.rs @@ -1265,9 +1265,9 @@ fn compact_16_control(block_bits: TokenStream, suffix: &str) -> (TokenStream, Id let #high_mask = (#block_bits) >> 8; let #low_control = crate::support::COMPRESS_8_CONTROLS[#low_mask]; let #high_control = crate::support::COMPRESS_8_CONTROLS[#high_mask]; - let #high_control = ((#high_control & 0x7f7f_7f7f_7f7f_7f7f) - + 0x0808_0808_0808_0808) - | (#high_control & 0x8080_8080_8080_8080); + // Selected indices are 0..=7; setting bit 3 offsets them by eight. + // Inactive indices retain their high bit and still zero the shuffled byte. + let #high_control = #high_control | 0x0808_0808_0808_0808; let #control = _mm_set_epi64x(#high_control.cast_signed(), #low_control.cast_signed()); let #low_count = #low_mask.count_ones() as usize; }; @@ -1283,8 +1283,9 @@ fn expand_16_control(block_bits: TokenStream) -> TokenStream { let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; let high_base = low_count * 0x0101_0101_0101_0101; - let high_control = ((high_control & 0x7f7f_7f7f_7f7f_7f7f) + high_base) - | (high_control & 0x8080_8080_8080_8080); + // Inactive controls are 0x80; adding at most eight cannot carry between + // bytes or clear the zeroing bit. Selected controls remain in 0..=15. + let high_control = high_control + high_base; let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); } } diff --git a/fearless_simd_tests/tests/harness/ops/compress.rs b/fearless_simd_tests/tests/harness/ops/compress.rs index 460ce4a4f..8ba4a2c68 100644 --- a/fearless_simd_tests/tests/harness/ops/compress.rs +++ b/fearless_simd_tests/tests/harness/ops/compress.rs @@ -2140,3 +2140,34 @@ fn compress_all_widths(simd: S) { check_width!(u8x32, mask8x32, 32, compress_u8x32, compress_merge_u8x32); check_width!(u8x64, mask8x64, 64, compress_u8x64, compress_merge_u8x64); } + +#[simd_test] +fn compress_u8x16_exhaustive_masks(simd: S) { + let values = [ + 0, 255, 128, 1, 127, 254, 2, 253, 3, 252, 4, 251, 5, 250, 6, 249, + ]; + let merge = [ + 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, + ]; + let values_vec = u8x16::simd_from(simd, values); + let merge_vec = u8x16::simd_from(simd, merge); + for bits in 0..=u16::MAX { + let mask = mask8x16::from_bitmask(simd, u64::from(bits)); + let mut expected = [0; 16]; + let mut expected_merge = merge; + let mut selected = 0; + for (lane, value) in values.into_iter().enumerate() { + if bits & (1 << lane) != 0 { + expected[selected] = value; + expected_merge[selected] = value; + selected += 1; + } + } + assert_eq!(*values_vec.compress(mask), expected, "mask {bits:#06x}"); + assert_eq!( + *values_vec.compress_merge(mask, merge_vec), + expected_merge, + "mask {bits:#06x}" + ); + } +} diff --git a/fearless_simd_tests/tests/harness/ops/expand.rs b/fearless_simd_tests/tests/harness/ops/expand.rs index a1f6be9f4..40ece8e52 100644 --- a/fearless_simd_tests/tests/harness/ops/expand.rs +++ b/fearless_simd_tests/tests/harness/ops/expand.rs @@ -49,3 +49,34 @@ fn expand_all_widths(simd: S) { check_width!(u8x32, mask8x32, 32, expand_u8x32, expand_merge_u8x32); check_width!(u8x64, mask8x64, 64, expand_u8x64, expand_merge_u8x64); } + +#[simd_test] +fn expand_u8x16_exhaustive_masks(simd: S) { + let values = [ + 0, 255, 128, 1, 127, 254, 2, 253, 3, 252, 4, 251, 5, 250, 6, 249, + ]; + let merge = [ + 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, + ]; + let values_vec = u8x16::simd_from(simd, values); + let merge_vec = u8x16::simd_from(simd, merge); + for bits in 0..=u16::MAX { + let mask = mask8x16::from_bitmask(simd, u64::from(bits)); + let mut expected = [0; 16]; + let mut expected_merge = merge; + let mut selected = 0; + for lane in 0..16 { + if bits & (1 << lane) != 0 { + expected[lane] = values[selected]; + expected_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!(*values_vec.expand(mask), expected, "mask {bits:#06x}"); + assert_eq!( + *values_vec.expand_merge(mask, merge_vec), + expected_merge, + "mask {bits:#06x}" + ); + } +} From 92fe11053069fb63641dd9ddccaf9beb8370f2cf Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Wed, 30 Sep 2026 11:42:48 +0100 Subject: [PATCH 4/8] Simplify mask control on NEON and WASM --- fearless_simd/src/generated/neon.rs | 36 ++++++++++------------------- fearless_simd/src/generated/wasm.rs | 36 ++++++++++------------------- fearless_simd_gen/src/generic.rs | 22 ++++++++++-------- 3 files changed, 37 insertions(+), 57 deletions(-) diff --git a/fearless_simd/src/generated/neon.rs b/fearless_simd/src/generated/neon.rs index 526afc83f..c60f16444 100644 --- a/fearless_simd/src/generated/neon.rs +++ b/fearless_simd/src/generated/neon.rs @@ -1232,8 +1232,7 @@ impl Simd for Neon { let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; let packed = crate::support::COMPRESS_8_CONTROLS[block_mask]; let base = (block * 8) as u64 * 0x0101_0101_0101_0101; - let adjusted = - ((packed & 0x7f7f_7f7f_7f7f_7f7f) + base) | (packed & 0x8080_8080_8080_8080); + let adjusted = packed | base; let adjusted = adjusted.to_le_bytes(); let write_len = core::cmp::min(8, 16 - output_lane); control[output_lane..output_lane + write_len].copy_from_slice(&adjusted[..write_len]); @@ -1256,8 +1255,7 @@ impl Simd for Neon { let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; let packed = crate::support::COMPRESS_8_CONTROLS[block_mask]; let base = (block * 8) as u64 * 0x0101_0101_0101_0101; - let adjusted = - ((packed & 0x7f7f_7f7f_7f7f_7f7f) + base) | (packed & 0x8080_8080_8080_8080); + let adjusted = packed | base; let adjusted = adjusted.to_le_bytes(); let write_len = core::cmp::min(8, 16 - output_lane); control[output_lane..output_lane + write_len].copy_from_slice(&adjusted[..write_len]); @@ -1295,8 +1293,7 @@ impl Simd for Neon { let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; let packed = crate::support::EXPAND_8_CONTROLS[block_mask]; let base = input_lane as u64 * 0x0101_0101_0101_0101; - let adjusted = - ((packed & 0x7f7f_7f7f_7f7f_7f7f) + base) | (packed & 0x8080_8080_8080_8080); + let adjusted = packed + base; let output_lane = block * 8; control[output_lane..output_lane + 8].copy_from_slice(&adjusted.to_le_bytes()); input_lane += crate::support::COMPACT_8_COUNTS[block_mask] as usize; @@ -1318,8 +1315,7 @@ impl Simd for Neon { let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; let packed = crate::support::EXPAND_8_CONTROLS[block_mask]; let base = input_lane as u64 * 0x0101_0101_0101_0101; - let adjusted = - ((packed & 0x7f7f_7f7f_7f7f_7f7f) + base) | (packed & 0x8080_8080_8080_8080); + let adjusted = packed + base; let output_lane = block * 8; control[output_lane..output_lane + 8].copy_from_slice(&adjusted.to_le_bytes()); input_lane += crate::support::COMPACT_8_COUNTS[block_mask] as usize; @@ -6354,8 +6350,7 @@ impl Simd for Neon { let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; let packed = crate::support::COMPRESS_8_CONTROLS[block_mask]; let base = (block * 8) as u64 * 0x0101_0101_0101_0101; - let adjusted = - ((packed & 0x7f7f_7f7f_7f7f_7f7f) + base) | (packed & 0x8080_8080_8080_8080); + let adjusted = packed | base; let adjusted = adjusted.to_le_bytes(); let write_len = core::cmp::min(8, 32 - output_lane); control[output_lane..output_lane + write_len].copy_from_slice(&adjusted[..write_len]); @@ -6378,8 +6373,7 @@ impl Simd for Neon { let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; let packed = crate::support::COMPRESS_8_CONTROLS[block_mask]; let base = (block * 8) as u64 * 0x0101_0101_0101_0101; - let adjusted = - ((packed & 0x7f7f_7f7f_7f7f_7f7f) + base) | (packed & 0x8080_8080_8080_8080); + let adjusted = packed | base; let adjusted = adjusted.to_le_bytes(); let write_len = core::cmp::min(8, 32 - output_lane); control[output_lane..output_lane + write_len].copy_from_slice(&adjusted[..write_len]); @@ -6420,8 +6414,7 @@ impl Simd for Neon { let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; let packed = crate::support::EXPAND_8_CONTROLS[block_mask]; let base = input_lane as u64 * 0x0101_0101_0101_0101; - let adjusted = - ((packed & 0x7f7f_7f7f_7f7f_7f7f) + base) | (packed & 0x8080_8080_8080_8080); + let adjusted = packed + base; let output_lane = block * 8; control[output_lane..output_lane + 8].copy_from_slice(&adjusted.to_le_bytes()); input_lane += crate::support::COMPACT_8_COUNTS[block_mask] as usize; @@ -6443,8 +6436,7 @@ impl Simd for Neon { let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; let packed = crate::support::EXPAND_8_CONTROLS[block_mask]; let base = input_lane as u64 * 0x0101_0101_0101_0101; - let adjusted = - ((packed & 0x7f7f_7f7f_7f7f_7f7f) + base) | (packed & 0x8080_8080_8080_8080); + let adjusted = packed + base; let output_lane = block * 8; control[output_lane..output_lane + 8].copy_from_slice(&adjusted.to_le_bytes()); input_lane += crate::support::COMPACT_8_COUNTS[block_mask] as usize; @@ -8010,8 +8002,7 @@ impl Simd for Neon { let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; let packed = crate::support::COMPRESS_8_CONTROLS[block_mask]; let base = (block * 8) as u64 * 0x0101_0101_0101_0101; - let adjusted = - ((packed & 0x7f7f_7f7f_7f7f_7f7f) + base) | (packed & 0x8080_8080_8080_8080); + let adjusted = packed | base; let adjusted = adjusted.to_le_bytes(); let write_len = core::cmp::min(8, 64 - output_lane); control[output_lane..output_lane + write_len].copy_from_slice(&adjusted[..write_len]); @@ -8034,8 +8025,7 @@ impl Simd for Neon { let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; let packed = crate::support::COMPRESS_8_CONTROLS[block_mask]; let base = (block * 8) as u64 * 0x0101_0101_0101_0101; - let adjusted = - ((packed & 0x7f7f_7f7f_7f7f_7f7f) + base) | (packed & 0x8080_8080_8080_8080); + let adjusted = packed | base; let adjusted = adjusted.to_le_bytes(); let write_len = core::cmp::min(8, 64 - output_lane); control[output_lane..output_lane + write_len].copy_from_slice(&adjusted[..write_len]); @@ -8078,8 +8068,7 @@ impl Simd for Neon { let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; let packed = crate::support::EXPAND_8_CONTROLS[block_mask]; let base = input_lane as u64 * 0x0101_0101_0101_0101; - let adjusted = - ((packed & 0x7f7f_7f7f_7f7f_7f7f) + base) | (packed & 0x8080_8080_8080_8080); + let adjusted = packed + base; let output_lane = block * 8; control[output_lane..output_lane + 8].copy_from_slice(&adjusted.to_le_bytes()); input_lane += crate::support::COMPACT_8_COUNTS[block_mask] as usize; @@ -8101,8 +8090,7 @@ impl Simd for Neon { let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; let packed = crate::support::EXPAND_8_CONTROLS[block_mask]; let base = input_lane as u64 * 0x0101_0101_0101_0101; - let adjusted = - ((packed & 0x7f7f_7f7f_7f7f_7f7f) + base) | (packed & 0x8080_8080_8080_8080); + let adjusted = packed + base; let output_lane = block * 8; control[output_lane..output_lane + 8].copy_from_slice(&adjusted.to_le_bytes()); input_lane += crate::support::COMPACT_8_COUNTS[block_mask] as usize; diff --git a/fearless_simd/src/generated/wasm.rs b/fearless_simd/src/generated/wasm.rs index f73a8dfb2..972955eb0 100644 --- a/fearless_simd/src/generated/wasm.rs +++ b/fearless_simd/src/generated/wasm.rs @@ -1049,8 +1049,7 @@ impl Simd for WasmSimd128 { let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; let packed = crate::support::COMPRESS_8_CONTROLS[block_mask]; let base = (block * 8) as u64 * 0x0101_0101_0101_0101; - let adjusted = - ((packed & 0x7f7f_7f7f_7f7f_7f7f) + base) | (packed & 0x8080_8080_8080_8080); + let adjusted = packed | base; let adjusted = adjusted.to_le_bytes(); let write_len = core::cmp::min(8, 16 - output_lane); control[output_lane..output_lane + write_len].copy_from_slice(&adjusted[..write_len]); @@ -1073,8 +1072,7 @@ impl Simd for WasmSimd128 { let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; let packed = crate::support::COMPRESS_8_CONTROLS[block_mask]; let base = (block * 8) as u64 * 0x0101_0101_0101_0101; - let adjusted = - ((packed & 0x7f7f_7f7f_7f7f_7f7f) + base) | (packed & 0x8080_8080_8080_8080); + let adjusted = packed | base; let adjusted = adjusted.to_le_bytes(); let write_len = core::cmp::min(8, 16 - output_lane); control[output_lane..output_lane + write_len].copy_from_slice(&adjusted[..write_len]); @@ -1099,8 +1097,7 @@ impl Simd for WasmSimd128 { let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; let packed = crate::support::EXPAND_8_CONTROLS[block_mask]; let base = input_lane as u64 * 0x0101_0101_0101_0101; - let adjusted = - ((packed & 0x7f7f_7f7f_7f7f_7f7f) + base) | (packed & 0x8080_8080_8080_8080); + let adjusted = packed + base; let output_lane = block * 8; control[output_lane..output_lane + 8].copy_from_slice(&adjusted.to_le_bytes()); input_lane += block_mask.count_ones() as usize; @@ -1122,8 +1119,7 @@ impl Simd for WasmSimd128 { let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; let packed = crate::support::EXPAND_8_CONTROLS[block_mask]; let base = input_lane as u64 * 0x0101_0101_0101_0101; - let adjusted = - ((packed & 0x7f7f_7f7f_7f7f_7f7f) + base) | (packed & 0x8080_8080_8080_8080); + let adjusted = packed + base; let output_lane = block * 8; control[output_lane..output_lane + 8].copy_from_slice(&adjusted.to_le_bytes()); input_lane += block_mask.count_ones() as usize; @@ -4738,8 +4734,7 @@ impl Simd for WasmSimd128 { let low_count = low_mask.count_ones() as usize; let low = crate::support::COMPRESS_8_CONTROLS[low_mask]; let high = crate::support::COMPRESS_8_CONTROLS[high_mask]; - let high = ((high & 0x7f7f_7f7f_7f7f_7f7f) + 0x0808_0808_0808_0808) - | (high & 0x8080_8080_8080_8080); + let high = high | 0x0808_0808_0808_0808; let mut control = [u8::MAX; 16]; control[..8].copy_from_slice(&low.to_le_bytes()); control[low_count..low_count + 8].copy_from_slice(&high.to_le_bytes()); @@ -4769,8 +4764,7 @@ impl Simd for WasmSimd128 { let low_count = low_mask.count_ones() as usize; let low = crate::support::COMPRESS_8_CONTROLS[low_mask]; let high = crate::support::COMPRESS_8_CONTROLS[high_mask]; - let high = ((high & 0x7f7f_7f7f_7f7f_7f7f) + 0x0808_0808_0808_0808) - | (high & 0x8080_8080_8080_8080); + let high = high | 0x0808_0808_0808_0808; let mut control = [u8::MAX; 16]; control[..8].copy_from_slice(&low.to_le_bytes()); control[low_count..low_count + 8].copy_from_slice(&high.to_le_bytes()); @@ -4804,8 +4798,7 @@ impl Simd for WasmSimd128 { let low = crate::support::EXPAND_8_CONTROLS[low_mask]; let high = crate::support::EXPAND_8_CONTROLS[high_mask]; let high_base = low_count as u64 * 0x0101_0101_0101_0101; - let high = - ((high & 0x7f7f_7f7f_7f7f_7f7f) + high_base) | (high & 0x8080_8080_8080_8080); + let high = high + high_base; let mut control = [0u8; 16]; control[..8].copy_from_slice(&low.to_le_bytes()); control[8..].copy_from_slice(&high.to_le_bytes()); @@ -4839,8 +4832,7 @@ impl Simd for WasmSimd128 { let low = crate::support::EXPAND_8_CONTROLS[low_mask]; let high = crate::support::EXPAND_8_CONTROLS[high_mask]; let high_base = low_count as u64 * 0x0101_0101_0101_0101; - let high = - ((high & 0x7f7f_7f7f_7f7f_7f7f) + high_base) | (high & 0x8080_8080_8080_8080); + let high = high + high_base; let mut control = [0u8; 16]; control[..8].copy_from_slice(&low.to_le_bytes()); control[8..].copy_from_slice(&high.to_le_bytes()); @@ -6002,8 +5994,7 @@ impl Simd for WasmSimd128 { let low_count = low_mask.count_ones() as usize; let low = crate::support::COMPRESS_8_CONTROLS[low_mask]; let high = crate::support::COMPRESS_8_CONTROLS[high_mask]; - let high = ((high & 0x7f7f_7f7f_7f7f_7f7f) + 0x0808_0808_0808_0808) - | (high & 0x8080_8080_8080_8080); + let high = high | 0x0808_0808_0808_0808; let mut control = [u8::MAX; 16]; control[..8].copy_from_slice(&low.to_le_bytes()); control[low_count..low_count + 8].copy_from_slice(&high.to_le_bytes()); @@ -6033,8 +6024,7 @@ impl Simd for WasmSimd128 { let low_count = low_mask.count_ones() as usize; let low = crate::support::COMPRESS_8_CONTROLS[low_mask]; let high = crate::support::COMPRESS_8_CONTROLS[high_mask]; - let high = ((high & 0x7f7f_7f7f_7f7f_7f7f) + 0x0808_0808_0808_0808) - | (high & 0x8080_8080_8080_8080); + let high = high | 0x0808_0808_0808_0808; let mut control = [u8::MAX; 16]; control[..8].copy_from_slice(&low.to_le_bytes()); control[low_count..low_count + 8].copy_from_slice(&high.to_le_bytes()); @@ -6068,8 +6058,7 @@ impl Simd for WasmSimd128 { let low = crate::support::EXPAND_8_CONTROLS[low_mask]; let high = crate::support::EXPAND_8_CONTROLS[high_mask]; let high_base = low_count as u64 * 0x0101_0101_0101_0101; - let high = - ((high & 0x7f7f_7f7f_7f7f_7f7f) + high_base) | (high & 0x8080_8080_8080_8080); + let high = high + high_base; let mut control = [0u8; 16]; control[..8].copy_from_slice(&low.to_le_bytes()); control[8..].copy_from_slice(&high.to_le_bytes()); @@ -6103,8 +6092,7 @@ impl Simd for WasmSimd128 { let low = crate::support::EXPAND_8_CONTROLS[low_mask]; let high = crate::support::EXPAND_8_CONTROLS[high_mask]; let high_base = low_count as u64 * 0x0101_0101_0101_0101; - let high = - ((high & 0x7f7f_7f7f_7f7f_7f7f) + high_base) | (high & 0x8080_8080_8080_8080); + let high = high + high_base; let mut control = [0u8; 16]; control[..8].copy_from_slice(&low.to_le_bytes()); control[8..].copy_from_slice(&high.to_le_bytes()); diff --git a/fearless_simd_gen/src/generic.rs b/fearless_simd_gen/src/generic.rs index 801d6e584..45497c95e 100644 --- a/fearless_simd_gen/src/generic.rs +++ b/fearless_simd_gen/src/generic.rs @@ -773,9 +773,9 @@ pub(crate) fn composed_compact_op(op: Op, ty: &VecType, options: CompactOptions) let low_count = #low_count; let low = crate::support::COMPRESS_8_CONTROLS[low_mask]; let high = crate::support::COMPRESS_8_CONTROLS[high_mask]; - let high = ((high & 0x7f7f_7f7f_7f7f_7f7f) - + 0x0808_0808_0808_0808) - | (high & 0x8080_8080_8080_8080); + // Selected indices are 0..7; OR adds eight while + // leaving inactive 0xff controls out of range. + let high = high | 0x0808_0808_0808_0808; let mut control = [u8::MAX; 16]; control[..8].copy_from_slice(&low.to_le_bytes()); control[low_count..low_count + 8] @@ -821,8 +821,9 @@ pub(crate) fn composed_compact_op(op: Op, ty: &VecType, options: CompactOptions) let low = crate::support::EXPAND_8_CONTROLS[low_mask]; let high = crate::support::EXPAND_8_CONTROLS[high_mask]; let high_base = low_count as u64 * 0x0101_0101_0101_0101; - let high = ((high & 0x7f7f_7f7f_7f7f_7f7f) + high_base) - | (high & 0x8080_8080_8080_8080); + // Adding at most eight preserves the inactive 0x80 + // controls' high bit without carrying between bytes. + let high = high + high_base; let mut control = [0u8; 16]; control[..8].copy_from_slice(&low.to_le_bytes()); control[8..].copy_from_slice(&high.to_le_bytes()); @@ -880,8 +881,9 @@ pub(crate) fn composed_compact_op(op: Op, ty: &VecType, options: CompactOptions) let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; let packed = crate::support::COMPRESS_8_CONTROLS[block_mask]; let base = (block * 8) as u64 * 0x0101_0101_0101_0101; - let adjusted = ((packed & 0x7f7f_7f7f_7f7f_7f7f) + base) - | (packed & 0x8080_8080_8080_8080); + // Block offsets are multiples of eight, disjoint from + // indices 0..7. Inactive 0xff controls stay out of range. + let adjusted = packed | base; let adjusted = adjusted.to_le_bytes(); let write_len = core::cmp::min(8, #len - output_lane); control[output_lane..output_lane + write_len] @@ -921,8 +923,10 @@ pub(crate) fn composed_compact_op(op: Op, ty: &VecType, options: CompactOptions) let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; let packed = crate::support::EXPAND_8_CONTROLS[block_mask]; let base = input_lane as u64 * 0x0101_0101_0101_0101; - let adjusted = ((packed & 0x7f7f_7f7f_7f7f_7f7f) + base) - | (packed & 0x8080_8080_8080_8080); + // The offset is at most 56 for a 64-byte vector. Adding + // it to inactive 0x80 controls keeps them out of range + // and cannot carry between bytes. + let adjusted = packed + base; let output_lane = block * 8; control[output_lane..output_lane + 8] .copy_from_slice(&adjusted.to_le_bytes()); From 6b27eeec5e8704219e0ad6bb9366ad082c097e0c Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Wed, 30 Sep 2026 11:54:06 +0100 Subject: [PATCH 5/8] Implement a dedicated 128-bit compress/expand path for wider elements for 1.3x to 1.8x speedup over the byte variant --- fearless_simd/src/generated/avx2.rs | 344 +++------ fearless_simd/src/generated/neon.rs | 728 ++++++++++++------ fearless_simd/src/generated/sse4_2.rs | 344 +++------ fearless_simd/src/generated/wasm.rs | 344 +++------ fearless_simd/src/support.rs | 57 ++ fearless_simd_gen/src/generic.rs | 67 ++ fearless_simd_gen/src/mk_neon.rs | 9 +- fearless_simd_gen/src/mk_wasm.rs | 15 +- fearless_simd_gen/src/mk_x86.rs | 15 +- .../tests/harness/ops/compress.rs | 8 + 10 files changed, 1019 insertions(+), 912 deletions(-) diff --git a/fearless_simd/src/generated/avx2.rs b/fearless_simd/src/generated/avx2.rs index 6c7051954..1139f9363 100644 --- a/fearless_simd/src/generated/avx2.rs +++ b/fearless_simd/src/generated/avx2.rs @@ -167,11 +167,9 @@ impl Simd for Avx2 { } #[inline(always)] fn compress_f32x4(self, values: f32x4, mask: mask32x4) -> f32x4 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn compress_merge_f32x4( @@ -180,23 +178,17 @@ impl Simd for Avx2 { mask: mask32x4, merge: f32x4, ) -> f32x4 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0); + let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control); + let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0)); + Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed)) } #[inline(always)] fn expand_f32x4(self, values: f32x4, mask: mask32x4) -> f32x4 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn expand_merge_f32x4( @@ -205,15 +197,10 @@ impl Simd for Avx2 { mask: mask32x4, merge: f32x4, ) -> f32x4 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0); + let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control)); + self.select_f32x4(mask, expanded, merge) } #[inline(always)] fn neg_f32x4(self, a: f32x4) -> f32x4 { @@ -2434,11 +2421,9 @@ impl Simd for Avx2 { } #[inline(always)] fn compress_i16x8(self, values: i16x8, mask: mask16x8) -> i16x8 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_16[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn compress_merge_i16x8( @@ -2447,23 +2432,17 @@ impl Simd for Avx2 { mask: mask16x8, merge: i16x8, ) -> i16x8 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_16[bits].0); + let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control); + let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0)); + Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed)) } #[inline(always)] fn expand_i16x8(self, values: i16x8, mask: mask16x8) -> i16x8 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_16[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn expand_merge_i16x8( @@ -2472,15 +2451,10 @@ impl Simd for Avx2 { mask: mask16x8, merge: i16x8, ) -> i16x8 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_16[bits].0); + let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control)); + self.select_i16x8(mask, expanded, merge) } #[inline(always)] fn count_ones_i16x8(self, a: i16x8) -> i16x8 { @@ -3032,11 +3006,9 @@ impl Simd for Avx2 { } #[inline(always)] fn compress_u16x8(self, values: u16x8, mask: mask16x8) -> u16x8 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_16[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn compress_merge_u16x8( @@ -3045,23 +3017,17 @@ impl Simd for Avx2 { mask: mask16x8, merge: u16x8, ) -> u16x8 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_16[bits].0); + let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control); + let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0)); + Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed)) } #[inline(always)] fn expand_u16x8(self, values: u16x8, mask: mask16x8) -> u16x8 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_16[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn expand_merge_u16x8( @@ -3070,15 +3036,10 @@ impl Simd for Avx2 { mask: mask16x8, merge: u16x8, ) -> u16x8 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_16[bits].0); + let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control)); + self.select_u16x8(mask, expanded, merge) } #[inline(always)] fn count_ones_u16x8(self, a: u16x8) -> u16x8 { @@ -3871,11 +3832,9 @@ impl Simd for Avx2 { } #[inline(always)] fn compress_i32x4(self, values: i32x4, mask: mask32x4) -> i32x4 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn compress_merge_i32x4( @@ -3884,23 +3843,17 @@ impl Simd for Avx2 { mask: mask32x4, merge: i32x4, ) -> i32x4 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0); + let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control); + let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0)); + Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed)) } #[inline(always)] fn expand_i32x4(self, values: i32x4, mask: mask32x4) -> i32x4 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn expand_merge_i32x4( @@ -3909,15 +3862,10 @@ impl Simd for Avx2 { mask: mask32x4, merge: i32x4, ) -> i32x4 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0); + let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control)); + self.select_i32x4(mask, expanded, merge) } #[inline(always)] fn count_ones_i32x4(self, a: i32x4) -> i32x4 { @@ -4460,11 +4408,9 @@ impl Simd for Avx2 { } #[inline(always)] fn compress_u32x4(self, values: u32x4, mask: mask32x4) -> u32x4 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn compress_merge_u32x4( @@ -4473,23 +4419,17 @@ impl Simd for Avx2 { mask: mask32x4, merge: u32x4, ) -> u32x4 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0); + let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control); + let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0)); + Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed)) } #[inline(always)] fn expand_u32x4(self, values: u32x4, mask: mask32x4) -> u32x4 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn expand_merge_u32x4( @@ -4498,15 +4438,10 @@ impl Simd for Avx2 { mask: mask32x4, merge: u32x4, ) -> u32x4 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0); + let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control)); + self.select_u32x4(mask, expanded, merge) } #[inline(always)] fn count_ones_u32x4(self, a: u32x4) -> u32x4 { @@ -5278,11 +5213,9 @@ impl Simd for Avx2 { } #[inline(always)] fn compress_f64x2(self, values: f64x2, mask: mask64x2) -> f64x2 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn compress_merge_f64x2( @@ -5291,23 +5224,17 @@ impl Simd for Avx2 { mask: mask64x2, merge: f64x2, ) -> f64x2 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0); + let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control); + let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0)); + Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed)) } #[inline(always)] fn expand_f64x2(self, values: f64x2, mask: mask64x2) -> f64x2 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn expand_merge_f64x2( @@ -5316,15 +5243,10 @@ impl Simd for Avx2 { mask: mask64x2, merge: f64x2, ) -> f64x2 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0); + let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control)); + self.select_f64x2(mask, expanded, merge) } #[inline(always)] fn neg_f64x2(self, a: f64x2) -> f64x2 { @@ -5970,11 +5892,9 @@ impl Simd for Avx2 { } #[inline(always)] fn compress_i64x2(self, values: i64x2, mask: mask64x2) -> i64x2 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn compress_merge_i64x2( @@ -5983,23 +5903,17 @@ impl Simd for Avx2 { mask: mask64x2, merge: i64x2, ) -> i64x2 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0); + let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control); + let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0)); + Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed)) } #[inline(always)] fn expand_i64x2(self, values: i64x2, mask: mask64x2) -> i64x2 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn expand_merge_i64x2( @@ -6008,15 +5922,10 @@ impl Simd for Avx2 { mask: mask64x2, merge: i64x2, ) -> i64x2 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0); + let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control)); + self.select_i64x2(mask, expanded, merge) } #[inline(always)] fn count_ones_i64x2(self, a: i64x2) -> i64x2 { @@ -6517,11 +6426,9 @@ impl Simd for Avx2 { } #[inline(always)] fn compress_u64x2(self, values: u64x2, mask: mask64x2) -> u64x2 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn compress_merge_u64x2( @@ -6530,23 +6437,17 @@ impl Simd for Avx2 { mask: mask64x2, merge: u64x2, ) -> u64x2 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0); + let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control); + let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0)); + Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed)) } #[inline(always)] fn expand_u64x2(self, values: u64x2, mask: mask64x2) -> u64x2 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn expand_merge_u64x2( @@ -6555,15 +6456,10 @@ impl Simd for Avx2 { mask: mask64x2, merge: u64x2, ) -> u64x2 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0); + let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control)); + self.select_u64x2(mask, expanded, merge) } #[inline(always)] fn count_ones_u64x2(self, a: u64x2) -> u64x2 { diff --git a/fearless_simd/src/generated/neon.rs b/fearless_simd/src/generated/neon.rs index c60f16444..37b4a22d4 100644 --- a/fearless_simd/src/generated/neon.rs +++ b/fearless_simd/src/generated/neon.rs @@ -160,11 +160,9 @@ impl Simd for Neon { } #[inline(always)] fn compress_f32x4(self, values: f32x4, mask: mask32x4) -> f32x4 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn compress_merge_f32x4( @@ -173,23 +171,40 @@ impl Simd for Neon { mask: mask32x4, merge: f32x4, ) -> f32x4 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0); + Bytes::from_bytes({ + crate::kernel!( + #[inline(always)] + fn merge_swizzle( + token: Neon, + values: u8x16, + control: u8x16, + merge: u8x16, + ) -> u8x16 { + let values: uint8x16_t = values.into(); + let control: uint8x16_t = control.into(); + let merge: uint8x16_t = merge.into(); + let result = vqtbx1q_u8(merge, values, control); + u8x16 { + val: crate::support::Aligned128(result), + simd: token, + } + } + ); + merge_swizzle( + self, + Bytes::to_bytes(values), + control, + Bytes::to_bytes(merge), + ) + }) } #[inline(always)] fn expand_f32x4(self, values: f32x4, mask: mask32x4) -> f32x4 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn expand_merge_f32x4( @@ -198,15 +213,34 @@ impl Simd for Neon { mask: mask32x4, merge: f32x4, ) -> f32x4 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0); + Bytes::from_bytes({ + crate::kernel!( + #[inline(always)] + fn merge_swizzle( + token: Neon, + values: u8x16, + control: u8x16, + merge: u8x16, + ) -> u8x16 { + let values: uint8x16_t = values.into(); + let control: uint8x16_t = control.into(); + let merge: uint8x16_t = merge.into(); + let result = vqtbx1q_u8(merge, values, control); + u8x16 { + val: crate::support::Aligned128(result), + simd: token, + } + } + ); + merge_swizzle( + self, + Bytes::to_bytes(values), + control, + Bytes::to_bytes(merge), + ) + }) } #[inline(always)] fn neg_f32x4(self, a: f32x4) -> f32x4 { @@ -1972,11 +2006,9 @@ impl Simd for Neon { } #[inline(always)] fn compress_i16x8(self, values: i16x8, mask: mask16x8) -> i16x8 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_16[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn compress_merge_i16x8( @@ -1985,23 +2017,40 @@ impl Simd for Neon { mask: mask16x8, merge: i16x8, ) -> i16x8 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_16[bits].0); + Bytes::from_bytes({ + crate::kernel!( + #[inline(always)] + fn merge_swizzle( + token: Neon, + values: u8x16, + control: u8x16, + merge: u8x16, + ) -> u8x16 { + let values: uint8x16_t = values.into(); + let control: uint8x16_t = control.into(); + let merge: uint8x16_t = merge.into(); + let result = vqtbx1q_u8(merge, values, control); + u8x16 { + val: crate::support::Aligned128(result), + simd: token, + } + } + ); + merge_swizzle( + self, + Bytes::to_bytes(values), + control, + Bytes::to_bytes(merge), + ) + }) } #[inline(always)] fn expand_i16x8(self, values: i16x8, mask: mask16x8) -> i16x8 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_16[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn expand_merge_i16x8( @@ -2010,15 +2059,34 @@ impl Simd for Neon { mask: mask16x8, merge: i16x8, ) -> i16x8 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_16[bits].0); + Bytes::from_bytes({ + crate::kernel!( + #[inline(always)] + fn merge_swizzle( + token: Neon, + values: u8x16, + control: u8x16, + merge: u8x16, + ) -> u8x16 { + let values: uint8x16_t = values.into(); + let control: uint8x16_t = control.into(); + let merge: uint8x16_t = merge.into(); + let result = vqtbx1q_u8(merge, values, control); + u8x16 { + val: crate::support::Aligned128(result), + simd: token, + } + } + ); + merge_swizzle( + self, + Bytes::to_bytes(values), + control, + Bytes::to_bytes(merge), + ) + }) } #[inline(always)] fn count_ones_i16x8(self, a: i16x8) -> i16x8 { @@ -2449,11 +2517,9 @@ impl Simd for Neon { } #[inline(always)] fn compress_u16x8(self, values: u16x8, mask: mask16x8) -> u16x8 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_16[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn compress_merge_u16x8( @@ -2462,23 +2528,40 @@ impl Simd for Neon { mask: mask16x8, merge: u16x8, ) -> u16x8 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_16[bits].0); + Bytes::from_bytes({ + crate::kernel!( + #[inline(always)] + fn merge_swizzle( + token: Neon, + values: u8x16, + control: u8x16, + merge: u8x16, + ) -> u8x16 { + let values: uint8x16_t = values.into(); + let control: uint8x16_t = control.into(); + let merge: uint8x16_t = merge.into(); + let result = vqtbx1q_u8(merge, values, control); + u8x16 { + val: crate::support::Aligned128(result), + simd: token, + } + } + ); + merge_swizzle( + self, + Bytes::to_bytes(values), + control, + Bytes::to_bytes(merge), + ) + }) } #[inline(always)] fn expand_u16x8(self, values: u16x8, mask: mask16x8) -> u16x8 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_16[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn expand_merge_u16x8( @@ -2487,15 +2570,34 @@ impl Simd for Neon { mask: mask16x8, merge: u16x8, ) -> u16x8 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_16[bits].0); + Bytes::from_bytes({ + crate::kernel!( + #[inline(always)] + fn merge_swizzle( + token: Neon, + values: u8x16, + control: u8x16, + merge: u8x16, + ) -> u8x16 { + let values: uint8x16_t = values.into(); + let control: uint8x16_t = control.into(); + let merge: uint8x16_t = merge.into(); + let result = vqtbx1q_u8(merge, values, control); + u8x16 { + val: crate::support::Aligned128(result), + simd: token, + } + } + ); + merge_swizzle( + self, + Bytes::to_bytes(values), + control, + Bytes::to_bytes(merge), + ) + }) } #[inline(always)] fn count_ones_u16x8(self, a: u16x8) -> u16x8 { @@ -3158,11 +3260,9 @@ impl Simd for Neon { } #[inline(always)] fn compress_i32x4(self, values: i32x4, mask: mask32x4) -> i32x4 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn compress_merge_i32x4( @@ -3171,23 +3271,40 @@ impl Simd for Neon { mask: mask32x4, merge: i32x4, ) -> i32x4 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0); + Bytes::from_bytes({ + crate::kernel!( + #[inline(always)] + fn merge_swizzle( + token: Neon, + values: u8x16, + control: u8x16, + merge: u8x16, + ) -> u8x16 { + let values: uint8x16_t = values.into(); + let control: uint8x16_t = control.into(); + let merge: uint8x16_t = merge.into(); + let result = vqtbx1q_u8(merge, values, control); + u8x16 { + val: crate::support::Aligned128(result), + simd: token, + } + } + ); + merge_swizzle( + self, + Bytes::to_bytes(values), + control, + Bytes::to_bytes(merge), + ) + }) } #[inline(always)] fn expand_i32x4(self, values: i32x4, mask: mask32x4) -> i32x4 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn expand_merge_i32x4( @@ -3196,19 +3313,38 @@ impl Simd for Neon { mask: mask32x4, merge: i32x4, ) -> i32x4 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) - } - #[inline(always)] - fn count_ones_i32x4(self, a: i32x4) -> i32x4 { - crate::kernel!( + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0); + Bytes::from_bytes({ + crate::kernel!( + #[inline(always)] + fn merge_swizzle( + token: Neon, + values: u8x16, + control: u8x16, + merge: u8x16, + ) -> u8x16 { + let values: uint8x16_t = values.into(); + let control: uint8x16_t = control.into(); + let merge: uint8x16_t = merge.into(); + let result = vqtbx1q_u8(merge, values, control); + u8x16 { + val: crate::support::Aligned128(result), + simd: token, + } + } + ); + merge_swizzle( + self, + Bytes::to_bytes(values), + control, + Bytes::to_bytes(merge), + ) + }) + } + #[inline(always)] + fn count_ones_i32x4(self, a: i32x4) -> i32x4 { + crate::kernel!( #[inline(always)] fn kernel(token: Neon, a: i32x4) -> i32x4 { vreinterpretq_s32_u32(vpaddlq_u16(vpaddlq_u8(vcntq_u8(vreinterpretq_u8_s32( @@ -3646,11 +3782,9 @@ impl Simd for Neon { } #[inline(always)] fn compress_u32x4(self, values: u32x4, mask: mask32x4) -> u32x4 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn compress_merge_u32x4( @@ -3659,23 +3793,40 @@ impl Simd for Neon { mask: mask32x4, merge: u32x4, ) -> u32x4 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0); + Bytes::from_bytes({ + crate::kernel!( + #[inline(always)] + fn merge_swizzle( + token: Neon, + values: u8x16, + control: u8x16, + merge: u8x16, + ) -> u8x16 { + let values: uint8x16_t = values.into(); + let control: uint8x16_t = control.into(); + let merge: uint8x16_t = merge.into(); + let result = vqtbx1q_u8(merge, values, control); + u8x16 { + val: crate::support::Aligned128(result), + simd: token, + } + } + ); + merge_swizzle( + self, + Bytes::to_bytes(values), + control, + Bytes::to_bytes(merge), + ) + }) } #[inline(always)] fn expand_u32x4(self, values: u32x4, mask: mask32x4) -> u32x4 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn expand_merge_u32x4( @@ -3684,15 +3835,34 @@ impl Simd for Neon { mask: mask32x4, merge: u32x4, ) -> u32x4 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0); + Bytes::from_bytes({ + crate::kernel!( + #[inline(always)] + fn merge_swizzle( + token: Neon, + values: u8x16, + control: u8x16, + merge: u8x16, + ) -> u8x16 { + let values: uint8x16_t = values.into(); + let control: uint8x16_t = control.into(); + let merge: uint8x16_t = merge.into(); + let result = vqtbx1q_u8(merge, values, control); + u8x16 { + val: crate::support::Aligned128(result), + simd: token, + } + } + ); + merge_swizzle( + self, + Bytes::to_bytes(values), + control, + Bytes::to_bytes(merge), + ) + }) } #[inline(always)] fn count_ones_u32x4(self, a: u32x4) -> u32x4 { @@ -4363,11 +4533,9 @@ impl Simd for Neon { } #[inline(always)] fn compress_f64x2(self, values: f64x2, mask: mask64x2) -> f64x2 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn compress_merge_f64x2( @@ -4376,23 +4544,40 @@ impl Simd for Neon { mask: mask64x2, merge: f64x2, ) -> f64x2 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0); + Bytes::from_bytes({ + crate::kernel!( + #[inline(always)] + fn merge_swizzle( + token: Neon, + values: u8x16, + control: u8x16, + merge: u8x16, + ) -> u8x16 { + let values: uint8x16_t = values.into(); + let control: uint8x16_t = control.into(); + let merge: uint8x16_t = merge.into(); + let result = vqtbx1q_u8(merge, values, control); + u8x16 { + val: crate::support::Aligned128(result), + simd: token, + } + } + ); + merge_swizzle( + self, + Bytes::to_bytes(values), + control, + Bytes::to_bytes(merge), + ) + }) } #[inline(always)] fn expand_f64x2(self, values: f64x2, mask: mask64x2) -> f64x2 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn expand_merge_f64x2( @@ -4401,15 +4586,34 @@ impl Simd for Neon { mask: mask64x2, merge: f64x2, ) -> f64x2 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0); + Bytes::from_bytes({ + crate::kernel!( + #[inline(always)] + fn merge_swizzle( + token: Neon, + values: u8x16, + control: u8x16, + merge: u8x16, + ) -> u8x16 { + let values: uint8x16_t = values.into(); + let control: uint8x16_t = control.into(); + let merge: uint8x16_t = merge.into(); + let result = vqtbx1q_u8(merge, values, control); + u8x16 { + val: crate::support::Aligned128(result), + simd: token, + } + } + ); + merge_swizzle( + self, + Bytes::to_bytes(values), + control, + Bytes::to_bytes(merge), + ) + }) } #[inline(always)] fn neg_f64x2(self, a: f64x2) -> f64x2 { @@ -4919,11 +5123,9 @@ impl Simd for Neon { } #[inline(always)] fn compress_i64x2(self, values: i64x2, mask: mask64x2) -> i64x2 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn compress_merge_i64x2( @@ -4932,23 +5134,40 @@ impl Simd for Neon { mask: mask64x2, merge: i64x2, ) -> i64x2 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0); + Bytes::from_bytes({ + crate::kernel!( + #[inline(always)] + fn merge_swizzle( + token: Neon, + values: u8x16, + control: u8x16, + merge: u8x16, + ) -> u8x16 { + let values: uint8x16_t = values.into(); + let control: uint8x16_t = control.into(); + let merge: uint8x16_t = merge.into(); + let result = vqtbx1q_u8(merge, values, control); + u8x16 { + val: crate::support::Aligned128(result), + simd: token, + } + } + ); + merge_swizzle( + self, + Bytes::to_bytes(values), + control, + Bytes::to_bytes(merge), + ) + }) } #[inline(always)] fn expand_i64x2(self, values: i64x2, mask: mask64x2) -> i64x2 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn expand_merge_i64x2( @@ -4957,15 +5176,34 @@ impl Simd for Neon { mask: mask64x2, merge: i64x2, ) -> i64x2 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0); + Bytes::from_bytes({ + crate::kernel!( + #[inline(always)] + fn merge_swizzle( + token: Neon, + values: u8x16, + control: u8x16, + merge: u8x16, + ) -> u8x16 { + let values: uint8x16_t = values.into(); + let control: uint8x16_t = control.into(); + let merge: uint8x16_t = merge.into(); + let result = vqtbx1q_u8(merge, values, control); + u8x16 { + val: crate::support::Aligned128(result), + simd: token, + } + } + ); + merge_swizzle( + self, + Bytes::to_bytes(values), + control, + Bytes::to_bytes(merge), + ) + }) } #[inline(always)] fn count_ones_i64x2(self, a: i64x2) -> i64x2 { @@ -5370,11 +5608,9 @@ impl Simd for Neon { } #[inline(always)] fn compress_u64x2(self, values: u64x2, mask: mask64x2) -> u64x2 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn compress_merge_u64x2( @@ -5383,23 +5619,40 @@ impl Simd for Neon { mask: mask64x2, merge: u64x2, ) -> u64x2 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0); + Bytes::from_bytes({ + crate::kernel!( + #[inline(always)] + fn merge_swizzle( + token: Neon, + values: u8x16, + control: u8x16, + merge: u8x16, + ) -> u8x16 { + let values: uint8x16_t = values.into(); + let control: uint8x16_t = control.into(); + let merge: uint8x16_t = merge.into(); + let result = vqtbx1q_u8(merge, values, control); + u8x16 { + val: crate::support::Aligned128(result), + simd: token, + } + } + ); + merge_swizzle( + self, + Bytes::to_bytes(values), + control, + Bytes::to_bytes(merge), + ) + }) } #[inline(always)] fn expand_u64x2(self, values: u64x2, mask: mask64x2) -> u64x2 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn expand_merge_u64x2( @@ -5408,15 +5661,34 @@ impl Simd for Neon { mask: mask64x2, merge: u64x2, ) -> u64x2 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0); + Bytes::from_bytes({ + crate::kernel!( + #[inline(always)] + fn merge_swizzle( + token: Neon, + values: u8x16, + control: u8x16, + merge: u8x16, + ) -> u8x16 { + let values: uint8x16_t = values.into(); + let control: uint8x16_t = control.into(); + let merge: uint8x16_t = merge.into(); + let result = vqtbx1q_u8(merge, values, control); + u8x16 { + val: crate::support::Aligned128(result), + simd: token, + } + } + ); + merge_swizzle( + self, + Bytes::to_bytes(values), + control, + Bytes::to_bytes(merge), + ) + }) } #[inline(always)] fn count_ones_u64x2(self, a: u64x2) -> u64x2 { diff --git a/fearless_simd/src/generated/sse4_2.rs b/fearless_simd/src/generated/sse4_2.rs index a8cd33f54..8c16a5971 100644 --- a/fearless_simd/src/generated/sse4_2.rs +++ b/fearless_simd/src/generated/sse4_2.rs @@ -171,11 +171,9 @@ impl Simd for Sse4_2 { } #[inline(always)] fn compress_f32x4(self, values: f32x4, mask: mask32x4) -> f32x4 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn compress_merge_f32x4( @@ -184,23 +182,17 @@ impl Simd for Sse4_2 { mask: mask32x4, merge: f32x4, ) -> f32x4 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0); + let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control); + let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0)); + Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed)) } #[inline(always)] fn expand_f32x4(self, values: f32x4, mask: mask32x4) -> f32x4 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn expand_merge_f32x4( @@ -209,15 +201,10 @@ impl Simd for Sse4_2 { mask: mask32x4, merge: f32x4, ) -> f32x4 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0); + let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control)); + self.select_f32x4(mask, expanded, merge) } #[inline(always)] fn neg_f32x4(self, a: f32x4) -> f32x4 { @@ -2438,11 +2425,9 @@ impl Simd for Sse4_2 { } #[inline(always)] fn compress_i16x8(self, values: i16x8, mask: mask16x8) -> i16x8 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_16[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn compress_merge_i16x8( @@ -2451,23 +2436,17 @@ impl Simd for Sse4_2 { mask: mask16x8, merge: i16x8, ) -> i16x8 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_16[bits].0); + let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control); + let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0)); + Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed)) } #[inline(always)] fn expand_i16x8(self, values: i16x8, mask: mask16x8) -> i16x8 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_16[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn expand_merge_i16x8( @@ -2476,15 +2455,10 @@ impl Simd for Sse4_2 { mask: mask16x8, merge: i16x8, ) -> i16x8 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_16[bits].0); + let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control)); + self.select_i16x8(mask, expanded, merge) } #[inline(always)] fn count_ones_i16x8(self, a: i16x8) -> i16x8 { @@ -3037,11 +3011,9 @@ impl Simd for Sse4_2 { } #[inline(always)] fn compress_u16x8(self, values: u16x8, mask: mask16x8) -> u16x8 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_16[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn compress_merge_u16x8( @@ -3050,23 +3022,17 @@ impl Simd for Sse4_2 { mask: mask16x8, merge: u16x8, ) -> u16x8 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_16[bits].0); + let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control); + let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0)); + Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed)) } #[inline(always)] fn expand_u16x8(self, values: u16x8, mask: mask16x8) -> u16x8 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_16[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn expand_merge_u16x8( @@ -3075,15 +3041,10 @@ impl Simd for Sse4_2 { mask: mask16x8, merge: u16x8, ) -> u16x8 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_16[bits].0); + let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control)); + self.select_u16x8(mask, expanded, merge) } #[inline(always)] fn count_ones_u16x8(self, a: u16x8) -> u16x8 { @@ -3874,11 +3835,9 @@ impl Simd for Sse4_2 { } #[inline(always)] fn compress_i32x4(self, values: i32x4, mask: mask32x4) -> i32x4 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn compress_merge_i32x4( @@ -3887,23 +3846,17 @@ impl Simd for Sse4_2 { mask: mask32x4, merge: i32x4, ) -> i32x4 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0); + let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control); + let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0)); + Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed)) } #[inline(always)] fn expand_i32x4(self, values: i32x4, mask: mask32x4) -> i32x4 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn expand_merge_i32x4( @@ -3912,15 +3865,10 @@ impl Simd for Sse4_2 { mask: mask32x4, merge: i32x4, ) -> i32x4 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0); + let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control)); + self.select_i32x4(mask, expanded, merge) } #[inline(always)] fn count_ones_i32x4(self, a: i32x4) -> i32x4 { @@ -4460,11 +4408,9 @@ impl Simd for Sse4_2 { } #[inline(always)] fn compress_u32x4(self, values: u32x4, mask: mask32x4) -> u32x4 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn compress_merge_u32x4( @@ -4473,23 +4419,17 @@ impl Simd for Sse4_2 { mask: mask32x4, merge: u32x4, ) -> u32x4 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0); + let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control); + let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0)); + Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed)) } #[inline(always)] fn expand_u32x4(self, values: u32x4, mask: mask32x4) -> u32x4 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn expand_merge_u32x4( @@ -4498,15 +4438,10 @@ impl Simd for Sse4_2 { mask: mask32x4, merge: u32x4, ) -> u32x4 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0); + let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control)); + self.select_u32x4(mask, expanded, merge) } #[inline(always)] fn count_ones_u32x4(self, a: u32x4) -> u32x4 { @@ -5272,11 +5207,9 @@ impl Simd for Sse4_2 { } #[inline(always)] fn compress_f64x2(self, values: f64x2, mask: mask64x2) -> f64x2 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn compress_merge_f64x2( @@ -5285,23 +5218,17 @@ impl Simd for Sse4_2 { mask: mask64x2, merge: f64x2, ) -> f64x2 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0); + let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control); + let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0)); + Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed)) } #[inline(always)] fn expand_f64x2(self, values: f64x2, mask: mask64x2) -> f64x2 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn expand_merge_f64x2( @@ -5310,15 +5237,10 @@ impl Simd for Sse4_2 { mask: mask64x2, merge: f64x2, ) -> f64x2 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0); + let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control)); + self.select_f64x2(mask, expanded, merge) } #[inline(always)] fn neg_f64x2(self, a: f64x2) -> f64x2 { @@ -5889,11 +5811,9 @@ impl Simd for Sse4_2 { } #[inline(always)] fn compress_i64x2(self, values: i64x2, mask: mask64x2) -> i64x2 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn compress_merge_i64x2( @@ -5902,23 +5822,17 @@ impl Simd for Sse4_2 { mask: mask64x2, merge: i64x2, ) -> i64x2 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0); + let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control); + let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0)); + Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed)) } #[inline(always)] fn expand_i64x2(self, values: i64x2, mask: mask64x2) -> i64x2 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn expand_merge_i64x2( @@ -5927,15 +5841,10 @@ impl Simd for Sse4_2 { mask: mask64x2, merge: i64x2, ) -> i64x2 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0); + let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control)); + self.select_i64x2(mask, expanded, merge) } #[inline(always)] fn count_ones_i64x2(self, a: i64x2) -> i64x2 { @@ -6436,11 +6345,9 @@ impl Simd for Sse4_2 { } #[inline(always)] fn compress_u64x2(self, values: u64x2, mask: mask64x2) -> u64x2 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn compress_merge_u64x2( @@ -6449,23 +6356,17 @@ impl Simd for Sse4_2 { mask: mask64x2, merge: u64x2, ) -> u64x2 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0); + let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control); + let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0)); + Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed)) } #[inline(always)] fn expand_u64x2(self, values: u64x2, mask: mask64x2) -> u64x2 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn expand_merge_u64x2( @@ -6474,15 +6375,10 @@ impl Simd for Sse4_2 { mask: mask64x2, merge: u64x2, ) -> u64x2 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0); + let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control)); + self.select_u64x2(mask, expanded, merge) } #[inline(always)] fn count_ones_u64x2(self, a: u64x2) -> u64x2 { diff --git a/fearless_simd/src/generated/wasm.rs b/fearless_simd/src/generated/wasm.rs index 972955eb0..0a5b07b8f 100644 --- a/fearless_simd/src/generated/wasm.rs +++ b/fearless_simd/src/generated/wasm.rs @@ -148,11 +148,9 @@ impl Simd for WasmSimd128 { } #[inline(always)] fn compress_f32x4(self, values: f32x4, mask: mask32x4) -> f32x4 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn compress_merge_f32x4( @@ -161,23 +159,17 @@ impl Simd for WasmSimd128 { mask: mask32x4, merge: f32x4, ) -> f32x4 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0); + let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control); + let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0)); + Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed)) } #[inline(always)] fn expand_f32x4(self, values: f32x4, mask: mask32x4) -> f32x4 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn expand_merge_f32x4( @@ -186,15 +178,10 @@ impl Simd for WasmSimd128 { mask: mask32x4, merge: f32x4, ) -> f32x4 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0); + let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control)); + self.select_f32x4(mask, expanded, merge) } #[inline(always)] fn neg_f32x4(self, a: f32x4) -> f32x4 { @@ -1638,11 +1625,9 @@ impl Simd for WasmSimd128 { } #[inline(always)] fn compress_i16x8(self, values: i16x8, mask: mask16x8) -> i16x8 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_16[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn compress_merge_i16x8( @@ -1651,23 +1636,17 @@ impl Simd for WasmSimd128 { mask: mask16x8, merge: i16x8, ) -> i16x8 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_16[bits].0); + let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control); + let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0)); + Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed)) } #[inline(always)] fn expand_i16x8(self, values: i16x8, mask: mask16x8) -> i16x8 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_16[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn expand_merge_i16x8( @@ -1676,15 +1655,10 @@ impl Simd for WasmSimd128 { mask: mask16x8, merge: i16x8, ) -> i16x8 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_16[bits].0); + let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control)); + self.select_i16x8(mask, expanded, merge) } #[inline(always)] fn count_ones_i16x8(self, a: i16x8) -> i16x8 { @@ -1979,11 +1953,9 @@ impl Simd for WasmSimd128 { } #[inline(always)] fn compress_u16x8(self, values: u16x8, mask: mask16x8) -> u16x8 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_16[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn compress_merge_u16x8( @@ -1992,23 +1964,17 @@ impl Simd for WasmSimd128 { mask: mask16x8, merge: u16x8, ) -> u16x8 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_16[bits].0); + let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control); + let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0)); + Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed)) } #[inline(always)] fn expand_u16x8(self, values: u16x8, mask: mask16x8) -> u16x8 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_16[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn expand_merge_u16x8( @@ -2017,15 +1983,10 @@ impl Simd for WasmSimd128 { mask: mask16x8, merge: u16x8, ) -> u16x8 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_16[bits].0); + let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control)); + self.select_u16x8(mask, expanded, merge) } #[inline(always)] fn count_ones_u16x8(self, a: u16x8) -> u16x8 { @@ -2460,11 +2421,9 @@ impl Simd for WasmSimd128 { } #[inline(always)] fn compress_i32x4(self, values: i32x4, mask: mask32x4) -> i32x4 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn compress_merge_i32x4( @@ -2473,23 +2432,17 @@ impl Simd for WasmSimd128 { mask: mask32x4, merge: i32x4, ) -> i32x4 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0); + let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control); + let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0)); + Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed)) } #[inline(always)] fn expand_i32x4(self, values: i32x4, mask: mask32x4) -> i32x4 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn expand_merge_i32x4( @@ -2498,15 +2451,10 @@ impl Simd for WasmSimd128 { mask: mask32x4, merge: i32x4, ) -> i32x4 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0); + let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control)); + self.select_i32x4(mask, expanded, merge) } #[inline(always)] fn count_ones_i32x4(self, a: i32x4) -> i32x4 { @@ -2796,11 +2744,9 @@ impl Simd for WasmSimd128 { } #[inline(always)] fn compress_u32x4(self, values: u32x4, mask: mask32x4) -> u32x4 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn compress_merge_u32x4( @@ -2809,23 +2755,17 @@ impl Simd for WasmSimd128 { mask: mask32x4, merge: u32x4, ) -> u32x4 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0); + let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control); + let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0)); + Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed)) } #[inline(always)] fn expand_u32x4(self, values: u32x4, mask: mask32x4) -> u32x4 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn expand_merge_u32x4( @@ -2834,15 +2774,10 @@ impl Simd for WasmSimd128 { mask: mask32x4, merge: u32x4, ) -> u32x4 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0); + let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control)); + self.select_u32x4(mask, expanded, merge) } #[inline(always)] fn count_ones_u32x4(self, a: u32x4) -> u32x4 { @@ -3266,11 +3201,9 @@ impl Simd for WasmSimd128 { } #[inline(always)] fn compress_f64x2(self, values: f64x2, mask: mask64x2) -> f64x2 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn compress_merge_f64x2( @@ -3279,23 +3212,17 @@ impl Simd for WasmSimd128 { mask: mask64x2, merge: f64x2, ) -> f64x2 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0); + let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control); + let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0)); + Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed)) } #[inline(always)] fn expand_f64x2(self, values: f64x2, mask: mask64x2) -> f64x2 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn expand_merge_f64x2( @@ -3304,15 +3231,10 @@ impl Simd for WasmSimd128 { mask: mask64x2, merge: f64x2, ) -> f64x2 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0); + let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control)); + self.select_f64x2(mask, expanded, merge) } #[inline(always)] fn neg_f64x2(self, a: f64x2) -> f64x2 { @@ -3661,11 +3583,9 @@ impl Simd for WasmSimd128 { } #[inline(always)] fn compress_i64x2(self, values: i64x2, mask: mask64x2) -> i64x2 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn compress_merge_i64x2( @@ -3674,23 +3594,17 @@ impl Simd for WasmSimd128 { mask: mask64x2, merge: i64x2, ) -> i64x2 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0); + let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control); + let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0)); + Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed)) } #[inline(always)] fn expand_i64x2(self, values: i64x2, mask: mask64x2) -> i64x2 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn expand_merge_i64x2( @@ -3699,15 +3613,10 @@ impl Simd for WasmSimd128 { mask: mask64x2, merge: i64x2, ) -> i64x2 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0); + let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control)); + self.select_i64x2(mask, expanded, merge) } #[inline(always)] fn count_ones_i64x2(self, a: i64x2) -> i64x2 { @@ -3991,11 +3900,9 @@ impl Simd for WasmSimd128 { } #[inline(always)] fn compress_u64x2(self, values: u64x2, mask: mask64x2) -> u64x2 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn compress_merge_u64x2( @@ -4004,23 +3911,17 @@ impl Simd for WasmSimd128 { mask: mask64x2, merge: u64x2, ) -> u64x2 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0); + let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control); + let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0)); + Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed)) } #[inline(always)] fn expand_u64x2(self, values: u64x2, mask: mask64x2) -> u64x2 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) } #[inline(always)] fn expand_merge_u64x2( @@ -4029,15 +3930,10 @@ impl Simd for WasmSimd128 { mask: mask64x2, merge: u64x2, ) -> u64x2 { - let mask = mask8x16 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_merge_u8x16( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0); + let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control)); + self.select_u64x2(mask, expanded, merge) } #[inline(always)] fn count_ones_u64x2(self, a: u64x2) -> u64x2 { diff --git a/fearless_simd/src/support.rs b/fearless_simd/src/support.rs index fb86edb30..9d234372c 100644 --- a/fearless_simd/src/support.rs +++ b/fearless_simd/src/support.rs @@ -64,6 +64,63 @@ pub(crate) fn cross_block_slide_blocks_at( [lo_block, hi_block] } +/// Byte-shuffle controls for whole 16-, 32-, or 64-bit elements in a 128-bit vector. +#[allow( + dead_code, + reason = "Used only by backends with native byte-table shuffles" +)] +pub(crate) mod compact_128 { + use super::Aligned128; + + #[allow( + clippy::cast_possible_truncation, + reason = "Byte indices are less than 16" + )] + const fn controls( + element_bytes: usize, + expand: bool, + ) -> [Aligned128<[u8; 16]>; N] { + assert!( + matches!(element_bytes, 2 | 4 | 8), + "unsupported element size" + ); + let lanes = 16 / element_bytes; + assert!( + N == 1 << lanes, + "one control row is required per selection mask" + ); + let mut table = [Aligned128([0xff; 16]); N]; + let mut mask = 0; + while mask < N { + let mut selected = 0; + let mut lane = 0; + while lane < lanes { + if mask & (1 << lane) != 0 { + let source = if expand { selected } else { lane }; + let destination = if expand { lane } else { selected }; + let mut byte = 0; + while byte < element_bytes { + table[mask].0[destination * element_bytes + byte] = + (source * element_bytes + byte) as u8; + byte += 1; + } + selected += 1; + } + lane += 1; + } + mask += 1; + } + table + } + + pub(crate) const COMPRESS_16: [Aligned128<[u8; 16]>; 256] = controls(2, false); + pub(crate) const EXPAND_16: [Aligned128<[u8; 16]>; 256] = controls(2, true); + pub(crate) const COMPRESS_32: [Aligned128<[u8; 16]>; 16] = controls(4, false); + pub(crate) const EXPAND_32: [Aligned128<[u8; 16]>; 16] = controls(4, true); + pub(crate) const COMPRESS_64: [Aligned128<[u8; 16]>; 4] = controls(8, false); + pub(crate) const EXPAND_64: [Aligned128<[u8; 16]>; 4] = controls(8, true); +} + const fn compact_control_table(expand: bool) -> [u64; 256] { // Expansion adds a lane offset to each byte of the packed control. Use 0x80 // for inactive lanes so these additions preserve the zeroing bit without diff --git a/fearless_simd_gen/src/generic.rs b/fearless_simd_gen/src/generic.rs index 45497c95e..eddf1a503 100644 --- a/fearless_simd_gen/src/generic.rs +++ b/fearless_simd_gen/src/generic.rs @@ -691,6 +691,73 @@ pub(crate) fn generic_op(op: &Op, ty: &VecType) -> TokenStream { pub(crate) type CompactMergeSwizzle = fn(&VecType, &TokenStream, &TokenStream, &TokenStream) -> TokenStream; +/// Compact whole elements with one table lookup and one native 128-bit byte shuffle. +pub(crate) fn compact_128_op( + op: Op, + ty: &VecType, + merge_swizzle: Option, +) -> TokenStream { + assert_eq!(ty.n_bits(), 128, "compact tables cover one 128-bit vector"); + assert!( + matches!(ty.scalar_bits, 16 | 32 | 64), + "compact tables require wider elements" + ); + let (expand, merge) = match op.sig { + OpSig::Compress { merge } => (false, merge), + OpSig::Expand { merge } => (true, merge), + _ => unreachable!("compact tables only implement compress/expand"), + }; + let method_sig = op.simd_trait_method_sig(ty); + let to_bitmask = generic_op_name("to_bitmask", &ty.mask_ty()); + let table = Ident::new( + &format!( + "{}_{}", + if expand { "EXPAND" } else { "COMPRESS" }, + ty.scalar_bits + ), + Span::call_site(), + ); + let lane_mask = Literal::usize_unsuffixed((1 << ty.len) - 1); + let finish = if !merge { + quote! { + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + } else if let Some(merge_swizzle) = merge_swizzle { + let result = merge_swizzle( + &ty.bytes_ty(), + "e! { Bytes::to_bytes(values) }, + "e! { control }, + "e! { Bytes::to_bytes(merge) }, + ); + quote! { Bytes::from_bytes(#result) } + } else if expand { + let select = generic_op_name("select", ty); + quote! { + let expanded = Bytes::from_bytes( + self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control), + ); + self.#select(mask, expanded, merge) + } + } else { + quote! { + let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control); + let inactive = self.simd_lt_i8x16( + i8x16::from_bytes(control), i8x16::splat(self, 0), + ); + Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed)) + } + }; + // Merge forms may use the imprecise shuffle: every out-of-range result is + // replaced by a merge byte. Compression's 0xff controls also identify its tail. + quote! { + #method_sig { + let bits = self.#to_bitmask(mask) as usize & #lane_mask; + let control = u8x16::simd_from(self, crate::support::compact_128::#table[bits].0); + #finish + } + } +} + /// Backend capabilities used by [`composed_compact_op`]. pub(crate) struct CompactOptions { /// Compact wide vectors a native 128-bit block at a time and splice them through memory. diff --git a/fearless_simd_gen/src/mk_neon.rs b/fearless_simd_gen/src/mk_neon.rs index 53369243a..3ab2bee59 100644 --- a/fearless_simd_gen/src/mk_neon.rs +++ b/fearless_simd_gen/src/mk_neon.rs @@ -5,8 +5,8 @@ use proc_macro2::{Ident, Literal, Span, TokenStream}; use quote::{ToTokens as _, format_ident, quote}; use crate::generic::{ - CompactOptions, byte_compact_op, composed_compact_op, count_zeros_method, fallback_method, - generic_classify, generic_mask_set, generic_op_name, integer_lane_mask_rotate, + CompactOptions, byte_compact_op, compact_128_op, composed_compact_op, count_zeros_method, + fallback_method, generic_classify, generic_mask_set, generic_op_name, integer_lane_mask_rotate, integer_lane_mask_splat_arg, reverse_method, reverse_vector_mask_method, }; use crate::level::Level; @@ -850,6 +850,11 @@ impl Level for Neon { } }) } + OpSig::Compress { .. } | OpSig::Expand { .. } + if vec_ty.n_bits() == 128 && matches!(vec_ty.scalar_bits, 16 | 32 | 64) => + { + compact_128_op(op, vec_ty, Some(Self::compact_merge_swizzle)) + } OpSig::Compress { .. } | OpSig::Expand { .. } if *vec_ty != vec_ty.bytes_ty() => { byte_compact_op(op, vec_ty) } diff --git a/fearless_simd_gen/src/mk_wasm.rs b/fearless_simd_gen/src/mk_wasm.rs index 6c54314e6..d0ca160ae 100644 --- a/fearless_simd_gen/src/mk_wasm.rs +++ b/fearless_simd_gen/src/mk_wasm.rs @@ -6,11 +6,11 @@ use quote::{format_ident, quote}; use crate::arch::wasm::{arch_prefix, v128_intrinsic}; use crate::generic::{ - CompactOptions, byte_compact_op, composed_compact_op, concat_swizzle_dyn_precise_body, - count_zeros_method, fallback_method, generic_block_combine, generic_block_split, - generic_classify, generic_mask_set, generic_op_name, integer_lane_mask_rotate, - integer_lane_mask_splat_arg, recursive_swizzle_dyn_precise_body, reverse_method, - reverse_vector_mask_method, + CompactOptions, byte_compact_op, compact_128_op, composed_compact_op, + concat_swizzle_dyn_precise_body, count_zeros_method, fallback_method, generic_block_combine, + generic_block_split, generic_classify, generic_mask_set, generic_op_name, + integer_lane_mask_rotate, integer_lane_mask_splat_arg, recursive_swizzle_dyn_precise_body, + reverse_method, reverse_vector_mask_method, }; use crate::level::Level; use crate::ops::{ @@ -1167,6 +1167,11 @@ impl Level for WasmSimd128 { } _ => unreachable!(), }, + OpSig::Compress { .. } | OpSig::Expand { .. } + if vec_ty.n_bits() == 128 && matches!(vec_ty.scalar_bits, 16 | 32 | 64) => + { + compact_128_op(op, vec_ty, None) + } OpSig::Compress { .. } | OpSig::Expand { .. } if *vec_ty != vec_ty.bytes_ty() => { byte_compact_op(op, vec_ty) } diff --git a/fearless_simd_gen/src/mk_x86.rs b/fearless_simd_gen/src/mk_x86.rs index 65c92a215..02467237a 100644 --- a/fearless_simd_gen/src/mk_x86.rs +++ b/fearless_simd_gen/src/mk_x86.rs @@ -7,11 +7,11 @@ use crate::arch::x86::{ unpack_intrinsic, }; use crate::generic::{ - byte_compact_op, concat_swizzle_dyn_precise_body, count_zeros_method, fallback_method, - generic_block_combine, generic_block_split, generic_classify, generic_mask_from_bitmask, - generic_mask_set, generic_op, generic_op_name, integer_lane_mask_rotate, - integer_lane_mask_splat_arg, recursive_swizzle_dyn_precise_body, reverse_method, - reverse_vector_mask_method, + byte_compact_op, compact_128_op, concat_swizzle_dyn_precise_body, count_zeros_method, + fallback_method, generic_block_combine, generic_block_split, generic_classify, + generic_mask_from_bitmask, generic_mask_set, generic_op, generic_op_name, + integer_lane_mask_rotate, integer_lane_mask_splat_arg, recursive_swizzle_dyn_precise_body, + reverse_method, reverse_vector_mask_method, }; use crate::level::Level; use crate::ops::{ @@ -332,6 +332,11 @@ impl Level for X86 { OpSig::Compress { .. } | OpSig::Expand { .. } => { if *self == Self::Avx512 { self.handle_avx512_compact_op(op, vec_ty) + } else if matches!(*self, Self::Sse4_2 | Self::Avx2) + && vec_ty.n_bits() == 128 + && matches!(vec_ty.scalar_bits, 16 | 32 | 64) + { + compact_128_op(op, vec_ty, None) } else if *vec_ty != vec_ty.bytes_ty() { byte_compact_op(op, vec_ty) } else if matches!(*self, Self::Sse4_2 | Self::Avx2) && vec_ty.len == 16 { diff --git a/fearless_simd_tests/tests/harness/ops/compress.rs b/fearless_simd_tests/tests/harness/ops/compress.rs index 8ba4a2c68..b8e18f3ba 100644 --- a/fearless_simd_tests/tests/harness/ops/compress.rs +++ b/fearless_simd_tests/tests/harness/ops/compress.rs @@ -399,6 +399,7 @@ fn compact_u16x8(simd: S) { .chain(single_lanes) .chain(prefixes) .chain(suffixes) + .chain(0..=all_lanes) { let mask = mask16x8::from_bitmask(simd, mask_bits); let mut expected_compress = [0; 8]; @@ -585,6 +586,7 @@ fn compact_i16x8(simd: S) { .chain(single_lanes) .chain(prefixes) .chain(suffixes) + .chain(0..=all_lanes) { let mask = mask16x8::from_bitmask(simd, mask_bits); let mut expected_compress = [0; 8]; @@ -771,6 +773,7 @@ fn compact_u32x4(simd: S) { .chain(single_lanes) .chain(prefixes) .chain(suffixes) + .chain(0..=all_lanes) { let mask = mask32x4::from_bitmask(simd, mask_bits); let mut expected_compress = [0; 4]; @@ -957,6 +960,7 @@ fn compact_i32x4(simd: S) { .chain(single_lanes) .chain(prefixes) .chain(suffixes) + .chain(0..=all_lanes) { let mask = mask32x4::from_bitmask(simd, mask_bits); let mut expected_compress = [0; 4]; @@ -1142,6 +1146,7 @@ fn compact_u64x2(simd: S) { .chain(single_lanes) .chain(prefixes) .chain(suffixes) + .chain(0..=all_lanes) { let mask = mask64x2::from_bitmask(simd, mask_bits); let mut expected_compress = [0; 2]; @@ -1326,6 +1331,7 @@ fn compact_i64x2(simd: S) { .chain(single_lanes) .chain(prefixes) .chain(suffixes) + .chain(0..=all_lanes) { let mask = mask64x2::from_bitmask(simd, mask_bits); let mut expected_compress = [0; 2]; @@ -1520,6 +1526,7 @@ fn compact_f32x4(simd: S) { .chain(single_lanes) .chain(prefixes) .chain(suffixes) + .chain(0..=all_lanes) { let mask = mask32x4::from_bitmask(simd, mask_bits); let mut expected_compress = [0; 4]; @@ -1808,6 +1815,7 @@ fn compact_f64x2(simd: S) { .chain(single_lanes) .chain(prefixes) .chain(suffixes) + .chain(0..=all_lanes) { let mask = mask64x2::from_bitmask(simd, mask_bits); let mut expected_compress = [0; 2]; From 12e56ed44a0bb44c3e20b48df470397dfe6be445 Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Wed, 30 Sep 2026 12:22:08 +0100 Subject: [PATCH 6/8] Implement a specialized AVX2 codepath for expand/compress for 32 bit values and larger that is 3x faster than the generic byte codepath --- fearless_simd/src/generated/avx2.rs | 1307 +++++++++-------- fearless_simd/src/generated/sse4_2.rs | 427 +++--- fearless_simd/src/support.rs | 55 + fearless_simd_gen/src/mk_x86.rs | 308 ++-- fearless_simd_gen/src/ops.rs | 42 +- .../tests/harness/ops/compress.rs | 51 + 6 files changed, 1153 insertions(+), 1037 deletions(-) diff --git a/fearless_simd/src/generated/avx2.rs b/fearless_simd/src/generated/avx2.rs index 1139f9363..fb2557403 100644 --- a/fearless_simd/src/generated/avx2.rs +++ b/fearless_simd/src/generated/avx2.rs @@ -1575,12 +1575,9 @@ impl Simd for Avx2 { let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64); let compressed = _mm_shuffle_epi8(compacted, splice); let count = mask_bits.count_ones() as usize; - let prefix = unsafe { - _mm_load_si128( - core::ptr::from_ref(&crate::support::COMPACT_PREFIX_MASKS[count]) - .cast::<__m128i>(), - ) - }; + let [prefix, _] = crate::transmute::checked_transmute_copy::<_, [__m128i; 2]>( + &crate::support::COMPACT_PREFIX_MASKS[count], + ); _mm_blendv_epi8(merge.into(), compressed, prefix).simd_into(token) } ); @@ -7190,11 +7187,20 @@ impl Simd for Avx2 { } #[inline(always)] fn compress_f32x8(self, values: f32x8, mask: mask32x8) -> f32x8 { - let mask = mask8x32 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: f32x8, mask: mask32x8) -> f32x8 { + let bits = token.to_bitmask_mask32x8(mask) as usize & 255; + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::COMPRESS_32[bits], + ); + let reordered = + _mm256_permutevar8x32_epi32(_mm256_castps_si256(values.into()), control); + let result = _mm256_andnot_si256(_mm256_srai_epi32::<31>(control), reordered); + _mm256_castsi256_ps(result).simd_into(token) + } + ); + kernel(self, values, mask) } #[inline(always)] fn compress_merge_f32x8( @@ -7203,23 +7209,43 @@ impl Simd for Avx2 { mask: mask32x8, merge: f32x8, ) -> f32x8 { - let mask = mask8x32 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_merge_u8x32( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: f32x8, + mask: mask32x8, + merge: f32x8, + ) -> f32x8 { + let bits = token.to_bitmask_mask32x8(mask) as usize & 255; + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::COMPRESS_32[bits], + ); + let reordered = + _mm256_permutevar8x32_epi32(_mm256_castps_si256(values.into()), control); + let result = + _mm256_blendv_epi8(reordered, _mm256_castps_si256(merge.into()), control); + _mm256_castsi256_ps(result).simd_into(token) + } + ); + kernel(self, values, mask, merge) } #[inline(always)] fn expand_f32x8(self, values: f32x8, mask: mask32x8) -> f32x8 { - let mask = mask8x32 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: f32x8, mask: mask32x8) -> f32x8 { + let bits = token.to_bitmask_mask32x8(mask) as usize & 255; + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::EXPAND_32[bits], + ); + let reordered = + _mm256_permutevar8x32_epi32(_mm256_castps_si256(values.into()), control); + let result = _mm256_and_si256(reordered, mask.into()); + _mm256_castsi256_ps(result).simd_into(token) + } + ); + kernel(self, values, mask) } #[inline(always)] fn expand_merge_f32x8( @@ -7228,15 +7254,26 @@ impl Simd for Avx2 { mask: mask32x8, merge: f32x8, ) -> f32x8 { - let mask = mask8x32 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_merge_u8x32( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: f32x8, + mask: mask32x8, + merge: f32x8, + ) -> f32x8 { + let bits = token.to_bitmask_mask32x8(mask) as usize & 255; + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::EXPAND_32[bits], + ); + let reordered = + _mm256_permutevar8x32_epi32(_mm256_castps_si256(values.into()), control); + let result = + _mm256_blendv_epi8(_mm256_castps_si256(merge.into()), reordered, mask.into()); + _mm256_castsi256_ps(result).simd_into(token) + } + ); + kernel(self, values, mask, merge) } #[inline(always)] fn neg_f32x8(self, a: f32x8) -> f32x8 { @@ -8471,15 +8508,9 @@ impl Simd for Avx2 { let splice_1 = _mm_set_epi64x((splice >> 64) as i64, splice as i64); let splice = _mm256_set_m128i(splice_1, splice_0); let compressed = _mm256_shuffle_epi8(compacted, splice); - let splice = unsafe { - _mm256_load_si256( - core::ptr::from_ref( - &crate::support::COMPACT_16_SPLICE_CONTROLS - [block_bits_0.count_ones() as usize], - ) - .cast::<__m256i>(), - ) - }; + let splice = crate::transmute::checked_transmute_copy( + &crate::support::COMPACT_16_SPLICE_CONTROLS[block_bits_0.count_ones() as usize], + ); let compressed_low = _mm256_permute2x128_si256::<0x80>(compressed, compressed); let compressed_high = _mm256_permute2x128_si256::<0x11>(compressed, compressed); let compressed_0 = @@ -8532,28 +8563,17 @@ impl Simd for Avx2 { let splice_1 = _mm_set_epi64x((splice >> 64) as i64, splice as i64); let splice = _mm256_set_m128i(splice_1, splice_0); let compressed = _mm256_shuffle_epi8(compacted, splice); - let splice = unsafe { - _mm256_load_si256( - core::ptr::from_ref( - &crate::support::COMPACT_16_SPLICE_CONTROLS - [block_bits_0.count_ones() as usize], - ) - .cast::<__m256i>(), - ) - }; + let splice = crate::transmute::checked_transmute_copy( + &crate::support::COMPACT_16_SPLICE_CONTROLS[block_bits_0.count_ones() as usize], + ); let compressed_low = _mm256_permute2x128_si256::<0x80>(compressed, compressed); let compressed_high = _mm256_permute2x128_si256::<0x11>(compressed, compressed); let compressed_0 = _mm256_or_si256(compressed_low, _mm256_shuffle_epi8(compressed_high, splice)); let output_lane = (block_bits_0.count_ones() + block_bits_1.count_ones()) as usize; - let prefix = unsafe { - _mm256_load_si256( - core::ptr::from_ref( - &crate::support::COMPACT_PREFIX_MASKS[output_lane.min(32)], - ) - .cast::<__m256i>(), - ) - }; + let prefix = crate::transmute::checked_transmute_copy( + &crate::support::COMPACT_PREFIX_MASKS[output_lane.min(32)], + ); let result_0 = _mm256_blendv_epi8(merge.val.0, compressed_0, prefix); result_0.simd_into(token) } @@ -8562,54 +8582,54 @@ impl Simd for Avx2 { } #[inline(always)] fn expand_u8x32(self, values: u8x32, mask: mask8x32) -> u8x32 { - #[inline] - #[target_feature(enable = "fxsr,sse4.2,cmpxchg16b,popcnt")] - unsafe fn kernel(token: Avx2, values: u8x32, mask: mask8x32) -> u8x32 { - let mask_bits = _mm_movemask_epi8(unsafe { - _mm_load_si128(core::ptr::from_ref(&mask.val.0).cast::<__m128i>().add(0)) - }) as u64 - | (_mm_movemask_epi8(unsafe { - _mm_load_si128(core::ptr::from_ref(&mask.val.0).cast::<__m128i>().add(1)) - }) as u64) - << 16; - let input_values = <[u8; 32]>::from(values); - let input = input_values.as_ptr(); - let mut output = [0u8; 32]; - let mut input_lane = 0usize; - let block_bits = (mask_bits & 0xffff) as usize; - let low_mask = (block_bits) & 0xff; - let high_mask = (block_bits) >> 8; - let low_count = low_mask.count_ones() as u64; - let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; - let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; - let high_base = low_count * 0x0101_0101_0101_0101; - let high_control = high_control + high_base; - let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); - let packed = unsafe { _mm_loadu_si128(input.add(input_lane).cast::<__m128i>()) }; - let expanded = _mm_shuffle_epi8(packed, control); - let result = expanded; - unsafe { - _mm_storeu_si128(output.as_mut_ptr().add(0).cast::<__m128i>(), result); - } - input_lane += block_bits.count_ones() as usize; - let block_bits = (mask_bits >> 16 & 0xffff) as usize; - let low_mask = (block_bits) & 0xff; - let high_mask = (block_bits) >> 8; - let low_count = low_mask.count_ones() as u64; - let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; - let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; - let high_base = low_count * 0x0101_0101_0101_0101; - let high_control = high_control + high_base; - let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); - let packed = unsafe { _mm_loadu_si128(input.add(input_lane).cast::<__m128i>()) }; - let expanded = _mm_shuffle_epi8(packed, control); - let result = expanded; - unsafe { - _mm_storeu_si128(output.as_mut_ptr().add(16).cast::<__m128i>(), result); - } - u8x32::simd_from(token, output) - } - unsafe { kernel(self, values, mask) } + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: u8x32, mask: mask8x32) -> u8x32 { + let input = <[u8; 32]>::from(values); + let mut output = [0u8; 32]; + let mut input_lane = 0; + let block_bits = _mm_movemask_epi8(_mm256_castsi256_si128(mask.val.0)) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = high_control + high_base; + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = crate::transmute::checked_transmute_copy( + input[input_lane..].first_chunk::<16>().unwrap(), + ); + let expanded = _mm_shuffle_epi8(packed, control); + let result = expanded; + crate::transmute::checked_transmute_store( + result, + output[0..].first_chunk_mut::<16>().unwrap(), + ); + input_lane += block_bits.count_ones() as usize; + let block_bits = + _mm_movemask_epi8(_mm256_extracti128_si256::<1>(mask.val.0)) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = high_control + high_base; + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = crate::transmute::checked_transmute_copy( + input[input_lane..].first_chunk::<16>().unwrap(), + ); + let expanded = _mm_shuffle_epi8(packed, control); + let result = expanded; + crate::transmute::checked_transmute_store( + result, + output[16..].first_chunk_mut::<16>().unwrap(), + ); + u8x32::simd_from(token, output) + } + ); + kernel(self, values, mask) } #[inline(always)] fn expand_merge_u8x32( @@ -8618,71 +8638,67 @@ impl Simd for Avx2 { mask: mask8x32, merge: u8x32, ) -> u8x32 { - #[inline] - #[target_feature(enable = "fxsr,sse4.2,cmpxchg16b,popcnt")] - unsafe fn kernel( - token: Avx2, - values: u8x32, - mask: mask8x32, - merge: u8x32, - ) -> u8x32 { - let mask_bits = _mm_movemask_epi8(unsafe { - _mm_load_si128(core::ptr::from_ref(&mask.val.0).cast::<__m128i>().add(0)) - }) as u64 - | (_mm_movemask_epi8(unsafe { - _mm_load_si128(core::ptr::from_ref(&mask.val.0).cast::<__m128i>().add(1)) - }) as u64) - << 16; - let input_values = <[u8; 32]>::from(values); - let input = input_values.as_ptr(); - let mut output = [0u8; 32]; - let mut input_lane = 0usize; - let block_bits = (mask_bits & 0xffff) as usize; - let low_mask = (block_bits) & 0xff; - let high_mask = (block_bits) >> 8; - let low_count = low_mask.count_ones() as u64; - let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; - let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; - let high_base = low_count * 0x0101_0101_0101_0101; - let high_control = high_control + high_base; - let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); - let packed = unsafe { _mm_loadu_si128(input.add(input_lane).cast::<__m128i>()) }; - let expanded = _mm_shuffle_epi8(packed, control); - let result = unsafe { - _mm_blendv_epi8( - _mm_load_si128(core::ptr::from_ref(&merge.val.0).cast::<__m128i>().add(0)), + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: u8x32, + mask: mask8x32, + merge: u8x32, + ) -> u8x32 { + let input = <[u8; 32]>::from(values); + let mut output = [0u8; 32]; + let mut input_lane = 0; + let block_bits = _mm_movemask_epi8(_mm256_castsi256_si128(mask.val.0)) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = high_control + high_base; + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = crate::transmute::checked_transmute_copy( + input[input_lane..].first_chunk::<16>().unwrap(), + ); + let expanded = _mm_shuffle_epi8(packed, control); + let result = _mm_blendv_epi8( + _mm256_castsi256_si128(merge.val.0), expanded, - _mm_load_si128(core::ptr::from_ref(&mask.val.0).cast::<__m128i>().add(0)), - ) - }; - unsafe { - _mm_storeu_si128(output.as_mut_ptr().add(0).cast::<__m128i>(), result); - } - input_lane += block_bits.count_ones() as usize; - let block_bits = (mask_bits >> 16 & 0xffff) as usize; - let low_mask = (block_bits) & 0xff; - let high_mask = (block_bits) >> 8; - let low_count = low_mask.count_ones() as u64; - let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; - let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; - let high_base = low_count * 0x0101_0101_0101_0101; - let high_control = high_control + high_base; - let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); - let packed = unsafe { _mm_loadu_si128(input.add(input_lane).cast::<__m128i>()) }; - let expanded = _mm_shuffle_epi8(packed, control); - let result = unsafe { - _mm_blendv_epi8( - _mm_load_si128(core::ptr::from_ref(&merge.val.0).cast::<__m128i>().add(1)), + _mm256_castsi256_si128(mask.val.0), + ); + crate::transmute::checked_transmute_store( + result, + output[0..].first_chunk_mut::<16>().unwrap(), + ); + input_lane += block_bits.count_ones() as usize; + let block_bits = + _mm_movemask_epi8(_mm256_extracti128_si256::<1>(mask.val.0)) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = high_control + high_base; + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = crate::transmute::checked_transmute_copy( + input[input_lane..].first_chunk::<16>().unwrap(), + ); + let expanded = _mm_shuffle_epi8(packed, control); + let result = _mm_blendv_epi8( + _mm256_extracti128_si256::<1>(merge.val.0), expanded, - _mm_load_si128(core::ptr::from_ref(&mask.val.0).cast::<__m128i>().add(1)), - ) - }; - unsafe { - _mm_storeu_si128(output.as_mut_ptr().add(16).cast::<__m128i>(), result); + _mm256_extracti128_si256::<1>(mask.val.0), + ); + crate::transmute::checked_transmute_store( + result, + output[16..].first_chunk_mut::<16>().unwrap(), + ); + u8x32::simd_from(token, output) } - u8x32::simd_from(token, output) - } - unsafe { kernel(self, values, mask, merge) } + ); + kernel(self, values, mask, merge) } #[inline(always)] fn count_ones_u8x32(self, a: u8x32) -> u8x32 { @@ -10770,11 +10786,19 @@ impl Simd for Avx2 { } #[inline(always)] fn compress_i32x8(self, values: i32x8, mask: mask32x8) -> i32x8 { - let mask = mask8x32 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: i32x8, mask: mask32x8) -> i32x8 { + let bits = token.to_bitmask_mask32x8(mask) as usize & 255; + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::COMPRESS_32[bits], + ); + let reordered = _mm256_permutevar8x32_epi32(values.into(), control); + let result = _mm256_andnot_si256(_mm256_srai_epi32::<31>(control), reordered); + result.simd_into(token) + } + ); + kernel(self, values, mask) } #[inline(always)] fn compress_merge_i32x8( @@ -10783,23 +10807,40 @@ impl Simd for Avx2 { mask: mask32x8, merge: i32x8, ) -> i32x8 { - let mask = mask8x32 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_merge_u8x32( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: i32x8, + mask: mask32x8, + merge: i32x8, + ) -> i32x8 { + let bits = token.to_bitmask_mask32x8(mask) as usize & 255; + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::COMPRESS_32[bits], + ); + let reordered = _mm256_permutevar8x32_epi32(values.into(), control); + let result = _mm256_blendv_epi8(reordered, merge.into(), control); + result.simd_into(token) + } + ); + kernel(self, values, mask, merge) } #[inline(always)] fn expand_i32x8(self, values: i32x8, mask: mask32x8) -> i32x8 { - let mask = mask8x32 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: i32x8, mask: mask32x8) -> i32x8 { + let bits = token.to_bitmask_mask32x8(mask) as usize & 255; + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::EXPAND_32[bits], + ); + let reordered = _mm256_permutevar8x32_epi32(values.into(), control); + let result = _mm256_and_si256(reordered, mask.into()); + result.simd_into(token) + } + ); + kernel(self, values, mask) } #[inline(always)] fn expand_merge_i32x8( @@ -10808,15 +10849,24 @@ impl Simd for Avx2 { mask: mask32x8, merge: i32x8, ) -> i32x8 { - let mask = mask8x32 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_merge_u8x32( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: i32x8, + mask: mask32x8, + merge: i32x8, + ) -> i32x8 { + let bits = token.to_bitmask_mask32x8(mask) as usize & 255; + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::EXPAND_32[bits], + ); + let reordered = _mm256_permutevar8x32_epi32(values.into(), control); + let result = _mm256_blendv_epi8(merge.into(), reordered, mask.into()); + result.simd_into(token) + } + ); + kernel(self, values, mask, merge) } #[inline(always)] fn count_ones_i32x8(self, a: i32x8) -> i32x8 { @@ -11305,11 +11355,19 @@ impl Simd for Avx2 { } #[inline(always)] fn compress_u32x8(self, values: u32x8, mask: mask32x8) -> u32x8 { - let mask = mask8x32 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: u32x8, mask: mask32x8) -> u32x8 { + let bits = token.to_bitmask_mask32x8(mask) as usize & 255; + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::COMPRESS_32[bits], + ); + let reordered = _mm256_permutevar8x32_epi32(values.into(), control); + let result = _mm256_andnot_si256(_mm256_srai_epi32::<31>(control), reordered); + result.simd_into(token) + } + ); + kernel(self, values, mask) } #[inline(always)] fn compress_merge_u32x8( @@ -11318,23 +11376,40 @@ impl Simd for Avx2 { mask: mask32x8, merge: u32x8, ) -> u32x8 { - let mask = mask8x32 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_merge_u8x32( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: u32x8, + mask: mask32x8, + merge: u32x8, + ) -> u32x8 { + let bits = token.to_bitmask_mask32x8(mask) as usize & 255; + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::COMPRESS_32[bits], + ); + let reordered = _mm256_permutevar8x32_epi32(values.into(), control); + let result = _mm256_blendv_epi8(reordered, merge.into(), control); + result.simd_into(token) + } + ); + kernel(self, values, mask, merge) } #[inline(always)] fn expand_u32x8(self, values: u32x8, mask: mask32x8) -> u32x8 { - let mask = mask8x32 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: u32x8, mask: mask32x8) -> u32x8 { + let bits = token.to_bitmask_mask32x8(mask) as usize & 255; + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::EXPAND_32[bits], + ); + let reordered = _mm256_permutevar8x32_epi32(values.into(), control); + let result = _mm256_and_si256(reordered, mask.into()); + result.simd_into(token) + } + ); + kernel(self, values, mask) } #[inline(always)] fn expand_merge_u32x8( @@ -11343,15 +11418,24 @@ impl Simd for Avx2 { mask: mask32x8, merge: u32x8, ) -> u32x8 { - let mask = mask8x32 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_merge_u8x32( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: u32x8, + mask: mask32x8, + merge: u32x8, + ) -> u32x8 { + let bits = token.to_bitmask_mask32x8(mask) as usize & 255; + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::EXPAND_32[bits], + ); + let reordered = _mm256_permutevar8x32_epi32(values.into(), control); + let result = _mm256_blendv_epi8(merge.into(), reordered, mask.into()); + result.simd_into(token) + } + ); + kernel(self, values, mask, merge) } #[inline(always)] fn count_ones_u32x8(self, a: u32x8) -> u32x8 { @@ -12084,12 +12168,21 @@ impl Simd for Avx2 { } #[inline(always)] fn compress_f64x4(self, values: f64x4, mask: mask64x4) -> f64x4 { - let mask = mask8x32 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) - } + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: f64x4, mask: mask64x4) -> f64x4 { + let bits = token.to_bitmask_mask64x4(mask) as usize & 15; + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::COMPRESS_64[bits], + ); + let reordered = + _mm256_permutevar8x32_epi32(_mm256_castpd_si256(values.into()), control); + let result = _mm256_andnot_si256(_mm256_srai_epi32::<31>(control), reordered); + _mm256_castsi256_pd(result).simd_into(token) + } + ); + kernel(self, values, mask) + } #[inline(always)] fn compress_merge_f64x4( self, @@ -12097,23 +12190,43 @@ impl Simd for Avx2 { mask: mask64x4, merge: f64x4, ) -> f64x4 { - let mask = mask8x32 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_merge_u8x32( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: f64x4, + mask: mask64x4, + merge: f64x4, + ) -> f64x4 { + let bits = token.to_bitmask_mask64x4(mask) as usize & 15; + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::COMPRESS_64[bits], + ); + let reordered = + _mm256_permutevar8x32_epi32(_mm256_castpd_si256(values.into()), control); + let result = + _mm256_blendv_epi8(reordered, _mm256_castpd_si256(merge.into()), control); + _mm256_castsi256_pd(result).simd_into(token) + } + ); + kernel(self, values, mask, merge) } #[inline(always)] fn expand_f64x4(self, values: f64x4, mask: mask64x4) -> f64x4 { - let mask = mask8x32 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: f64x4, mask: mask64x4) -> f64x4 { + let bits = token.to_bitmask_mask64x4(mask) as usize & 15; + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::EXPAND_64[bits], + ); + let reordered = + _mm256_permutevar8x32_epi32(_mm256_castpd_si256(values.into()), control); + let result = _mm256_and_si256(reordered, mask.into()); + _mm256_castsi256_pd(result).simd_into(token) + } + ); + kernel(self, values, mask) } #[inline(always)] fn expand_merge_f64x4( @@ -12122,15 +12235,26 @@ impl Simd for Avx2 { mask: mask64x4, merge: f64x4, ) -> f64x4 { - let mask = mask8x32 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_merge_u8x32( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: f64x4, + mask: mask64x4, + merge: f64x4, + ) -> f64x4 { + let bits = token.to_bitmask_mask64x4(mask) as usize & 15; + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::EXPAND_64[bits], + ); + let reordered = + _mm256_permutevar8x32_epi32(_mm256_castpd_si256(values.into()), control); + let result = + _mm256_blendv_epi8(_mm256_castpd_si256(merge.into()), reordered, mask.into()); + _mm256_castsi256_pd(result).simd_into(token) + } + ); + kernel(self, values, mask, merge) } #[inline(always)] fn neg_f64x4(self, a: f64x4) -> f64x4 { @@ -12706,11 +12830,19 @@ impl Simd for Avx2 { } #[inline(always)] fn compress_i64x4(self, values: i64x4, mask: mask64x4) -> i64x4 { - let mask = mask8x32 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: i64x4, mask: mask64x4) -> i64x4 { + let bits = token.to_bitmask_mask64x4(mask) as usize & 15; + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::COMPRESS_64[bits], + ); + let reordered = _mm256_permutevar8x32_epi32(values.into(), control); + let result = _mm256_andnot_si256(_mm256_srai_epi32::<31>(control), reordered); + result.simd_into(token) + } + ); + kernel(self, values, mask) } #[inline(always)] fn compress_merge_i64x4( @@ -12719,23 +12851,40 @@ impl Simd for Avx2 { mask: mask64x4, merge: i64x4, ) -> i64x4 { - let mask = mask8x32 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_merge_u8x32( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: i64x4, + mask: mask64x4, + merge: i64x4, + ) -> i64x4 { + let bits = token.to_bitmask_mask64x4(mask) as usize & 15; + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::COMPRESS_64[bits], + ); + let reordered = _mm256_permutevar8x32_epi32(values.into(), control); + let result = _mm256_blendv_epi8(reordered, merge.into(), control); + result.simd_into(token) + } + ); + kernel(self, values, mask, merge) } #[inline(always)] fn expand_i64x4(self, values: i64x4, mask: mask64x4) -> i64x4 { - let mask = mask8x32 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: i64x4, mask: mask64x4) -> i64x4 { + let bits = token.to_bitmask_mask64x4(mask) as usize & 15; + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::EXPAND_64[bits], + ); + let reordered = _mm256_permutevar8x32_epi32(values.into(), control); + let result = _mm256_and_si256(reordered, mask.into()); + result.simd_into(token) + } + ); + kernel(self, values, mask) } #[inline(always)] fn expand_merge_i64x4( @@ -12744,15 +12893,24 @@ impl Simd for Avx2 { mask: mask64x4, merge: i64x4, ) -> i64x4 { - let mask = mask8x32 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_merge_u8x32( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: i64x4, + mask: mask64x4, + merge: i64x4, + ) -> i64x4 { + let bits = token.to_bitmask_mask64x4(mask) as usize & 15; + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::EXPAND_64[bits], + ); + let reordered = _mm256_permutevar8x32_epi32(values.into(), control); + let result = _mm256_blendv_epi8(merge.into(), reordered, mask.into()); + result.simd_into(token) + } + ); + kernel(self, values, mask, merge) } #[inline(always)] fn count_ones_i64x4(self, a: i64x4) -> i64x4 { @@ -13227,11 +13385,19 @@ impl Simd for Avx2 { } #[inline(always)] fn compress_u64x4(self, values: u64x4, mask: mask64x4) -> u64x4 { - let mask = mask8x32 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: u64x4, mask: mask64x4) -> u64x4 { + let bits = token.to_bitmask_mask64x4(mask) as usize & 15; + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::COMPRESS_64[bits], + ); + let reordered = _mm256_permutevar8x32_epi32(values.into(), control); + let result = _mm256_andnot_si256(_mm256_srai_epi32::<31>(control), reordered); + result.simd_into(token) + } + ); + kernel(self, values, mask) } #[inline(always)] fn compress_merge_u64x4( @@ -13240,23 +13406,40 @@ impl Simd for Avx2 { mask: mask64x4, merge: u64x4, ) -> u64x4 { - let mask = mask8x32 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_merge_u8x32( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: u64x4, + mask: mask64x4, + merge: u64x4, + ) -> u64x4 { + let bits = token.to_bitmask_mask64x4(mask) as usize & 15; + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::COMPRESS_64[bits], + ); + let reordered = _mm256_permutevar8x32_epi32(values.into(), control); + let result = _mm256_blendv_epi8(reordered, merge.into(), control); + result.simd_into(token) + } + ); + kernel(self, values, mask, merge) } #[inline(always)] fn expand_u64x4(self, values: u64x4, mask: mask64x4) -> u64x4 { - let mask = mask8x32 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: u64x4, mask: mask64x4) -> u64x4 { + let bits = token.to_bitmask_mask64x4(mask) as usize & 15; + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::EXPAND_64[bits], + ); + let reordered = _mm256_permutevar8x32_epi32(values.into(), control); + let result = _mm256_and_si256(reordered, mask.into()); + result.simd_into(token) + } + ); + kernel(self, values, mask) } #[inline(always)] fn expand_merge_u64x4( @@ -13265,15 +13448,24 @@ impl Simd for Avx2 { mask: mask64x4, merge: u64x4, ) -> u64x4 { - let mask = mask8x32 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_merge_u8x32( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: u64x4, + mask: mask64x4, + merge: u64x4, + ) -> u64x4 { + let bits = token.to_bitmask_mask64x4(mask) as usize & 15; + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::EXPAND_64[bits], + ); + let reordered = _mm256_permutevar8x32_epi32(values.into(), control); + let result = _mm256_blendv_epi8(merge.into(), reordered, mask.into()); + result.simd_into(token) + } + ); + kernel(self, values, mask, merge) } #[inline(always)] fn count_ones_u64x4(self, a: u64x4) -> u64x4 { @@ -14308,15 +14500,9 @@ impl Simd for Avx2 { let splice_1 = _mm_set_epi64x((splice >> 64) as i64, splice as i64); let splice = _mm256_set_m128i(splice_1, splice_0); let compressed = _mm256_shuffle_epi8(compacted, splice); - let splice = unsafe { - _mm256_load_si256( - core::ptr::from_ref( - &crate::support::COMPACT_16_SPLICE_CONTROLS - [block_bits_0.count_ones() as usize], - ) - .cast::<__m256i>(), - ) - }; + let splice = crate::transmute::checked_transmute_copy( + &crate::support::COMPACT_16_SPLICE_CONTROLS[block_bits_0.count_ones() as usize], + ); let compressed_low = _mm256_permute2x128_si256::<0x80>(compressed, compressed); let compressed_high = _mm256_permute2x128_si256::<0x11>(compressed, compressed); let compressed_0 = @@ -14347,33 +14533,19 @@ impl Simd for Avx2 { let splice_3 = _mm_set_epi64x((splice >> 64) as i64, splice as i64); let splice = _mm256_set_m128i(splice_3, splice_2); let compressed = _mm256_shuffle_epi8(compacted, splice); - let splice = unsafe { - _mm256_load_si256( - core::ptr::from_ref( - &crate::support::COMPACT_16_SPLICE_CONTROLS - [block_bits_2.count_ones() as usize], - ) - .cast::<__m256i>(), - ) - }; + let splice = crate::transmute::checked_transmute_copy( + &crate::support::COMPACT_16_SPLICE_CONTROLS[block_bits_2.count_ones() as usize], + ); let compressed_low = _mm256_permute2x128_si256::<0x80>(compressed, compressed); let compressed_high = _mm256_permute2x128_si256::<0x11>(compressed, compressed); let compressed_1 = _mm256_or_si256(compressed_low, _mm256_shuffle_epi8(compressed_high, splice)); let first_count = (block_bits_0.count_ones() + block_bits_1.count_ones()) as usize; let controls = &crate::support::COMPACT_32_STITCH_CONTROLS[first_count]; - let left_same = unsafe { - _mm256_load_si256(core::ptr::from_ref(&controls.left_same).cast::<__m256i>()) - }; - let left_cross = unsafe { - _mm256_load_si256(core::ptr::from_ref(&controls.left_cross).cast::<__m256i>()) - }; - let right_same = unsafe { - _mm256_load_si256(core::ptr::from_ref(&controls.right_same).cast::<__m256i>()) - }; - let right_cross = unsafe { - _mm256_load_si256(core::ptr::from_ref(&controls.right_cross).cast::<__m256i>()) - }; + let left_same = crate::transmute::checked_transmute_copy(&controls.left_same); + let left_cross = crate::transmute::checked_transmute_copy(&controls.left_cross); + let right_same = crate::transmute::checked_transmute_copy(&controls.right_same); + let right_cross = crate::transmute::checked_transmute_copy(&controls.right_cross); let low_to_high = _mm256_permute2x128_si256::<0x08>(compressed_1, compressed_1); let shifted_left = _mm256_or_si256( _mm256_shuffle_epi8(compressed_1, left_same), @@ -14454,15 +14626,9 @@ impl Simd for Avx2 { let splice_1 = _mm_set_epi64x((splice >> 64) as i64, splice as i64); let splice = _mm256_set_m128i(splice_1, splice_0); let compressed = _mm256_shuffle_epi8(compacted, splice); - let splice = unsafe { - _mm256_load_si256( - core::ptr::from_ref( - &crate::support::COMPACT_16_SPLICE_CONTROLS - [block_bits_0.count_ones() as usize], - ) - .cast::<__m256i>(), - ) - }; + let splice = crate::transmute::checked_transmute_copy( + &crate::support::COMPACT_16_SPLICE_CONTROLS[block_bits_0.count_ones() as usize], + ); let compressed_low = _mm256_permute2x128_si256::<0x80>(compressed, compressed); let compressed_high = _mm256_permute2x128_si256::<0x11>(compressed, compressed); let compressed_0 = @@ -14493,33 +14659,19 @@ impl Simd for Avx2 { let splice_3 = _mm_set_epi64x((splice >> 64) as i64, splice as i64); let splice = _mm256_set_m128i(splice_3, splice_2); let compressed = _mm256_shuffle_epi8(compacted, splice); - let splice = unsafe { - _mm256_load_si256( - core::ptr::from_ref( - &crate::support::COMPACT_16_SPLICE_CONTROLS - [block_bits_2.count_ones() as usize], - ) - .cast::<__m256i>(), - ) - }; + let splice = crate::transmute::checked_transmute_copy( + &crate::support::COMPACT_16_SPLICE_CONTROLS[block_bits_2.count_ones() as usize], + ); let compressed_low = _mm256_permute2x128_si256::<0x80>(compressed, compressed); let compressed_high = _mm256_permute2x128_si256::<0x11>(compressed, compressed); let compressed_1 = _mm256_or_si256(compressed_low, _mm256_shuffle_epi8(compressed_high, splice)); let first_count = (block_bits_0.count_ones() + block_bits_1.count_ones()) as usize; let controls = &crate::support::COMPACT_32_STITCH_CONTROLS[first_count]; - let left_same = unsafe { - _mm256_load_si256(core::ptr::from_ref(&controls.left_same).cast::<__m256i>()) - }; - let left_cross = unsafe { - _mm256_load_si256(core::ptr::from_ref(&controls.left_cross).cast::<__m256i>()) - }; - let right_same = unsafe { - _mm256_load_si256(core::ptr::from_ref(&controls.right_same).cast::<__m256i>()) - }; - let right_cross = unsafe { - _mm256_load_si256(core::ptr::from_ref(&controls.right_cross).cast::<__m256i>()) - }; + let left_same = crate::transmute::checked_transmute_copy(&controls.left_same); + let left_cross = crate::transmute::checked_transmute_copy(&controls.left_cross); + let right_same = crate::transmute::checked_transmute_copy(&controls.right_same); + let right_cross = crate::transmute::checked_transmute_copy(&controls.right_cross); let low_to_high = _mm256_permute2x128_si256::<0x08>(compressed_1, compressed_1); let shifted_left = _mm256_or_si256( _mm256_shuffle_epi8(compressed_1, left_same), @@ -14532,24 +14684,13 @@ impl Simd for Avx2 { _mm256_shuffle_epi8(high_to_low, right_cross), ); let output_lane = mask_bits.count_ones() as usize; - let prefix = unsafe { - _mm256_load_si256( - core::ptr::from_ref( - &crate::support::COMPACT_PREFIX_MASKS[output_lane.min(32)], - ) - .cast::<__m256i>(), - ) - }; + let prefix = crate::transmute::checked_transmute_copy( + &crate::support::COMPACT_PREFIX_MASKS[output_lane.min(32)], + ); let result_0 = _mm256_blendv_epi8(merge.val.0[0], compressed_0, prefix); - let prefix = unsafe { - _mm256_load_si256( - core::ptr::from_ref( - &crate::support::COMPACT_PREFIX_MASKS - [output_lane.saturating_sub(32).min(32)], - ) - .cast::<__m256i>(), - ) - }; + let prefix = crate::transmute::checked_transmute_copy( + &crate::support::COMPACT_PREFIX_MASKS[output_lane.saturating_sub(32).min(32)], + ); let result_1 = _mm256_blendv_epi8(merge.val.0[1], compressed_1, prefix); u8x64 { val: crate::support::Aligned512([result_0, result_1]), @@ -14561,94 +14702,93 @@ impl Simd for Avx2 { } #[inline(always)] fn expand_u8x64(self, values: u8x64, mask: mask8x64) -> u8x64 { - #[inline] - #[target_feature(enable = "fxsr,sse4.2,cmpxchg16b,popcnt")] - unsafe fn kernel(token: Avx2, values: u8x64, mask: mask8x64) -> u8x64 { - let mask_bits = _mm_movemask_epi8(unsafe { - _mm_load_si128(core::ptr::from_ref(&mask.val.0).cast::<__m128i>().add(0)) - }) as u64 - | (_mm_movemask_epi8(unsafe { - _mm_load_si128(core::ptr::from_ref(&mask.val.0).cast::<__m128i>().add(1)) - }) as u64) - << 16 - | (_mm_movemask_epi8(unsafe { - _mm_load_si128(core::ptr::from_ref(&mask.val.0).cast::<__m128i>().add(2)) - }) as u64) - << 32 - | (_mm_movemask_epi8(unsafe { - _mm_load_si128(core::ptr::from_ref(&mask.val.0).cast::<__m128i>().add(3)) - }) as u64) - << 48; - let input_values = <[u8; 64]>::from(values); - let input = input_values.as_ptr(); - let mut output = [0u8; 64]; - let mut input_lane = 0usize; - let block_bits = (mask_bits & 0xffff) as usize; - let low_mask = (block_bits) & 0xff; - let high_mask = (block_bits) >> 8; - let low_count = low_mask.count_ones() as u64; - let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; - let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; - let high_base = low_count * 0x0101_0101_0101_0101; - let high_control = high_control + high_base; - let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); - let packed = unsafe { _mm_loadu_si128(input.add(input_lane).cast::<__m128i>()) }; - let expanded = _mm_shuffle_epi8(packed, control); - let result = expanded; - unsafe { - _mm_storeu_si128(output.as_mut_ptr().add(0).cast::<__m128i>(), result); - } - input_lane += block_bits.count_ones() as usize; - let block_bits = (mask_bits >> 16 & 0xffff) as usize; - let low_mask = (block_bits) & 0xff; - let high_mask = (block_bits) >> 8; - let low_count = low_mask.count_ones() as u64; - let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; - let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; - let high_base = low_count * 0x0101_0101_0101_0101; - let high_control = high_control + high_base; - let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); - let packed = unsafe { _mm_loadu_si128(input.add(input_lane).cast::<__m128i>()) }; - let expanded = _mm_shuffle_epi8(packed, control); - let result = expanded; - unsafe { - _mm_storeu_si128(output.as_mut_ptr().add(16).cast::<__m128i>(), result); - } - input_lane += block_bits.count_ones() as usize; - let block_bits = (mask_bits >> 32 & 0xffff) as usize; - let low_mask = (block_bits) & 0xff; - let high_mask = (block_bits) >> 8; - let low_count = low_mask.count_ones() as u64; - let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; - let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; - let high_base = low_count * 0x0101_0101_0101_0101; - let high_control = high_control + high_base; - let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); - let packed = unsafe { _mm_loadu_si128(input.add(input_lane).cast::<__m128i>()) }; - let expanded = _mm_shuffle_epi8(packed, control); - let result = expanded; - unsafe { - _mm_storeu_si128(output.as_mut_ptr().add(32).cast::<__m128i>(), result); - } - input_lane += block_bits.count_ones() as usize; - let block_bits = (mask_bits >> 48 & 0xffff) as usize; - let low_mask = (block_bits) & 0xff; - let high_mask = (block_bits) >> 8; - let low_count = low_mask.count_ones() as u64; - let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; - let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; - let high_base = low_count * 0x0101_0101_0101_0101; - let high_control = high_control + high_base; - let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); - let packed = unsafe { _mm_loadu_si128(input.add(input_lane).cast::<__m128i>()) }; - let expanded = _mm_shuffle_epi8(packed, control); - let result = expanded; - unsafe { - _mm_storeu_si128(output.as_mut_ptr().add(48).cast::<__m128i>(), result); - } - u8x64::simd_from(token, output) - } - unsafe { kernel(self, values, mask) } + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: u8x64, mask: mask8x64) -> u8x64 { + let input = <[u8; 64]>::from(values); + let mut output = [0u8; 64]; + let mut input_lane = 0; + let block_bits = _mm_movemask_epi8(_mm256_castsi256_si128(mask.val.0[0])) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = high_control + high_base; + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = crate::transmute::checked_transmute_copy( + input[input_lane..].first_chunk::<16>().unwrap(), + ); + let expanded = _mm_shuffle_epi8(packed, control); + let result = expanded; + crate::transmute::checked_transmute_store( + result, + output[0..].first_chunk_mut::<16>().unwrap(), + ); + input_lane += block_bits.count_ones() as usize; + let block_bits = + _mm_movemask_epi8(_mm256_extracti128_si256::<1>(mask.val.0[0])) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = high_control + high_base; + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = crate::transmute::checked_transmute_copy( + input[input_lane..].first_chunk::<16>().unwrap(), + ); + let expanded = _mm_shuffle_epi8(packed, control); + let result = expanded; + crate::transmute::checked_transmute_store( + result, + output[16..].first_chunk_mut::<16>().unwrap(), + ); + input_lane += block_bits.count_ones() as usize; + let block_bits = _mm_movemask_epi8(_mm256_castsi256_si128(mask.val.0[1])) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = high_control + high_base; + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = crate::transmute::checked_transmute_copy( + input[input_lane..].first_chunk::<16>().unwrap(), + ); + let expanded = _mm_shuffle_epi8(packed, control); + let result = expanded; + crate::transmute::checked_transmute_store( + result, + output[32..].first_chunk_mut::<16>().unwrap(), + ); + input_lane += block_bits.count_ones() as usize; + let block_bits = + _mm_movemask_epi8(_mm256_extracti128_si256::<1>(mask.val.0[1])) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = high_control + high_base; + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = crate::transmute::checked_transmute_copy( + input[input_lane..].first_chunk::<16>().unwrap(), + ); + let expanded = _mm_shuffle_epi8(packed, control); + let result = expanded; + crate::transmute::checked_transmute_store( + result, + output[48..].first_chunk_mut::<16>().unwrap(), + ); + u8x64::simd_from(token, output) + } + ); + kernel(self, values, mask) } #[inline(always)] fn expand_merge_u8x64( @@ -14657,123 +14797,114 @@ impl Simd for Avx2 { mask: mask8x64, merge: u8x64, ) -> u8x64 { - #[inline] - #[target_feature(enable = "fxsr,sse4.2,cmpxchg16b,popcnt")] - unsafe fn kernel( - token: Avx2, - values: u8x64, - mask: mask8x64, - merge: u8x64, - ) -> u8x64 { - let mask_bits = _mm_movemask_epi8(unsafe { - _mm_load_si128(core::ptr::from_ref(&mask.val.0).cast::<__m128i>().add(0)) - }) as u64 - | (_mm_movemask_epi8(unsafe { - _mm_load_si128(core::ptr::from_ref(&mask.val.0).cast::<__m128i>().add(1)) - }) as u64) - << 16 - | (_mm_movemask_epi8(unsafe { - _mm_load_si128(core::ptr::from_ref(&mask.val.0).cast::<__m128i>().add(2)) - }) as u64) - << 32 - | (_mm_movemask_epi8(unsafe { - _mm_load_si128(core::ptr::from_ref(&mask.val.0).cast::<__m128i>().add(3)) - }) as u64) - << 48; - let input_values = <[u8; 64]>::from(values); - let input = input_values.as_ptr(); - let mut output = [0u8; 64]; - let mut input_lane = 0usize; - let block_bits = (mask_bits & 0xffff) as usize; - let low_mask = (block_bits) & 0xff; - let high_mask = (block_bits) >> 8; - let low_count = low_mask.count_ones() as u64; - let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; - let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; - let high_base = low_count * 0x0101_0101_0101_0101; - let high_control = high_control + high_base; - let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); - let packed = unsafe { _mm_loadu_si128(input.add(input_lane).cast::<__m128i>()) }; - let expanded = _mm_shuffle_epi8(packed, control); - let result = unsafe { - _mm_blendv_epi8( - _mm_load_si128(core::ptr::from_ref(&merge.val.0).cast::<__m128i>().add(0)), + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: u8x64, + mask: mask8x64, + merge: u8x64, + ) -> u8x64 { + let input = <[u8; 64]>::from(values); + let mut output = [0u8; 64]; + let mut input_lane = 0; + let block_bits = _mm_movemask_epi8(_mm256_castsi256_si128(mask.val.0[0])) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = high_control + high_base; + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = crate::transmute::checked_transmute_copy( + input[input_lane..].first_chunk::<16>().unwrap(), + ); + let expanded = _mm_shuffle_epi8(packed, control); + let result = _mm_blendv_epi8( + _mm256_castsi256_si128(merge.val.0[0]), expanded, - _mm_load_si128(core::ptr::from_ref(&mask.val.0).cast::<__m128i>().add(0)), - ) - }; - unsafe { - _mm_storeu_si128(output.as_mut_ptr().add(0).cast::<__m128i>(), result); - } - input_lane += block_bits.count_ones() as usize; - let block_bits = (mask_bits >> 16 & 0xffff) as usize; - let low_mask = (block_bits) & 0xff; - let high_mask = (block_bits) >> 8; - let low_count = low_mask.count_ones() as u64; - let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; - let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; - let high_base = low_count * 0x0101_0101_0101_0101; - let high_control = high_control + high_base; - let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); - let packed = unsafe { _mm_loadu_si128(input.add(input_lane).cast::<__m128i>()) }; - let expanded = _mm_shuffle_epi8(packed, control); - let result = unsafe { - _mm_blendv_epi8( - _mm_load_si128(core::ptr::from_ref(&merge.val.0).cast::<__m128i>().add(1)), + _mm256_castsi256_si128(mask.val.0[0]), + ); + crate::transmute::checked_transmute_store( + result, + output[0..].first_chunk_mut::<16>().unwrap(), + ); + input_lane += block_bits.count_ones() as usize; + let block_bits = + _mm_movemask_epi8(_mm256_extracti128_si256::<1>(mask.val.0[0])) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = high_control + high_base; + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = crate::transmute::checked_transmute_copy( + input[input_lane..].first_chunk::<16>().unwrap(), + ); + let expanded = _mm_shuffle_epi8(packed, control); + let result = _mm_blendv_epi8( + _mm256_extracti128_si256::<1>(merge.val.0[0]), expanded, - _mm_load_si128(core::ptr::from_ref(&mask.val.0).cast::<__m128i>().add(1)), - ) - }; - unsafe { - _mm_storeu_si128(output.as_mut_ptr().add(16).cast::<__m128i>(), result); - } - input_lane += block_bits.count_ones() as usize; - let block_bits = (mask_bits >> 32 & 0xffff) as usize; - let low_mask = (block_bits) & 0xff; - let high_mask = (block_bits) >> 8; - let low_count = low_mask.count_ones() as u64; - let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; - let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; - let high_base = low_count * 0x0101_0101_0101_0101; - let high_control = high_control + high_base; - let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); - let packed = unsafe { _mm_loadu_si128(input.add(input_lane).cast::<__m128i>()) }; - let expanded = _mm_shuffle_epi8(packed, control); - let result = unsafe { - _mm_blendv_epi8( - _mm_load_si128(core::ptr::from_ref(&merge.val.0).cast::<__m128i>().add(2)), + _mm256_extracti128_si256::<1>(mask.val.0[0]), + ); + crate::transmute::checked_transmute_store( + result, + output[16..].first_chunk_mut::<16>().unwrap(), + ); + input_lane += block_bits.count_ones() as usize; + let block_bits = _mm_movemask_epi8(_mm256_castsi256_si128(mask.val.0[1])) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = high_control + high_base; + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = crate::transmute::checked_transmute_copy( + input[input_lane..].first_chunk::<16>().unwrap(), + ); + let expanded = _mm_shuffle_epi8(packed, control); + let result = _mm_blendv_epi8( + _mm256_castsi256_si128(merge.val.0[1]), expanded, - _mm_load_si128(core::ptr::from_ref(&mask.val.0).cast::<__m128i>().add(2)), - ) - }; - unsafe { - _mm_storeu_si128(output.as_mut_ptr().add(32).cast::<__m128i>(), result); - } - input_lane += block_bits.count_ones() as usize; - let block_bits = (mask_bits >> 48 & 0xffff) as usize; - let low_mask = (block_bits) & 0xff; - let high_mask = (block_bits) >> 8; - let low_count = low_mask.count_ones() as u64; - let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; - let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; - let high_base = low_count * 0x0101_0101_0101_0101; - let high_control = high_control + high_base; - let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); - let packed = unsafe { _mm_loadu_si128(input.add(input_lane).cast::<__m128i>()) }; - let expanded = _mm_shuffle_epi8(packed, control); - let result = unsafe { - _mm_blendv_epi8( - _mm_load_si128(core::ptr::from_ref(&merge.val.0).cast::<__m128i>().add(3)), + _mm256_castsi256_si128(mask.val.0[1]), + ); + crate::transmute::checked_transmute_store( + result, + output[32..].first_chunk_mut::<16>().unwrap(), + ); + input_lane += block_bits.count_ones() as usize; + let block_bits = + _mm_movemask_epi8(_mm256_extracti128_si256::<1>(mask.val.0[1])) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = high_control + high_base; + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = crate::transmute::checked_transmute_copy( + input[input_lane..].first_chunk::<16>().unwrap(), + ); + let expanded = _mm_shuffle_epi8(packed, control); + let result = _mm_blendv_epi8( + _mm256_extracti128_si256::<1>(merge.val.0[1]), expanded, - _mm_load_si128(core::ptr::from_ref(&mask.val.0).cast::<__m128i>().add(3)), - ) - }; - unsafe { - _mm_storeu_si128(output.as_mut_ptr().add(48).cast::<__m128i>(), result); + _mm256_extracti128_si256::<1>(mask.val.0[1]), + ); + crate::transmute::checked_transmute_store( + result, + output[48..].first_chunk_mut::<16>().unwrap(), + ); + u8x64::simd_from(token, output) } - u8x64::simd_from(token, output) - } - unsafe { kernel(self, values, mask, merge) } + ); + kernel(self, values, mask, merge) } #[inline(always)] fn split_u8x64(self, a: u8x64) -> (u8x32, u8x32) { diff --git a/fearless_simd/src/generated/sse4_2.rs b/fearless_simd/src/generated/sse4_2.rs index 8c16a5971..5dc5e94f1 100644 --- a/fearless_simd/src/generated/sse4_2.rs +++ b/fearless_simd/src/generated/sse4_2.rs @@ -1573,12 +1573,9 @@ impl Simd for Sse4_2 { let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64); let compressed = _mm_shuffle_epi8(compacted, splice); let count = mask_bits.count_ones() as usize; - let prefix = unsafe { - _mm_load_si128( - core::ptr::from_ref(&crate::support::COMPACT_PREFIX_MASKS[count]) - .cast::<__m128i>(), - ) - }; + let [prefix, _] = crate::transmute::checked_transmute_copy::<_, [__m128i; 2]>( + &crate::support::COMPACT_PREFIX_MASKS[count], + ); _mm_blendv_epi8(merge.into(), compressed, prefix).simd_into(token) } ); @@ -7342,12 +7339,10 @@ impl Simd for Sse4_2 { let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count]; let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64); let compressed = _mm_shuffle_epi8(compacted, splice); - unsafe { - _mm_storeu_si128( - output.as_mut_ptr().add(output_lane).cast::<__m128i>(), - compressed, - ); - } + crate::transmute::checked_transmute_store( + compressed, + output[output_lane..].first_chunk_mut::<16>().unwrap(), + ); output_lane += block_bits.count_ones() as usize; let block_bits = (mask_bits >> 16 & 0xffff) as usize; let low_mask = (block_bits) & 0xff; @@ -7361,12 +7356,10 @@ impl Simd for Sse4_2 { let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count]; let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64); let compressed = _mm_shuffle_epi8(compacted, splice); - unsafe { - _mm_storeu_si128( - output.as_mut_ptr().add(output_lane).cast::<__m128i>(), - compressed, - ); - } + crate::transmute::checked_transmute_store( + compressed, + output[output_lane..].first_chunk_mut::<16>().unwrap(), + ); u8x32::simd_from(token, output) } ); @@ -7402,12 +7395,10 @@ impl Simd for Sse4_2 { let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count]; let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64); let compressed = _mm_shuffle_epi8(compacted, splice); - unsafe { - _mm_storeu_si128( - output.as_mut_ptr().add(output_lane).cast::<__m128i>(), - compressed, - ); - } + crate::transmute::checked_transmute_store( + compressed, + output[output_lane..].first_chunk_mut::<16>().unwrap(), + ); output_lane += block_bits.count_ones() as usize; let block_bits = (mask_bits >> 16 & 0xffff) as usize; let low_mask = (block_bits) & 0xff; @@ -7421,39 +7412,35 @@ impl Simd for Sse4_2 { let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count]; let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64); let compressed = _mm_shuffle_epi8(compacted, splice); - unsafe { - _mm_storeu_si128( - output.as_mut_ptr().add(output_lane).cast::<__m128i>(), - compressed, - ); - } + crate::transmute::checked_transmute_store( + compressed, + output[output_lane..].first_chunk_mut::<16>().unwrap(), + ); output_lane += block_bits.count_ones() as usize; let remaining = output_lane.saturating_sub(0).min(16); - let prefix = unsafe { - _mm_load_si128( - core::ptr::from_ref(&crate::support::COMPACT_PREFIX_MASKS[remaining]) - .cast::<__m128i>(), - ) - }; - let compressed = - unsafe { _mm_loadu_si128(output.as_ptr().add(0).cast::<__m128i>()) }; + let [prefix, _] = crate::transmute::checked_transmute_copy::<_, [__m128i; 2]>( + &crate::support::COMPACT_PREFIX_MASKS[remaining], + ); + let compressed = crate::transmute::checked_transmute_copy( + output[0..].first_chunk::<16>().unwrap(), + ); let blended = _mm_blendv_epi8(merge.val.0[0], compressed, prefix); - unsafe { - _mm_storeu_si128(output.as_mut_ptr().add(0).cast::<__m128i>(), blended); - } + crate::transmute::checked_transmute_store( + blended, + output[0..].first_chunk_mut::<16>().unwrap(), + ); let remaining = output_lane.saturating_sub(16).min(16); - let prefix = unsafe { - _mm_load_si128( - core::ptr::from_ref(&crate::support::COMPACT_PREFIX_MASKS[remaining]) - .cast::<__m128i>(), - ) - }; - let compressed = - unsafe { _mm_loadu_si128(output.as_ptr().add(16).cast::<__m128i>()) }; + let [prefix, _] = crate::transmute::checked_transmute_copy::<_, [__m128i; 2]>( + &crate::support::COMPACT_PREFIX_MASKS[remaining], + ); + let compressed = crate::transmute::checked_transmute_copy( + output[16..].first_chunk::<16>().unwrap(), + ); let blended = _mm_blendv_epi8(merge.val.0[1], compressed, prefix); - unsafe { - _mm_storeu_si128(output.as_mut_ptr().add(16).cast::<__m128i>(), blended); - } + crate::transmute::checked_transmute_store( + blended, + output[16..].first_chunk_mut::<16>().unwrap(), + ); u8x32::simd_from(token, output) } ); @@ -7468,11 +7455,10 @@ impl Simd for Sse4_2 { values: u8x32, mask: mask8x32, ) -> u8x32 { - let mask_bits = token.to_bitmask_mask8x32(mask); let input = <[u8; 32]>::from(values); let mut output = [0u8; 32]; let mut input_lane = 0; - let block_bits = (mask_bits & 0xffff) as usize; + let block_bits = _mm_movemask_epi8(mask.val.0[0]) as usize; let low_mask = (block_bits) & 0xff; let high_mask = (block_bits) >> 8; let low_count = low_mask.count_ones() as u64; @@ -7481,15 +7467,17 @@ impl Simd for Sse4_2 { let high_base = low_count * 0x0101_0101_0101_0101; let high_control = high_control + high_base; let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); - let packed = - unsafe { _mm_loadu_si128(input.as_ptr().add(input_lane).cast::<__m128i>()) }; + let packed = crate::transmute::checked_transmute_copy( + input[input_lane..].first_chunk::<16>().unwrap(), + ); let expanded = _mm_shuffle_epi8(packed, control); let result = expanded; - unsafe { - _mm_storeu_si128(output.as_mut_ptr().add(0).cast::<__m128i>(), result); - } + crate::transmute::checked_transmute_store( + result, + output[0..].first_chunk_mut::<16>().unwrap(), + ); input_lane += block_bits.count_ones() as usize; - let block_bits = (mask_bits >> 16 & 0xffff) as usize; + let block_bits = _mm_movemask_epi8(mask.val.0[1]) as usize; let low_mask = (block_bits) & 0xff; let high_mask = (block_bits) >> 8; let low_count = low_mask.count_ones() as u64; @@ -7498,13 +7486,15 @@ impl Simd for Sse4_2 { let high_base = low_count * 0x0101_0101_0101_0101; let high_control = high_control + high_base; let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); - let packed = - unsafe { _mm_loadu_si128(input.as_ptr().add(input_lane).cast::<__m128i>()) }; + let packed = crate::transmute::checked_transmute_copy( + input[input_lane..].first_chunk::<16>().unwrap(), + ); let expanded = _mm_shuffle_epi8(packed, control); let result = expanded; - unsafe { - _mm_storeu_si128(output.as_mut_ptr().add(16).cast::<__m128i>(), result); - } + crate::transmute::checked_transmute_store( + result, + output[16..].first_chunk_mut::<16>().unwrap(), + ); u8x32::simd_from(token, output) } ); @@ -7525,11 +7515,10 @@ impl Simd for Sse4_2 { mask: mask8x32, merge: u8x32, ) -> u8x32 { - let mask_bits = token.to_bitmask_mask8x32(mask); let input = <[u8; 32]>::from(values); let mut output = [0u8; 32]; let mut input_lane = 0; - let block_bits = (mask_bits & 0xffff) as usize; + let block_bits = _mm_movemask_epi8(mask.val.0[0]) as usize; let low_mask = (block_bits) & 0xff; let high_mask = (block_bits) >> 8; let low_count = low_mask.count_ones() as u64; @@ -7538,15 +7527,17 @@ impl Simd for Sse4_2 { let high_base = low_count * 0x0101_0101_0101_0101; let high_control = high_control + high_base; let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); - let packed = - unsafe { _mm_loadu_si128(input.as_ptr().add(input_lane).cast::<__m128i>()) }; + let packed = crate::transmute::checked_transmute_copy( + input[input_lane..].first_chunk::<16>().unwrap(), + ); let expanded = _mm_shuffle_epi8(packed, control); let result = _mm_blendv_epi8(merge.val.0[0], expanded, mask.val.0[0]); - unsafe { - _mm_storeu_si128(output.as_mut_ptr().add(0).cast::<__m128i>(), result); - } + crate::transmute::checked_transmute_store( + result, + output[0..].first_chunk_mut::<16>().unwrap(), + ); input_lane += block_bits.count_ones() as usize; - let block_bits = (mask_bits >> 16 & 0xffff) as usize; + let block_bits = _mm_movemask_epi8(mask.val.0[1]) as usize; let low_mask = (block_bits) & 0xff; let high_mask = (block_bits) >> 8; let low_count = low_mask.count_ones() as u64; @@ -7555,13 +7546,15 @@ impl Simd for Sse4_2 { let high_base = low_count * 0x0101_0101_0101_0101; let high_control = high_control + high_base; let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); - let packed = - unsafe { _mm_loadu_si128(input.as_ptr().add(input_lane).cast::<__m128i>()) }; + let packed = crate::transmute::checked_transmute_copy( + input[input_lane..].first_chunk::<16>().unwrap(), + ); let expanded = _mm_shuffle_epi8(packed, control); let result = _mm_blendv_epi8(merge.val.0[1], expanded, mask.val.0[1]); - unsafe { - _mm_storeu_si128(output.as_mut_ptr().add(16).cast::<__m128i>(), result); - } + crate::transmute::checked_transmute_store( + result, + output[16..].first_chunk_mut::<16>().unwrap(), + ); u8x32::simd_from(token, output) } ); @@ -8764,12 +8757,10 @@ impl Simd for Sse4_2 { let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count]; let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64); let compressed = _mm_shuffle_epi8(compacted, splice); - unsafe { - _mm_storeu_si128( - output.as_mut_ptr().add(output_lane).cast::<__m128i>(), - compressed, - ); - } + crate::transmute::checked_transmute_store( + compressed, + output[output_lane..].first_chunk_mut::<16>().unwrap(), + ); output_lane += block_bits.count_ones() as usize; let block_bits = (mask_bits >> 16 & 0xffff) as usize; let low_mask = (block_bits) & 0xff; @@ -8783,12 +8774,10 @@ impl Simd for Sse4_2 { let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count]; let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64); let compressed = _mm_shuffle_epi8(compacted, splice); - unsafe { - _mm_storeu_si128( - output.as_mut_ptr().add(output_lane).cast::<__m128i>(), - compressed, - ); - } + crate::transmute::checked_transmute_store( + compressed, + output[output_lane..].first_chunk_mut::<16>().unwrap(), + ); output_lane += block_bits.count_ones() as usize; let block_bits = (mask_bits >> 32 & 0xffff) as usize; let low_mask = (block_bits) & 0xff; @@ -8802,12 +8791,10 @@ impl Simd for Sse4_2 { let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count]; let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64); let compressed = _mm_shuffle_epi8(compacted, splice); - unsafe { - _mm_storeu_si128( - output.as_mut_ptr().add(output_lane).cast::<__m128i>(), - compressed, - ); - } + crate::transmute::checked_transmute_store( + compressed, + output[output_lane..].first_chunk_mut::<16>().unwrap(), + ); output_lane += block_bits.count_ones() as usize; let block_bits = (mask_bits >> 48 & 0xffff) as usize; let low_mask = (block_bits) & 0xff; @@ -8821,12 +8808,10 @@ impl Simd for Sse4_2 { let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count]; let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64); let compressed = _mm_shuffle_epi8(compacted, splice); - unsafe { - _mm_storeu_si128( - output.as_mut_ptr().add(output_lane).cast::<__m128i>(), - compressed, - ); - } + crate::transmute::checked_transmute_store( + compressed, + output[output_lane..].first_chunk_mut::<16>().unwrap(), + ); u8x64::simd_from(token, output) } ); @@ -8862,12 +8847,10 @@ impl Simd for Sse4_2 { let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count]; let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64); let compressed = _mm_shuffle_epi8(compacted, splice); - unsafe { - _mm_storeu_si128( - output.as_mut_ptr().add(output_lane).cast::<__m128i>(), - compressed, - ); - } + crate::transmute::checked_transmute_store( + compressed, + output[output_lane..].first_chunk_mut::<16>().unwrap(), + ); output_lane += block_bits.count_ones() as usize; let block_bits = (mask_bits >> 16 & 0xffff) as usize; let low_mask = (block_bits) & 0xff; @@ -8881,12 +8864,10 @@ impl Simd for Sse4_2 { let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count]; let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64); let compressed = _mm_shuffle_epi8(compacted, splice); - unsafe { - _mm_storeu_si128( - output.as_mut_ptr().add(output_lane).cast::<__m128i>(), - compressed, - ); - } + crate::transmute::checked_transmute_store( + compressed, + output[output_lane..].first_chunk_mut::<16>().unwrap(), + ); output_lane += block_bits.count_ones() as usize; let block_bits = (mask_bits >> 32 & 0xffff) as usize; let low_mask = (block_bits) & 0xff; @@ -8900,12 +8881,10 @@ impl Simd for Sse4_2 { let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count]; let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64); let compressed = _mm_shuffle_epi8(compacted, splice); - unsafe { - _mm_storeu_si128( - output.as_mut_ptr().add(output_lane).cast::<__m128i>(), - compressed, - ); - } + crate::transmute::checked_transmute_store( + compressed, + output[output_lane..].first_chunk_mut::<16>().unwrap(), + ); output_lane += block_bits.count_ones() as usize; let block_bits = (mask_bits >> 48 & 0xffff) as usize; let low_mask = (block_bits) & 0xff; @@ -8919,65 +8898,59 @@ impl Simd for Sse4_2 { let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count]; let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64); let compressed = _mm_shuffle_epi8(compacted, splice); - unsafe { - _mm_storeu_si128( - output.as_mut_ptr().add(output_lane).cast::<__m128i>(), - compressed, - ); - } + crate::transmute::checked_transmute_store( + compressed, + output[output_lane..].first_chunk_mut::<16>().unwrap(), + ); output_lane += block_bits.count_ones() as usize; let remaining = output_lane.saturating_sub(0).min(16); - let prefix = unsafe { - _mm_load_si128( - core::ptr::from_ref(&crate::support::COMPACT_PREFIX_MASKS[remaining]) - .cast::<__m128i>(), - ) - }; - let compressed = - unsafe { _mm_loadu_si128(output.as_ptr().add(0).cast::<__m128i>()) }; + let [prefix, _] = crate::transmute::checked_transmute_copy::<_, [__m128i; 2]>( + &crate::support::COMPACT_PREFIX_MASKS[remaining], + ); + let compressed = crate::transmute::checked_transmute_copy( + output[0..].first_chunk::<16>().unwrap(), + ); let blended = _mm_blendv_epi8(merge.val.0[0], compressed, prefix); - unsafe { - _mm_storeu_si128(output.as_mut_ptr().add(0).cast::<__m128i>(), blended); - } + crate::transmute::checked_transmute_store( + blended, + output[0..].first_chunk_mut::<16>().unwrap(), + ); let remaining = output_lane.saturating_sub(16).min(16); - let prefix = unsafe { - _mm_load_si128( - core::ptr::from_ref(&crate::support::COMPACT_PREFIX_MASKS[remaining]) - .cast::<__m128i>(), - ) - }; - let compressed = - unsafe { _mm_loadu_si128(output.as_ptr().add(16).cast::<__m128i>()) }; + let [prefix, _] = crate::transmute::checked_transmute_copy::<_, [__m128i; 2]>( + &crate::support::COMPACT_PREFIX_MASKS[remaining], + ); + let compressed = crate::transmute::checked_transmute_copy( + output[16..].first_chunk::<16>().unwrap(), + ); let blended = _mm_blendv_epi8(merge.val.0[1], compressed, prefix); - unsafe { - _mm_storeu_si128(output.as_mut_ptr().add(16).cast::<__m128i>(), blended); - } + crate::transmute::checked_transmute_store( + blended, + output[16..].first_chunk_mut::<16>().unwrap(), + ); let remaining = output_lane.saturating_sub(32).min(16); - let prefix = unsafe { - _mm_load_si128( - core::ptr::from_ref(&crate::support::COMPACT_PREFIX_MASKS[remaining]) - .cast::<__m128i>(), - ) - }; - let compressed = - unsafe { _mm_loadu_si128(output.as_ptr().add(32).cast::<__m128i>()) }; + let [prefix, _] = crate::transmute::checked_transmute_copy::<_, [__m128i; 2]>( + &crate::support::COMPACT_PREFIX_MASKS[remaining], + ); + let compressed = crate::transmute::checked_transmute_copy( + output[32..].first_chunk::<16>().unwrap(), + ); let blended = _mm_blendv_epi8(merge.val.0[2], compressed, prefix); - unsafe { - _mm_storeu_si128(output.as_mut_ptr().add(32).cast::<__m128i>(), blended); - } + crate::transmute::checked_transmute_store( + blended, + output[32..].first_chunk_mut::<16>().unwrap(), + ); let remaining = output_lane.saturating_sub(48).min(16); - let prefix = unsafe { - _mm_load_si128( - core::ptr::from_ref(&crate::support::COMPACT_PREFIX_MASKS[remaining]) - .cast::<__m128i>(), - ) - }; - let compressed = - unsafe { _mm_loadu_si128(output.as_ptr().add(48).cast::<__m128i>()) }; + let [prefix, _] = crate::transmute::checked_transmute_copy::<_, [__m128i; 2]>( + &crate::support::COMPACT_PREFIX_MASKS[remaining], + ); + let compressed = crate::transmute::checked_transmute_copy( + output[48..].first_chunk::<16>().unwrap(), + ); let blended = _mm_blendv_epi8(merge.val.0[3], compressed, prefix); - unsafe { - _mm_storeu_si128(output.as_mut_ptr().add(48).cast::<__m128i>(), blended); - } + crate::transmute::checked_transmute_store( + blended, + output[48..].first_chunk_mut::<16>().unwrap(), + ); u8x64::simd_from(token, output) } ); @@ -8992,11 +8965,10 @@ impl Simd for Sse4_2 { values: u8x64, mask: mask8x64, ) -> u8x64 { - let mask_bits = token.to_bitmask_mask8x64(mask); let input = <[u8; 64]>::from(values); let mut output = [0u8; 64]; let mut input_lane = 0; - let block_bits = (mask_bits & 0xffff) as usize; + let block_bits = _mm_movemask_epi8(mask.val.0[0]) as usize; let low_mask = (block_bits) & 0xff; let high_mask = (block_bits) >> 8; let low_count = low_mask.count_ones() as u64; @@ -9005,15 +8977,17 @@ impl Simd for Sse4_2 { let high_base = low_count * 0x0101_0101_0101_0101; let high_control = high_control + high_base; let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); - let packed = - unsafe { _mm_loadu_si128(input.as_ptr().add(input_lane).cast::<__m128i>()) }; + let packed = crate::transmute::checked_transmute_copy( + input[input_lane..].first_chunk::<16>().unwrap(), + ); let expanded = _mm_shuffle_epi8(packed, control); let result = expanded; - unsafe { - _mm_storeu_si128(output.as_mut_ptr().add(0).cast::<__m128i>(), result); - } + crate::transmute::checked_transmute_store( + result, + output[0..].first_chunk_mut::<16>().unwrap(), + ); input_lane += block_bits.count_ones() as usize; - let block_bits = (mask_bits >> 16 & 0xffff) as usize; + let block_bits = _mm_movemask_epi8(mask.val.0[1]) as usize; let low_mask = (block_bits) & 0xff; let high_mask = (block_bits) >> 8; let low_count = low_mask.count_ones() as u64; @@ -9022,15 +8996,17 @@ impl Simd for Sse4_2 { let high_base = low_count * 0x0101_0101_0101_0101; let high_control = high_control + high_base; let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); - let packed = - unsafe { _mm_loadu_si128(input.as_ptr().add(input_lane).cast::<__m128i>()) }; + let packed = crate::transmute::checked_transmute_copy( + input[input_lane..].first_chunk::<16>().unwrap(), + ); let expanded = _mm_shuffle_epi8(packed, control); let result = expanded; - unsafe { - _mm_storeu_si128(output.as_mut_ptr().add(16).cast::<__m128i>(), result); - } + crate::transmute::checked_transmute_store( + result, + output[16..].first_chunk_mut::<16>().unwrap(), + ); input_lane += block_bits.count_ones() as usize; - let block_bits = (mask_bits >> 32 & 0xffff) as usize; + let block_bits = _mm_movemask_epi8(mask.val.0[2]) as usize; let low_mask = (block_bits) & 0xff; let high_mask = (block_bits) >> 8; let low_count = low_mask.count_ones() as u64; @@ -9039,15 +9015,17 @@ impl Simd for Sse4_2 { let high_base = low_count * 0x0101_0101_0101_0101; let high_control = high_control + high_base; let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); - let packed = - unsafe { _mm_loadu_si128(input.as_ptr().add(input_lane).cast::<__m128i>()) }; + let packed = crate::transmute::checked_transmute_copy( + input[input_lane..].first_chunk::<16>().unwrap(), + ); let expanded = _mm_shuffle_epi8(packed, control); let result = expanded; - unsafe { - _mm_storeu_si128(output.as_mut_ptr().add(32).cast::<__m128i>(), result); - } + crate::transmute::checked_transmute_store( + result, + output[32..].first_chunk_mut::<16>().unwrap(), + ); input_lane += block_bits.count_ones() as usize; - let block_bits = (mask_bits >> 48 & 0xffff) as usize; + let block_bits = _mm_movemask_epi8(mask.val.0[3]) as usize; let low_mask = (block_bits) & 0xff; let high_mask = (block_bits) >> 8; let low_count = low_mask.count_ones() as u64; @@ -9056,13 +9034,15 @@ impl Simd for Sse4_2 { let high_base = low_count * 0x0101_0101_0101_0101; let high_control = high_control + high_base; let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); - let packed = - unsafe { _mm_loadu_si128(input.as_ptr().add(input_lane).cast::<__m128i>()) }; + let packed = crate::transmute::checked_transmute_copy( + input[input_lane..].first_chunk::<16>().unwrap(), + ); let expanded = _mm_shuffle_epi8(packed, control); let result = expanded; - unsafe { - _mm_storeu_si128(output.as_mut_ptr().add(48).cast::<__m128i>(), result); - } + crate::transmute::checked_transmute_store( + result, + output[48..].first_chunk_mut::<16>().unwrap(), + ); u8x64::simd_from(token, output) } ); @@ -9083,11 +9063,10 @@ impl Simd for Sse4_2 { mask: mask8x64, merge: u8x64, ) -> u8x64 { - let mask_bits = token.to_bitmask_mask8x64(mask); let input = <[u8; 64]>::from(values); let mut output = [0u8; 64]; let mut input_lane = 0; - let block_bits = (mask_bits & 0xffff) as usize; + let block_bits = _mm_movemask_epi8(mask.val.0[0]) as usize; let low_mask = (block_bits) & 0xff; let high_mask = (block_bits) >> 8; let low_count = low_mask.count_ones() as u64; @@ -9096,15 +9075,17 @@ impl Simd for Sse4_2 { let high_base = low_count * 0x0101_0101_0101_0101; let high_control = high_control + high_base; let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); - let packed = - unsafe { _mm_loadu_si128(input.as_ptr().add(input_lane).cast::<__m128i>()) }; + let packed = crate::transmute::checked_transmute_copy( + input[input_lane..].first_chunk::<16>().unwrap(), + ); let expanded = _mm_shuffle_epi8(packed, control); let result = _mm_blendv_epi8(merge.val.0[0], expanded, mask.val.0[0]); - unsafe { - _mm_storeu_si128(output.as_mut_ptr().add(0).cast::<__m128i>(), result); - } + crate::transmute::checked_transmute_store( + result, + output[0..].first_chunk_mut::<16>().unwrap(), + ); input_lane += block_bits.count_ones() as usize; - let block_bits = (mask_bits >> 16 & 0xffff) as usize; + let block_bits = _mm_movemask_epi8(mask.val.0[1]) as usize; let low_mask = (block_bits) & 0xff; let high_mask = (block_bits) >> 8; let low_count = low_mask.count_ones() as u64; @@ -9113,15 +9094,17 @@ impl Simd for Sse4_2 { let high_base = low_count * 0x0101_0101_0101_0101; let high_control = high_control + high_base; let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); - let packed = - unsafe { _mm_loadu_si128(input.as_ptr().add(input_lane).cast::<__m128i>()) }; + let packed = crate::transmute::checked_transmute_copy( + input[input_lane..].first_chunk::<16>().unwrap(), + ); let expanded = _mm_shuffle_epi8(packed, control); let result = _mm_blendv_epi8(merge.val.0[1], expanded, mask.val.0[1]); - unsafe { - _mm_storeu_si128(output.as_mut_ptr().add(16).cast::<__m128i>(), result); - } + crate::transmute::checked_transmute_store( + result, + output[16..].first_chunk_mut::<16>().unwrap(), + ); input_lane += block_bits.count_ones() as usize; - let block_bits = (mask_bits >> 32 & 0xffff) as usize; + let block_bits = _mm_movemask_epi8(mask.val.0[2]) as usize; let low_mask = (block_bits) & 0xff; let high_mask = (block_bits) >> 8; let low_count = low_mask.count_ones() as u64; @@ -9130,15 +9113,17 @@ impl Simd for Sse4_2 { let high_base = low_count * 0x0101_0101_0101_0101; let high_control = high_control + high_base; let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); - let packed = - unsafe { _mm_loadu_si128(input.as_ptr().add(input_lane).cast::<__m128i>()) }; + let packed = crate::transmute::checked_transmute_copy( + input[input_lane..].first_chunk::<16>().unwrap(), + ); let expanded = _mm_shuffle_epi8(packed, control); let result = _mm_blendv_epi8(merge.val.0[2], expanded, mask.val.0[2]); - unsafe { - _mm_storeu_si128(output.as_mut_ptr().add(32).cast::<__m128i>(), result); - } + crate::transmute::checked_transmute_store( + result, + output[32..].first_chunk_mut::<16>().unwrap(), + ); input_lane += block_bits.count_ones() as usize; - let block_bits = (mask_bits >> 48 & 0xffff) as usize; + let block_bits = _mm_movemask_epi8(mask.val.0[3]) as usize; let low_mask = (block_bits) & 0xff; let high_mask = (block_bits) >> 8; let low_count = low_mask.count_ones() as u64; @@ -9147,13 +9132,15 @@ impl Simd for Sse4_2 { let high_base = low_count * 0x0101_0101_0101_0101; let high_control = high_control + high_base; let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); - let packed = - unsafe { _mm_loadu_si128(input.as_ptr().add(input_lane).cast::<__m128i>()) }; + let packed = crate::transmute::checked_transmute_copy( + input[input_lane..].first_chunk::<16>().unwrap(), + ); let expanded = _mm_shuffle_epi8(packed, control); let result = _mm_blendv_epi8(merge.val.0[3], expanded, mask.val.0[3]); - unsafe { - _mm_storeu_si128(output.as_mut_ptr().add(48).cast::<__m128i>(), result); - } + crate::transmute::checked_transmute_store( + result, + output[48..].first_chunk_mut::<16>().unwrap(), + ); u8x64::simd_from(token, output) } ); diff --git a/fearless_simd/src/support.rs b/fearless_simd/src/support.rs index 9d234372c..7ea1c3982 100644 --- a/fearless_simd/src/support.rs +++ b/fearless_simd/src/support.rs @@ -121,6 +121,61 @@ pub(crate) mod compact_128 { pub(crate) const EXPAND_64: [Aligned128<[u8; 16]>; 4] = controls(8, true); } +/// Dword-permutation controls for whole 32- or 64-bit elements in an AVX2 vector. +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +pub(crate) mod compact_256 { + use super::Aligned256; + + #[allow( + clippy::cast_possible_truncation, + clippy::cast_possible_wrap, + reason = "Dword indices are less than eight" + )] + const fn controls( + element_dwords: usize, + expand: bool, + ) -> [Aligned256<[i32; 8]>; N] { + assert!(matches!(element_dwords, 1 | 2), "unsupported element size"); + let lanes = 8 / element_dwords; + assert!( + N == 1 << lanes, + "one control row is required per selection mask" + ); + // VPERMD only uses the low three index bits. The -1 sentinel separately + // identifies inactive lanes for zero/merge fill after the permutation. + let mut table = [Aligned256([-1; 8]); N]; + let mut mask = 0; + while mask < N { + let mut selected = 0; + let mut lane = 0; + while lane < lanes { + if mask & (1 << lane) != 0 { + let source = if expand { selected } else { lane }; + let destination = if expand { lane } else { selected }; + let mut dword = 0; + while dword < element_dwords { + // Adjacent dwords of a qword always move together. + table[mask].0[destination * element_dwords + dword] = + (source * element_dwords + dword) as i32; + dword += 1; + } + selected += 1; + } + lane += 1; + } + mask += 1; + } + table + } + + // Full dword controls avoid the extra widening/validity work of packed byte + // controls, particularly on Zen 1/3. Signed, unsigned and float ops share them. + pub(crate) static COMPRESS_32: [Aligned256<[i32; 8]>; 256] = controls(1, false); + pub(crate) static EXPAND_32: [Aligned256<[i32; 8]>; 256] = controls(1, true); + pub(crate) static COMPRESS_64: [Aligned256<[i32; 8]>; 16] = controls(2, false); + pub(crate) static EXPAND_64: [Aligned256<[i32; 8]>; 16] = controls(2, true); +} + const fn compact_control_table(expand: bool) -> [u64; 256] { // Expansion adds a lane offset to each byte of the packed control. Use 0x80 // for inactive lanes so these additions preserve the zeroing bit without diff --git a/fearless_simd_gen/src/mk_x86.rs b/fearless_simd_gen/src/mk_x86.rs index 02467237a..cbef15c7b 100644 --- a/fearless_simd_gen/src/mk_x86.rs +++ b/fearless_simd_gen/src/mk_x86.rs @@ -332,6 +332,11 @@ impl Level for X86 { OpSig::Compress { .. } | OpSig::Expand { .. } => { if *self == Self::Avx512 { self.handle_avx512_compact_op(op, vec_ty) + } else if *self == Self::Avx2 + && vec_ty.n_bits() == 256 + && matches!(vec_ty.scalar_bits, 32 | 64) + { + self.handle_avx2_dword_compact(op, vec_ty) } else if matches!(*self, Self::Sse4_2 | Self::Avx2) && vec_ty.n_bits() == 128 && matches!(vec_ty.scalar_bits, 16 | 32 | 64) @@ -343,9 +348,7 @@ impl Level for X86 { self.handle_x86_compact_16(op, vec_ty) } else if *self == Self::Avx2 && matches!(sig, OpSig::Compress { .. }) { self.handle_avx2_shuffle_compact(op, vec_ty) - } else if *self == Self::Avx2 { - self.handle_avx2_sse_expand(op, vec_ty) - } else if *self == Self::Sse4_2 { + } else if matches!(*self, Self::Sse4_2 | Self::Avx2) { self.handle_x86_wide_compact(op, vec_ty) } else { generic_op(&op, vec_ty) @@ -1296,6 +1299,77 @@ fn expand_16_control(block_bits: TokenStream) -> TokenStream { } impl X86 { + /// Compact whole dwords/qwords with a single full-width AVX2 permutation. + fn handle_avx2_dword_compact(&self, op: Op, vec_ty: &VecType) -> TokenStream { + assert!(*self == Self::Avx2, "dword compaction requires AVX2"); + assert_eq!(vec_ty.n_bits(), 256, "requires one native AVX2 vector"); + assert!( + matches!(vec_ty.scalar_bits, 32 | 64), + "requires dword/qword elements" + ); + let (expand, merge) = match op.sig { + OpSig::Compress { merge } => (false, merge), + OpSig::Expand { merge } => (true, merge), + _ => unreachable!("only compact operations use dword permutations"), + }; + let table = format_ident!( + "{}_{}", + if expand { "EXPAND" } else { "COMPRESS" }, + vec_ty.scalar_bits, + ); + let to_bitmask = generic_op_name("to_bitmask", &vec_ty.mask_ty()); + let lane_mask = Literal::usize_unsuffixed((1 << vec_ty.len) - 1); + let to_int = |value: TokenStream| { + if vec_ty.scalar == ScalarType::Float { + let cast = cast_ident( + ScalarType::Float, + ScalarType::Int, + vec_ty.scalar_bits, + vec_ty.scalar_bits, + 256, + ); + quote! { #cast(#value.into()) } + } else { + quote! { #value.into() } + } + }; + let values = to_int(quote! { values }); + let merge_value = to_int(quote! { merge }); + let finish = match (expand, merge) { + (false, false) => quote! { + _mm256_andnot_si256(_mm256_srai_epi32::<31>(control), reordered) + }, + // Valid controls 0..7 have no byte sign bits; -1 selects the merge. + (false, true) => quote! { _mm256_blendv_epi8(reordered, #merge_value, control) }, + // Expansion writes exactly the original mask lanes. + (true, false) => quote! { _mm256_and_si256(reordered, mask.into()) }, + (true, true) => quote! { _mm256_blendv_epi8(#merge_value, reordered, mask.into()) }, + }; + let result = if vec_ty.scalar == ScalarType::Float { + let cast = cast_ident( + ScalarType::Int, + ScalarType::Float, + vec_ty.scalar_bits, + vec_ty.scalar_bits, + 256, + ); + quote! { #cast(result) } + } else { + quote! { result } + }; + self.kernel_method(op, vec_ty, |token| { + quote! { + let bits = #token.#to_bitmask(mask) as usize & #lane_mask; + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::#table[bits], + ); + let reordered = _mm256_permutevar8x32_epi32(#values, control); + let result = #finish; + #result.simd_into(#token) + } + }) + } + fn handle_x86_compact_16(&self, op: Op, vec_ty: &VecType) -> TokenStream { assert!( matches!(*self, Self::Sse4_2 | Self::Avx2), @@ -1311,14 +1385,9 @@ impl X86 { let finish = if merge { quote! { let count = mask_bits.count_ones() as usize; - let prefix = unsafe { - _mm_load_si128( - core::ptr::from_ref( - &crate::support::COMPACT_PREFIX_MASKS[count], - ) - .cast::<__m128i>(), - ) - }; + let [prefix, _] = crate::transmute::checked_transmute_copy::<_, [__m128i; 2]>( + &crate::support::COMPACT_PREFIX_MASKS[count], + ); _mm_blendv_epi8(merge.into(), compressed, prefix).simd_into(#token) } } else { @@ -1355,7 +1424,10 @@ impl X86 { } fn handle_x86_wide_compact(&self, op: Op, vec_ty: &VecType) -> TokenStream { - assert!(*self == Self::Sse4_2, "wide x86 compact requires SSE4.2"); + assert!( + matches!(*self, Self::Sse4_2 | Self::Avx2), + "wide x86 compact requires SSE4.2 or AVX2" + ); assert!( matches!(vec_ty.len, 32 | 64), "wide x86 compact requires a 32- or 64-byte vector" @@ -1405,12 +1477,10 @@ impl X86 { let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[#low_count]; let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64); let compressed = _mm_shuffle_epi8(compacted, splice); - unsafe { - _mm_storeu_si128( - output.as_mut_ptr().add(output_lane).cast::<__m128i>(), - compressed, - ); - } + crate::transmute::checked_transmute_store( + compressed, + output[output_lane..].first_chunk_mut::<16>().unwrap(), + ); #advance } }); @@ -1420,24 +1490,17 @@ impl X86 { let merge = block_value("merge", block); quote! { let remaining = output_lane.saturating_sub(#offset).min(16); - let prefix = unsafe { - _mm_load_si128( - core::ptr::from_ref( - &crate::support::COMPACT_PREFIX_MASKS[remaining], - ) - .cast::<__m128i>(), - ) - }; - let compressed = unsafe { - _mm_loadu_si128(output.as_ptr().add(#offset).cast::<__m128i>()) - }; + let [prefix, _] = crate::transmute::checked_transmute_copy::<_, [__m128i; 2]>( + &crate::support::COMPACT_PREFIX_MASKS[remaining], + ); + let compressed = crate::transmute::checked_transmute_copy( + output[#offset..].first_chunk::<16>().unwrap(), + ); let blended = _mm_blendv_epi8(#merge, compressed, prefix); - unsafe { - _mm_storeu_si128( - output.as_mut_ptr().add(#offset).cast::<__m128i>(), - blended, - ); - } + crate::transmute::checked_transmute_store( + blended, + output[#offset..].first_chunk_mut::<16>().unwrap(), + ); } }); quote! { #(#blend_blocks)* } @@ -1456,12 +1519,13 @@ impl X86 { OpSig::Expand { merge } => { let input = quote! { <[u8; #len]>::from(values) }; let expand_blocks = (0..block_count).map(|block| { - let shifted_mask = shifted_mask_bits(block * 16); + // Each shuffle consumes one 128-bit mask. Extract it directly; + // a wide movemask would require extra scalar shifts to split it. + let mask = block_value("mask", block); let offset = Literal::usize_unsuffixed(block * 16); let setup = expand_16_control(quote! { block_bits }); let result = if merge { let merge = block_value("merge", block); - let mask = block_value("mask", block); quote! { _mm_blendv_epi8(#merge, expanded, #mask) } } else { quote! { expanded } @@ -1472,24 +1536,21 @@ impl X86 { TokenStream::new() }; quote! { - let block_bits = (#shifted_mask & 0xffff) as usize; + let block_bits = _mm_movemask_epi8(#mask) as usize; #setup - let packed = unsafe { - _mm_loadu_si128(input.as_ptr().add(input_lane).cast::<__m128i>()) - }; + let packed = crate::transmute::checked_transmute_copy( + input[input_lane..].first_chunk::<16>().unwrap(), + ); let expanded = _mm_shuffle_epi8(packed, control); let result = #result; - unsafe { - _mm_storeu_si128( - output.as_mut_ptr().add(#offset).cast::<__m128i>(), - result, - ); - } + crate::transmute::checked_transmute_store( + result, + output[#offset..].first_chunk_mut::<16>().unwrap(), + ); #advance } }); quote! { - let mask_bits = #token.#to_bitmask(mask); let input = #input; let mut output = [0u8; #len]; let mut input_lane = 0; @@ -1501,109 +1562,6 @@ impl X86 { }) } - fn handle_avx2_sse_expand(&self, op: Op, vec_ty: &VecType) -> TokenStream { - assert!( - *self == Self::Avx2, - "lower-width expand helpers require AVX2" - ); - assert!( - matches!(vec_ty.len, 32 | 64), - "lower-width expand helpers require a 32- or 64-byte vector" - ); - assert!( - matches!(op.sig, OpSig::Expand { .. }), - "lower-width expand helpers only implement expand operations" - ); - let vec = vec_ty.rust(); - let len = Literal::usize_unsuffixed(vec_ty.len); - let block_count = vec_ty.len / 16; - - op.simd_trait_kernel_method_with_target_features( - self.token(), - SSE4_2_FEATURES, - vec_ty, - |token| { - let input_setup = quote! { let input_values = <[u8; #len]>::from(values); }; - let input = quote! { input_values.as_ptr() }; - let mask_parts = (0..block_count).map(|block| { - let block_literal = Literal::usize_unsuffixed(block); - let movemask = quote! { - _mm_movemask_epi8(unsafe { - _mm_load_si128( - core::ptr::from_ref(&mask.val.0) - .cast::<__m128i>() - .add(#block_literal), - ) - }) as u64 - }; - if block == 0 { - movemask - } else { - let shift = Literal::usize_unsuffixed(block * 16); - quote! { (#movemask) << #shift } - } - }); - let blocks = (0..block_count).map(|block| { - let shifted_mask = shifted_mask_bits(block * 16); - let offset = Literal::usize_unsuffixed(block * 16); - let setup = expand_16_control(quote! { block_bits }); - let blend = if matches!(op.sig, OpSig::Expand { merge: true }) { - let block = Literal::usize_unsuffixed(block); - quote! { - unsafe { - _mm_blendv_epi8( - _mm_load_si128( - core::ptr::from_ref(&merge.val.0) - .cast::<__m128i>() - .add(#block), - ), - expanded, - _mm_load_si128( - core::ptr::from_ref(&mask.val.0) - .cast::<__m128i>() - .add(#block), - ), - ) - } - } - } else { - quote! { expanded } - }; - let advance = if block + 1 < block_count { - quote! { input_lane += block_bits.count_ones() as usize; } - } else { - TokenStream::new() - }; - quote! { - let block_bits = (#shifted_mask & 0xffff) as usize; - #setup - let packed = unsafe { - _mm_loadu_si128(input.add(input_lane).cast::<__m128i>()) - }; - let expanded = _mm_shuffle_epi8(packed, control); - let result = #blend; - unsafe { - _mm_storeu_si128( - output.as_mut_ptr().add(#offset).cast::<__m128i>(), - result, - ); - } - #advance - } - }); - quote! { - let mask_bits = #(#mask_parts)|*; - #input_setup - let input = #input; - let mut output = [0u8; #len]; - let mut input_lane = 0usize; - #(#blocks)* - #vec::simd_from(#token, output) - } - }, - ) - } - fn handle_avx2_shuffle_compact(&self, op: Op, vec_ty: &VecType) -> TokenStream { assert!(*self == Self::Avx2, "wide shuffle compact requires AVX2"); assert!( @@ -1701,15 +1659,10 @@ impl X86 { TokenStream::new() }; let finish_pair = quote! { - let splice = unsafe { - _mm256_load_si256( - core::ptr::from_ref( - &crate::support::COMPACT_16_SPLICE_CONTROLS - [#bits0.count_ones() as usize], - ) - .cast::<__m256i>(), - ) - }; + let splice = crate::transmute::checked_transmute_copy( + &crate::support::COMPACT_16_SPLICE_CONTROLS + [#bits0.count_ones() as usize], + ); let compressed_low = _mm256_permute2x128_si256::<0x80>(compressed, compressed); let compressed_high = @@ -1749,26 +1702,10 @@ impl X86 { (block_bits_0.count_ones() + block_bits_1.count_ones()) as usize; let controls = &crate::support::COMPACT_32_STITCH_CONTROLS[first_count]; - let left_same = unsafe { - _mm256_load_si256( - core::ptr::from_ref(&controls.left_same).cast::<__m256i>(), - ) - }; - let left_cross = unsafe { - _mm256_load_si256( - core::ptr::from_ref(&controls.left_cross).cast::<__m256i>(), - ) - }; - let right_same = unsafe { - _mm256_load_si256( - core::ptr::from_ref(&controls.right_same).cast::<__m256i>(), - ) - }; - let right_cross = unsafe { - _mm256_load_si256( - core::ptr::from_ref(&controls.right_cross).cast::<__m256i>(), - ) - }; + let left_same = crate::transmute::checked_transmute_copy(&controls.left_same); + let left_cross = crate::transmute::checked_transmute_copy(&controls.left_cross); + let right_same = crate::transmute::checked_transmute_copy(&controls.right_same); + let right_cross = crate::transmute::checked_transmute_copy(&controls.right_cross); let low_to_high = _mm256_permute2x128_si256::<0x08>(compressed_1, compressed_1); @@ -1806,14 +1743,9 @@ impl X86 { quote! { merge.val.0[#pair] } }; quote! { - let prefix = unsafe { - _mm256_load_si256( - core::ptr::from_ref( - &crate::support::COMPACT_PREFIX_MASKS[#remaining], - ) - .cast::<__m256i>(), - ) - }; + let prefix = crate::transmute::checked_transmute_copy( + &crate::support::COMPACT_PREFIX_MASKS[#remaining], + ); let #result = _mm256_blendv_epi8(#merge, #compressed, prefix); } } else { diff --git a/fearless_simd_gen/src/ops.rs b/fearless_simd_gen/src/ops.rs index 5cec40ba5..d0576d0b0 100644 --- a/fearless_simd_gen/src/ops.rs +++ b/fearless_simd_gen/src/ops.rs @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 OR MIT use anyhow::{Context, anyhow}; -use proc_macro2::{Ident, Literal, Span, TokenStream}; +use proc_macro2::{Ident, Span, TokenStream}; use quote::{format_ident, quote}; use std::fmt::Write; @@ -284,46 +284,6 @@ impl Op { } } - /// Generate a SIMD trait method whose local kernel uses an explicit target-feature set rather - /// than all features associated with the method's backend. This allows a higher backend to - /// reuse a lower backend's instruction formulation when wider instructions do not improve it. - pub(crate) fn simd_trait_kernel_method_with_target_features( - &self, - method_level: Ident, - target_features: &str, - vec_ty: &VecType, - body: impl FnOnce(&Ident) -> TokenStream, - ) -> TokenStream { - assert!( - !matches!(self.sig, OpSig::Slide { .. }), - "kernel! does not support const-generic methods" - ); - - let method_sig = self.simd_trait_method_sig(vec_ty); - let token = Ident::new("token", Span::call_site()); - let target_features = Literal::string(target_features); - let kernel_body = body(&token); - let sig = self.simd_trait_sig_parts(vec_ty, quote! { #method_level }); - let arg_decls = sig.arg_decls(); - let call_args = &sig.arg_names; - let ret = &sig.ret; - - quote! { - #method_sig { - #[inline] - #[target_feature(enable = #target_features)] - unsafe fn kernel( - #token: #method_level #(, #arg_decls)* - ) -> #ret { - #kernel_body - } - - // SAFETY: the backend token guarantees this kernel's feature subset. - unsafe { kernel(self #(, #call_args)*) } - } - } - } - fn simd_trait_sig_parts(&self, vec_ty: &VecType, simd_ty: TokenStream) -> SimdTraitSigParts { let ty = vec_ty.rust(); let arg_names = self diff --git a/fearless_simd_tests/tests/harness/ops/compress.rs b/fearless_simd_tests/tests/harness/ops/compress.rs index b8e18f3ba..0f889cf9a 100644 --- a/fearless_simd_tests/tests/harness/ops/compress.rs +++ b/fearless_simd_tests/tests/harness/ops/compress.rs @@ -836,6 +836,7 @@ fn compact_u32x8(simd: S) { .chain(single_lanes) .chain(prefixes) .chain(suffixes) + .chain(0..=all_lanes) { let mask = mask32x8::from_bitmask(simd, mask_bits); let mut expected_compress = [0; 8]; @@ -1023,6 +1024,7 @@ fn compact_i32x8(simd: S) { .chain(single_lanes) .chain(prefixes) .chain(suffixes) + .chain(0..=all_lanes) { let mask = mask32x8::from_bitmask(simd, mask_bits); let mut expected_compress = [0; 8]; @@ -1208,6 +1210,7 @@ fn compact_u64x4(simd: S) { .chain(single_lanes) .chain(prefixes) .chain(suffixes) + .chain(0..=all_lanes) { let mask = mask64x4::from_bitmask(simd, mask_bits); let mut expected_compress = [0; 4]; @@ -1394,6 +1397,7 @@ fn compact_i64x4(simd: S) { .chain(single_lanes) .chain(prefixes) .chain(suffixes) + .chain(0..=all_lanes) { let mask = mask64x4::from_bitmask(simd, mask_bits); let mut expected_compress = [0; 4]; @@ -1623,6 +1627,7 @@ fn compact_f32x8(simd: S) { .chain(single_lanes) .chain(prefixes) .chain(suffixes) + .chain(0..=all_lanes) { let mask = mask32x8::from_bitmask(simd, mask_bits); let mut expected_compress = [0; 8]; @@ -1912,6 +1917,7 @@ fn compact_f64x4(simd: S) { .chain(single_lanes) .chain(prefixes) .chain(suffixes) + .chain(0..=all_lanes) { let mask = mask64x4::from_bitmask(simd, mask_bits); let mut expected_compress = [0; 4]; @@ -2179,3 +2185,48 @@ fn compress_u8x16_exhaustive_masks(simd: S) { ); } } + +#[simd_test] +fn compress_u32x8_exhaustive_masks(simd: S) { + let values = [ + 0, + u32::MAX, + 0x8000_0000, + 1, + 0x7fff_ffff, + 0x1234_5678, + 0xfedc_ba98, + 0x0102_0304, + ]; + let merge = [ + 0xa0b0_c001, + 0xa0b0_c002, + 0xa0b0_c003, + 0xa0b0_c004, + 0xa0b0_c005, + 0xa0b0_c006, + 0xa0b0_c007, + 0xa0b0_c008, + ]; + let values_vec = u32x8::simd_from(simd, values); + let merge_vec = u32x8::simd_from(simd, merge); + for bits in 0..=u8::MAX { + let mask = mask32x8::from_bitmask(simd, u64::from(bits)); + let mut expected = [0; 8]; + let mut expected_merge = merge; + let mut selected = 0; + for (lane, value) in values.into_iter().enumerate() { + if bits & (1 << lane) != 0 { + expected[selected] = value; + expected_merge[selected] = value; + selected += 1; + } + } + assert_eq!(*values_vec.compress(mask), expected, "mask {bits:#04x}"); + assert_eq!( + *values_vec.compress_merge(mask, merge_vec), + expected_merge, + "mask {bits:#04x}" + ); + } +} From e187a6bf4033e3022c7cba9a6f1e76292c3fc43e Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Wed, 30 Sep 2026 12:35:45 +0100 Subject: [PATCH 7/8] Extend the AVX2 optimization for 32 bits and up to 512-bit wide vectors, same 3x gains seen there as for the 256-bit wide vectors --- fearless_simd/src/generated/avx2.rs | 882 ++++++++++++++---- fearless_simd/src/support.rs | 24 + fearless_simd_gen/src/mk_x86.rs | 110 +++ .../tests/harness/ops/compress.rs | 6 + 4 files changed, 854 insertions(+), 168 deletions(-) diff --git a/fearless_simd/src/generated/avx2.rs b/fearless_simd/src/generated/avx2.rs index fb2557403..283a70685 100644 --- a/fearless_simd/src/generated/avx2.rs +++ b/fearless_simd/src/generated/avx2.rs @@ -14106,11 +14106,36 @@ impl Simd for Avx2 { } #[inline(always)] fn compress_f32x16(self, values: f32x16, mask: mask32x16) -> f32x16 { - let mask = mask8x64 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: f32x16, mask: mask32x16) -> f32x16 { + let (low_mask, high_mask) = token.split_mask32x16(mask); + let low_count = low_mask.to_bitmask().count_ones() as usize; + let (low, high) = values.split(); + let bits = mask.to_bitmask(); + if bits == 0 { + return f32x16::splat(token, 0.0); + } + if bits == 65535 { + return values; + } + let low = low.compress(low_mask); + let high = high.compress(high_mask); + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::SHIFTS[low_count], + ); + let low_bits = crate::transmute::checked_transmute_copy(&low.val); + let high_bits = crate::transmute::checked_transmute_copy(&high.val); + let shifted = _mm256_permutevar8x32_epi32(high_bits, control); + let result_low = _mm256_blendv_epi8(shifted, low_bits, control); + let result_high = _mm256_and_si256(shifted, _mm256_srai_epi32::<31>(control)); + f32x16 { + val: crate::transmute::checked_transmute_copy(&[result_low, result_high]), + simd: token, + } + } + ); + kernel(self, values, mask) } #[inline(always)] fn compress_merge_f32x16( @@ -14119,23 +14144,69 @@ impl Simd for Avx2 { mask: mask32x16, merge: f32x16, ) -> f32x16 { - let mask = mask8x64 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_merge_u8x64( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: f32x16, + mask: mask32x16, + merge: f32x16, + ) -> f32x16 { + let (low_mask, high_mask) = token.split_mask32x16(mask); + let low_count = low_mask.to_bitmask().count_ones() as usize; + let (low, high) = values.split(); + let bits = mask.to_bitmask(); + if bits == 0 { + return merge; + } + if bits == 65535 { + return values; + } + let low = low.compress(low_mask); + let high = high.compress(high_mask); + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::SHIFTS[low_count], + ); + let low_bits = crate::transmute::checked_transmute_copy(&low.val); + let high_bits = crate::transmute::checked_transmute_copy(&high.val); + let shifted = _mm256_permutevar8x32_epi32(high_bits, control); + let result_low = _mm256_blendv_epi8(shifted, low_bits, control); + let result_high = _mm256_and_si256(shifted, _mm256_srai_epi32::<31>(control)); + let compressed = f32x16 { + val: crate::transmute::checked_transmute_copy(&[result_low, result_high]), + simd: token, + }; + let count = low_count + high_mask.to_bitmask().count_ones() as usize; + let prefix = mask32x16::from_bitmask(token, (1u64 << count) - 1); + token.select_f32x16(prefix, compressed, merge) + } + ); + kernel(self, values, mask, merge) } #[inline(always)] fn expand_f32x16(self, values: f32x16, mask: mask32x16) -> f32x16 { - let mask = mask8x64 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: f32x16, mask: mask32x16) -> f32x16 { + let (low_mask, high_mask) = token.split_mask32x16(mask); + let low_count = low_mask.to_bitmask().count_ones() as usize; + let (low, high) = values.split(); + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::SHIFTS[8 - (low_count)], + ); + let low_bits = crate::transmute::checked_transmute_copy(&low.val); + let high_bits = crate::transmute::checked_transmute_copy(&high.val); + let from_low = _mm256_permutevar8x32_epi32(low_bits, control); + let from_high = _mm256_permutevar8x32_epi32(high_bits, control); + let packed = _mm256_blendv_epi8(from_high, from_low, control); + let high = f32x8 { + val: crate::transmute::checked_transmute_copy(&packed), + simd: token, + }; + low.expand(low_mask).combine(high.expand(high_mask)) + } + ); + kernel(self, values, mask) } #[inline(always)] fn expand_merge_f32x16( @@ -14144,15 +14215,35 @@ impl Simd for Avx2 { mask: mask32x16, merge: f32x16, ) -> f32x16 { - let mask = mask8x64 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_merge_u8x64( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: f32x16, + mask: mask32x16, + merge: f32x16, + ) -> f32x16 { + let (low_mask, high_mask) = token.split_mask32x16(mask); + let low_count = low_mask.to_bitmask().count_ones() as usize; + let (low, high) = values.split(); + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::SHIFTS[8 - (low_count)], + ); + let low_bits = crate::transmute::checked_transmute_copy(&low.val); + let high_bits = crate::transmute::checked_transmute_copy(&high.val); + let from_low = _mm256_permutevar8x32_epi32(low_bits, control); + let from_high = _mm256_permutevar8x32_epi32(high_bits, control); + let packed = _mm256_blendv_epi8(from_high, from_low, control); + let high = f32x8 { + val: crate::transmute::checked_transmute_copy(&packed), + simd: token, + }; + let (low_merge, high_merge) = merge.split(); + low.expand_merge(low_mask, low_merge) + .combine(high.expand_merge(high_mask, high_merge)) + } + ); + kernel(self, values, mask, merge) } #[inline(always)] fn is_nan_f32x16(self, a: f32x16) -> mask32x16 { @@ -15260,11 +15351,36 @@ impl Simd for Avx2 { } #[inline(always)] fn compress_i32x16(self, values: i32x16, mask: mask32x16) -> i32x16 { - let mask = mask8x64 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: i32x16, mask: mask32x16) -> i32x16 { + let (low_mask, high_mask) = token.split_mask32x16(mask); + let low_count = low_mask.to_bitmask().count_ones() as usize; + let (low, high) = values.split(); + let bits = mask.to_bitmask(); + if bits == 0 { + return i32x16::splat(token, 0); + } + if bits == 65535 { + return values; + } + let low = low.compress(low_mask); + let high = high.compress(high_mask); + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::SHIFTS[low_count], + ); + let low_bits = crate::transmute::checked_transmute_copy(&low.val); + let high_bits = crate::transmute::checked_transmute_copy(&high.val); + let shifted = _mm256_permutevar8x32_epi32(high_bits, control); + let result_low = _mm256_blendv_epi8(shifted, low_bits, control); + let result_high = _mm256_and_si256(shifted, _mm256_srai_epi32::<31>(control)); + i32x16 { + val: crate::transmute::checked_transmute_copy(&[result_low, result_high]), + simd: token, + } + } + ); + kernel(self, values, mask) } #[inline(always)] fn compress_merge_i32x16( @@ -15273,23 +15389,69 @@ impl Simd for Avx2 { mask: mask32x16, merge: i32x16, ) -> i32x16 { - let mask = mask8x64 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_merge_u8x64( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: i32x16, + mask: mask32x16, + merge: i32x16, + ) -> i32x16 { + let (low_mask, high_mask) = token.split_mask32x16(mask); + let low_count = low_mask.to_bitmask().count_ones() as usize; + let (low, high) = values.split(); + let bits = mask.to_bitmask(); + if bits == 0 { + return merge; + } + if bits == 65535 { + return values; + } + let low = low.compress(low_mask); + let high = high.compress(high_mask); + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::SHIFTS[low_count], + ); + let low_bits = crate::transmute::checked_transmute_copy(&low.val); + let high_bits = crate::transmute::checked_transmute_copy(&high.val); + let shifted = _mm256_permutevar8x32_epi32(high_bits, control); + let result_low = _mm256_blendv_epi8(shifted, low_bits, control); + let result_high = _mm256_and_si256(shifted, _mm256_srai_epi32::<31>(control)); + let compressed = i32x16 { + val: crate::transmute::checked_transmute_copy(&[result_low, result_high]), + simd: token, + }; + let count = low_count + high_mask.to_bitmask().count_ones() as usize; + let prefix = mask32x16::from_bitmask(token, (1u64 << count) - 1); + token.select_i32x16(prefix, compressed, merge) + } + ); + kernel(self, values, mask, merge) } #[inline(always)] fn expand_i32x16(self, values: i32x16, mask: mask32x16) -> i32x16 { - let mask = mask8x64 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: i32x16, mask: mask32x16) -> i32x16 { + let (low_mask, high_mask) = token.split_mask32x16(mask); + let low_count = low_mask.to_bitmask().count_ones() as usize; + let (low, high) = values.split(); + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::SHIFTS[8 - (low_count)], + ); + let low_bits = crate::transmute::checked_transmute_copy(&low.val); + let high_bits = crate::transmute::checked_transmute_copy(&high.val); + let from_low = _mm256_permutevar8x32_epi32(low_bits, control); + let from_high = _mm256_permutevar8x32_epi32(high_bits, control); + let packed = _mm256_blendv_epi8(from_high, from_low, control); + let high = i32x8 { + val: crate::transmute::checked_transmute_copy(&packed), + simd: token, + }; + low.expand(low_mask).combine(high.expand(high_mask)) + } + ); + kernel(self, values, mask) } #[inline(always)] fn expand_merge_i32x16( @@ -15298,15 +15460,35 @@ impl Simd for Avx2 { mask: mask32x16, merge: i32x16, ) -> i32x16 { - let mask = mask8x64 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_merge_u8x64( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: i32x16, + mask: mask32x16, + merge: i32x16, + ) -> i32x16 { + let (low_mask, high_mask) = token.split_mask32x16(mask); + let low_count = low_mask.to_bitmask().count_ones() as usize; + let (low, high) = values.split(); + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::SHIFTS[8 - (low_count)], + ); + let low_bits = crate::transmute::checked_transmute_copy(&low.val); + let high_bits = crate::transmute::checked_transmute_copy(&high.val); + let from_low = _mm256_permutevar8x32_epi32(low_bits, control); + let from_high = _mm256_permutevar8x32_epi32(high_bits, control); + let packed = _mm256_blendv_epi8(from_high, from_low, control); + let high = i32x8 { + val: crate::transmute::checked_transmute_copy(&packed), + simd: token, + }; + let (low_merge, high_merge) = merge.split(); + low.expand_merge(low_mask, low_merge) + .combine(high.expand_merge(high_mask, high_merge)) + } + ); + kernel(self, values, mask, merge) } #[inline(always)] fn split_i32x16(self, a: i32x16) -> (i32x8, i32x8) { @@ -15339,11 +15521,36 @@ impl Simd for Avx2 { } #[inline(always)] fn compress_u32x16(self, values: u32x16, mask: mask32x16) -> u32x16 { - let mask = mask8x64 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: u32x16, mask: mask32x16) -> u32x16 { + let (low_mask, high_mask) = token.split_mask32x16(mask); + let low_count = low_mask.to_bitmask().count_ones() as usize; + let (low, high) = values.split(); + let bits = mask.to_bitmask(); + if bits == 0 { + return u32x16::splat(token, 0); + } + if bits == 65535 { + return values; + } + let low = low.compress(low_mask); + let high = high.compress(high_mask); + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::SHIFTS[low_count], + ); + let low_bits = crate::transmute::checked_transmute_copy(&low.val); + let high_bits = crate::transmute::checked_transmute_copy(&high.val); + let shifted = _mm256_permutevar8x32_epi32(high_bits, control); + let result_low = _mm256_blendv_epi8(shifted, low_bits, control); + let result_high = _mm256_and_si256(shifted, _mm256_srai_epi32::<31>(control)); + u32x16 { + val: crate::transmute::checked_transmute_copy(&[result_low, result_high]), + simd: token, + } + } + ); + kernel(self, values, mask) } #[inline(always)] fn compress_merge_u32x16( @@ -15352,23 +15559,69 @@ impl Simd for Avx2 { mask: mask32x16, merge: u32x16, ) -> u32x16 { - let mask = mask8x64 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_merge_u8x64( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: u32x16, + mask: mask32x16, + merge: u32x16, + ) -> u32x16 { + let (low_mask, high_mask) = token.split_mask32x16(mask); + let low_count = low_mask.to_bitmask().count_ones() as usize; + let (low, high) = values.split(); + let bits = mask.to_bitmask(); + if bits == 0 { + return merge; + } + if bits == 65535 { + return values; + } + let low = low.compress(low_mask); + let high = high.compress(high_mask); + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::SHIFTS[low_count], + ); + let low_bits = crate::transmute::checked_transmute_copy(&low.val); + let high_bits = crate::transmute::checked_transmute_copy(&high.val); + let shifted = _mm256_permutevar8x32_epi32(high_bits, control); + let result_low = _mm256_blendv_epi8(shifted, low_bits, control); + let result_high = _mm256_and_si256(shifted, _mm256_srai_epi32::<31>(control)); + let compressed = u32x16 { + val: crate::transmute::checked_transmute_copy(&[result_low, result_high]), + simd: token, + }; + let count = low_count + high_mask.to_bitmask().count_ones() as usize; + let prefix = mask32x16::from_bitmask(token, (1u64 << count) - 1); + token.select_u32x16(prefix, compressed, merge) + } + ); + kernel(self, values, mask, merge) } #[inline(always)] fn expand_u32x16(self, values: u32x16, mask: mask32x16) -> u32x16 { - let mask = mask8x64 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: u32x16, mask: mask32x16) -> u32x16 { + let (low_mask, high_mask) = token.split_mask32x16(mask); + let low_count = low_mask.to_bitmask().count_ones() as usize; + let (low, high) = values.split(); + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::SHIFTS[8 - (low_count)], + ); + let low_bits = crate::transmute::checked_transmute_copy(&low.val); + let high_bits = crate::transmute::checked_transmute_copy(&high.val); + let from_low = _mm256_permutevar8x32_epi32(low_bits, control); + let from_high = _mm256_permutevar8x32_epi32(high_bits, control); + let packed = _mm256_blendv_epi8(from_high, from_low, control); + let high = u32x8 { + val: crate::transmute::checked_transmute_copy(&packed), + simd: token, + }; + low.expand(low_mask).combine(high.expand(high_mask)) + } + ); + kernel(self, values, mask) } #[inline(always)] fn expand_merge_u32x16( @@ -15377,15 +15630,35 @@ impl Simd for Avx2 { mask: mask32x16, merge: u32x16, ) -> u32x16 { - let mask = mask8x64 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_merge_u8x64( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: u32x16, + mask: mask32x16, + merge: u32x16, + ) -> u32x16 { + let (low_mask, high_mask) = token.split_mask32x16(mask); + let low_count = low_mask.to_bitmask().count_ones() as usize; + let (low, high) = values.split(); + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::SHIFTS[8 - (low_count)], + ); + let low_bits = crate::transmute::checked_transmute_copy(&low.val); + let high_bits = crate::transmute::checked_transmute_copy(&high.val); + let from_low = _mm256_permutevar8x32_epi32(low_bits, control); + let from_high = _mm256_permutevar8x32_epi32(high_bits, control); + let packed = _mm256_blendv_epi8(from_high, from_low, control); + let high = u32x8 { + val: crate::transmute::checked_transmute_copy(&packed), + simd: token, + }; + let (low_merge, high_merge) = merge.split(); + low.expand_merge(low_mask, low_merge) + .combine(high.expand_merge(high_mask, high_merge)) + } + ); + kernel(self, values, mask, merge) } #[inline(always)] fn split_u32x16(self, a: u32x16) -> (u32x8, u32x8) { @@ -15499,11 +15772,36 @@ impl Simd for Avx2 { } #[inline(always)] fn compress_f64x8(self, values: f64x8, mask: mask64x8) -> f64x8 { - let mask = mask8x64 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: f64x8, mask: mask64x8) -> f64x8 { + let (low_mask, high_mask) = token.split_mask64x8(mask); + let low_count = low_mask.to_bitmask().count_ones() as usize; + let (low, high) = values.split(); + let bits = mask.to_bitmask(); + if bits == 0 { + return f64x8::splat(token, 0.0); + } + if bits == 255 { + return values; + } + let low = low.compress(low_mask); + let high = high.compress(high_mask); + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::SHIFTS[low_count * 2], + ); + let low_bits = crate::transmute::checked_transmute_copy(&low.val); + let high_bits = crate::transmute::checked_transmute_copy(&high.val); + let shifted = _mm256_permutevar8x32_epi32(high_bits, control); + let result_low = _mm256_blendv_epi8(shifted, low_bits, control); + let result_high = _mm256_and_si256(shifted, _mm256_srai_epi32::<31>(control)); + f64x8 { + val: crate::transmute::checked_transmute_copy(&[result_low, result_high]), + simd: token, + } + } + ); + kernel(self, values, mask) } #[inline(always)] fn compress_merge_f64x8( @@ -15512,23 +15810,69 @@ impl Simd for Avx2 { mask: mask64x8, merge: f64x8, ) -> f64x8 { - let mask = mask8x64 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_merge_u8x64( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: f64x8, + mask: mask64x8, + merge: f64x8, + ) -> f64x8 { + let (low_mask, high_mask) = token.split_mask64x8(mask); + let low_count = low_mask.to_bitmask().count_ones() as usize; + let (low, high) = values.split(); + let bits = mask.to_bitmask(); + if bits == 0 { + return merge; + } + if bits == 255 { + return values; + } + let low = low.compress(low_mask); + let high = high.compress(high_mask); + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::SHIFTS[low_count * 2], + ); + let low_bits = crate::transmute::checked_transmute_copy(&low.val); + let high_bits = crate::transmute::checked_transmute_copy(&high.val); + let shifted = _mm256_permutevar8x32_epi32(high_bits, control); + let result_low = _mm256_blendv_epi8(shifted, low_bits, control); + let result_high = _mm256_and_si256(shifted, _mm256_srai_epi32::<31>(control)); + let compressed = f64x8 { + val: crate::transmute::checked_transmute_copy(&[result_low, result_high]), + simd: token, + }; + let count = low_count + high_mask.to_bitmask().count_ones() as usize; + let prefix = mask64x8::from_bitmask(token, (1u64 << count) - 1); + token.select_f64x8(prefix, compressed, merge) + } + ); + kernel(self, values, mask, merge) } #[inline(always)] fn expand_f64x8(self, values: f64x8, mask: mask64x8) -> f64x8 { - let mask = mask8x64 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: f64x8, mask: mask64x8) -> f64x8 { + let (low_mask, high_mask) = token.split_mask64x8(mask); + let low_count = low_mask.to_bitmask().count_ones() as usize; + let (low, high) = values.split(); + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::SHIFTS[8 - (low_count * 2)], + ); + let low_bits = crate::transmute::checked_transmute_copy(&low.val); + let high_bits = crate::transmute::checked_transmute_copy(&high.val); + let from_low = _mm256_permutevar8x32_epi32(low_bits, control); + let from_high = _mm256_permutevar8x32_epi32(high_bits, control); + let packed = _mm256_blendv_epi8(from_high, from_low, control); + let high = f64x4 { + val: crate::transmute::checked_transmute_copy(&packed), + simd: token, + }; + low.expand(low_mask).combine(high.expand(high_mask)) + } + ); + kernel(self, values, mask) } #[inline(always)] fn expand_merge_f64x8( @@ -15537,15 +15881,35 @@ impl Simd for Avx2 { mask: mask64x8, merge: f64x8, ) -> f64x8 { - let mask = mask8x64 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_merge_u8x64( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: f64x8, + mask: mask64x8, + merge: f64x8, + ) -> f64x8 { + let (low_mask, high_mask) = token.split_mask64x8(mask); + let low_count = low_mask.to_bitmask().count_ones() as usize; + let (low, high) = values.split(); + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::SHIFTS[8 - (low_count * 2)], + ); + let low_bits = crate::transmute::checked_transmute_copy(&low.val); + let high_bits = crate::transmute::checked_transmute_copy(&high.val); + let from_low = _mm256_permutevar8x32_epi32(low_bits, control); + let from_high = _mm256_permutevar8x32_epi32(high_bits, control); + let packed = _mm256_blendv_epi8(from_high, from_low, control); + let high = f64x4 { + val: crate::transmute::checked_transmute_copy(&packed), + simd: token, + }; + let (low_merge, high_merge) = merge.split(); + low.expand_merge(low_mask, low_merge) + .combine(high.expand_merge(high_mask, high_merge)) + } + ); + kernel(self, values, mask, merge) } #[inline(always)] fn is_nan_f64x8(self, a: f64x8) -> mask64x8 { @@ -15610,11 +15974,36 @@ impl Simd for Avx2 { } #[inline(always)] fn compress_i64x8(self, values: i64x8, mask: mask64x8) -> i64x8 { - let mask = mask8x64 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: i64x8, mask: mask64x8) -> i64x8 { + let (low_mask, high_mask) = token.split_mask64x8(mask); + let low_count = low_mask.to_bitmask().count_ones() as usize; + let (low, high) = values.split(); + let bits = mask.to_bitmask(); + if bits == 0 { + return i64x8::splat(token, 0); + } + if bits == 255 { + return values; + } + let low = low.compress(low_mask); + let high = high.compress(high_mask); + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::SHIFTS[low_count * 2], + ); + let low_bits = crate::transmute::checked_transmute_copy(&low.val); + let high_bits = crate::transmute::checked_transmute_copy(&high.val); + let shifted = _mm256_permutevar8x32_epi32(high_bits, control); + let result_low = _mm256_blendv_epi8(shifted, low_bits, control); + let result_high = _mm256_and_si256(shifted, _mm256_srai_epi32::<31>(control)); + i64x8 { + val: crate::transmute::checked_transmute_copy(&[result_low, result_high]), + simd: token, + } + } + ); + kernel(self, values, mask) } #[inline(always)] fn compress_merge_i64x8( @@ -15623,23 +16012,69 @@ impl Simd for Avx2 { mask: mask64x8, merge: i64x8, ) -> i64x8 { - let mask = mask8x64 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_merge_u8x64( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: i64x8, + mask: mask64x8, + merge: i64x8, + ) -> i64x8 { + let (low_mask, high_mask) = token.split_mask64x8(mask); + let low_count = low_mask.to_bitmask().count_ones() as usize; + let (low, high) = values.split(); + let bits = mask.to_bitmask(); + if bits == 0 { + return merge; + } + if bits == 255 { + return values; + } + let low = low.compress(low_mask); + let high = high.compress(high_mask); + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::SHIFTS[low_count * 2], + ); + let low_bits = crate::transmute::checked_transmute_copy(&low.val); + let high_bits = crate::transmute::checked_transmute_copy(&high.val); + let shifted = _mm256_permutevar8x32_epi32(high_bits, control); + let result_low = _mm256_blendv_epi8(shifted, low_bits, control); + let result_high = _mm256_and_si256(shifted, _mm256_srai_epi32::<31>(control)); + let compressed = i64x8 { + val: crate::transmute::checked_transmute_copy(&[result_low, result_high]), + simd: token, + }; + let count = low_count + high_mask.to_bitmask().count_ones() as usize; + let prefix = mask64x8::from_bitmask(token, (1u64 << count) - 1); + token.select_i64x8(prefix, compressed, merge) + } + ); + kernel(self, values, mask, merge) } #[inline(always)] fn expand_i64x8(self, values: i64x8, mask: mask64x8) -> i64x8 { - let mask = mask8x64 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: i64x8, mask: mask64x8) -> i64x8 { + let (low_mask, high_mask) = token.split_mask64x8(mask); + let low_count = low_mask.to_bitmask().count_ones() as usize; + let (low, high) = values.split(); + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::SHIFTS[8 - (low_count * 2)], + ); + let low_bits = crate::transmute::checked_transmute_copy(&low.val); + let high_bits = crate::transmute::checked_transmute_copy(&high.val); + let from_low = _mm256_permutevar8x32_epi32(low_bits, control); + let from_high = _mm256_permutevar8x32_epi32(high_bits, control); + let packed = _mm256_blendv_epi8(from_high, from_low, control); + let high = i64x4 { + val: crate::transmute::checked_transmute_copy(&packed), + simd: token, + }; + low.expand(low_mask).combine(high.expand(high_mask)) + } + ); + kernel(self, values, mask) } #[inline(always)] fn expand_merge_i64x8( @@ -15648,15 +16083,35 @@ impl Simd for Avx2 { mask: mask64x8, merge: i64x8, ) -> i64x8 { - let mask = mask8x64 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_merge_u8x64( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: i64x8, + mask: mask64x8, + merge: i64x8, + ) -> i64x8 { + let (low_mask, high_mask) = token.split_mask64x8(mask); + let low_count = low_mask.to_bitmask().count_ones() as usize; + let (low, high) = values.split(); + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::SHIFTS[8 - (low_count * 2)], + ); + let low_bits = crate::transmute::checked_transmute_copy(&low.val); + let high_bits = crate::transmute::checked_transmute_copy(&high.val); + let from_low = _mm256_permutevar8x32_epi32(low_bits, control); + let from_high = _mm256_permutevar8x32_epi32(high_bits, control); + let packed = _mm256_blendv_epi8(from_high, from_low, control); + let high = i64x4 { + val: crate::transmute::checked_transmute_copy(&packed), + simd: token, + }; + let (low_merge, high_merge) = merge.split(); + low.expand_merge(low_mask, low_merge) + .combine(high.expand_merge(high_mask, high_merge)) + } + ); + kernel(self, values, mask, merge) } #[inline(always)] fn split_i64x8(self, a: i64x8) -> (i64x4, i64x4) { @@ -15689,11 +16144,36 @@ impl Simd for Avx2 { } #[inline(always)] fn compress_u64x8(self, values: u64x8, mask: mask64x8) -> u64x8 { - let mask = mask8x64 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: u64x8, mask: mask64x8) -> u64x8 { + let (low_mask, high_mask) = token.split_mask64x8(mask); + let low_count = low_mask.to_bitmask().count_ones() as usize; + let (low, high) = values.split(); + let bits = mask.to_bitmask(); + if bits == 0 { + return u64x8::splat(token, 0); + } + if bits == 255 { + return values; + } + let low = low.compress(low_mask); + let high = high.compress(high_mask); + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::SHIFTS[low_count * 2], + ); + let low_bits = crate::transmute::checked_transmute_copy(&low.val); + let high_bits = crate::transmute::checked_transmute_copy(&high.val); + let shifted = _mm256_permutevar8x32_epi32(high_bits, control); + let result_low = _mm256_blendv_epi8(shifted, low_bits, control); + let result_high = _mm256_and_si256(shifted, _mm256_srai_epi32::<31>(control)); + u64x8 { + val: crate::transmute::checked_transmute_copy(&[result_low, result_high]), + simd: token, + } + } + ); + kernel(self, values, mask) } #[inline(always)] fn compress_merge_u64x8( @@ -15702,23 +16182,69 @@ impl Simd for Avx2 { mask: mask64x8, merge: u64x8, ) -> u64x8 { - let mask = mask8x64 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.compress_merge_u8x64( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: u64x8, + mask: mask64x8, + merge: u64x8, + ) -> u64x8 { + let (low_mask, high_mask) = token.split_mask64x8(mask); + let low_count = low_mask.to_bitmask().count_ones() as usize; + let (low, high) = values.split(); + let bits = mask.to_bitmask(); + if bits == 0 { + return merge; + } + if bits == 255 { + return values; + } + let low = low.compress(low_mask); + let high = high.compress(high_mask); + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::SHIFTS[low_count * 2], + ); + let low_bits = crate::transmute::checked_transmute_copy(&low.val); + let high_bits = crate::transmute::checked_transmute_copy(&high.val); + let shifted = _mm256_permutevar8x32_epi32(high_bits, control); + let result_low = _mm256_blendv_epi8(shifted, low_bits, control); + let result_high = _mm256_and_si256(shifted, _mm256_srai_epi32::<31>(control)); + let compressed = u64x8 { + val: crate::transmute::checked_transmute_copy(&[result_low, result_high]), + simd: token, + }; + let count = low_count + high_mask.to_bitmask().count_ones() as usize; + let prefix = mask64x8::from_bitmask(token, (1u64 << count) - 1); + token.select_u64x8(prefix, compressed, merge) + } + ); + kernel(self, values, mask, merge) } #[inline(always)] fn expand_u64x8(self, values: u64x8, mask: mask64x8) -> u64x8 { - let mask = mask8x64 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: u64x8, mask: mask64x8) -> u64x8 { + let (low_mask, high_mask) = token.split_mask64x8(mask); + let low_count = low_mask.to_bitmask().count_ones() as usize; + let (low, high) = values.split(); + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::SHIFTS[8 - (low_count * 2)], + ); + let low_bits = crate::transmute::checked_transmute_copy(&low.val); + let high_bits = crate::transmute::checked_transmute_copy(&high.val); + let from_low = _mm256_permutevar8x32_epi32(low_bits, control); + let from_high = _mm256_permutevar8x32_epi32(high_bits, control); + let packed = _mm256_blendv_epi8(from_high, from_low, control); + let high = u64x4 { + val: crate::transmute::checked_transmute_copy(&packed), + simd: token, + }; + low.expand(low_mask).combine(high.expand(high_mask)) + } + ); + kernel(self, values, mask) } #[inline(always)] fn expand_merge_u64x8( @@ -15727,15 +16253,35 @@ impl Simd for Avx2 { mask: mask64x8, merge: u64x8, ) -> u64x8 { - let mask = mask8x64 { - val: crate::transmute::checked_transmute_copy(&mask.val), - simd: self, - }; - Bytes::from_bytes(self.expand_merge_u8x64( - Bytes::to_bytes(values), - mask, - Bytes::to_bytes(merge), - )) + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: u64x8, + mask: mask64x8, + merge: u64x8, + ) -> u64x8 { + let (low_mask, high_mask) = token.split_mask64x8(mask); + let low_count = low_mask.to_bitmask().count_ones() as usize; + let (low, high) = values.split(); + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::SHIFTS[8 - (low_count * 2)], + ); + let low_bits = crate::transmute::checked_transmute_copy(&low.val); + let high_bits = crate::transmute::checked_transmute_copy(&high.val); + let from_low = _mm256_permutevar8x32_epi32(low_bits, control); + let from_high = _mm256_permutevar8x32_epi32(high_bits, control); + let packed = _mm256_blendv_epi8(from_high, from_low, control); + let high = u64x4 { + val: crate::transmute::checked_transmute_copy(&packed), + simd: token, + }; + let (low_merge, high_merge) = merge.split(); + low.expand_merge(low_mask, low_merge) + .combine(high.expand_merge(high_mask, high_merge)) + } + ); + kernel(self, values, mask, merge) } #[inline(always)] fn split_u64x8(self, a: u64x8) -> (u64x4, u64x4) { diff --git a/fearless_simd/src/support.rs b/fearless_simd/src/support.rs index 7ea1c3982..5a87cae35 100644 --- a/fearless_simd/src/support.rs +++ b/fearless_simd/src/support.rs @@ -174,6 +174,30 @@ pub(crate) mod compact_256 { pub(crate) static EXPAND_32: [Aligned256<[i32; 8]>; 256] = controls(1, true); pub(crate) static COMPRESS_64: [Aligned256<[i32; 8]>; 16] = controls(2, false); pub(crate) static EXPAND_64: [Aligned256<[i32; 8]>; 16] = controls(2, true); + + #[allow( + clippy::cast_possible_truncation, + clippy::cast_possible_wrap, + reason = "Dword indices and offsets are at most eight" + )] + const fn shifts() -> [Aligned256<[i32; 8]>; 9] { + let mut table = [Aligned256([0; 8]); 9]; + let mut offset = 0; + while offset <= 8 { + let mut lane = 0; + while lane < 8 { + table[offset].0[lane] = lane as i32 - offset as i32; + lane += 1; + } + offset += 1; + } + table + } + + // VPERMD uses the low three bits to rotate by the offset. Negative controls + // (-8..=-1) have every byte's sign bit set, so the same row is a blend mask + // for the prefix crossing between two YMM registers. Qwords use even offsets. + pub(crate) static SHIFTS: [Aligned256<[i32; 8]>; 9] = shifts(); } const fn compact_control_table(expand: bool) -> [u64; 256] { diff --git a/fearless_simd_gen/src/mk_x86.rs b/fearless_simd_gen/src/mk_x86.rs index cbef15c7b..ac11838fc 100644 --- a/fearless_simd_gen/src/mk_x86.rs +++ b/fearless_simd_gen/src/mk_x86.rs @@ -332,6 +332,11 @@ impl Level for X86 { OpSig::Compress { .. } | OpSig::Expand { .. } => { if *self == Self::Avx512 { self.handle_avx512_compact_op(op, vec_ty) + } else if *self == Self::Avx2 + && vec_ty.n_bits() == 512 + && matches!(vec_ty.scalar_bits, 32 | 64) + { + self.handle_avx2_wide_dword_compact(op, vec_ty) } else if *self == Self::Avx2 && vec_ty.n_bits() == 256 && matches!(vec_ty.scalar_bits, 32 | 64) @@ -1299,6 +1304,111 @@ fn expand_16_control(block_bits: TokenStream) -> TokenStream { } impl X86 { + /// Reuse native dword/qword compaction, joining the halves with register permutations. + fn handle_avx2_wide_dword_compact(&self, op: Op, vec_ty: &VecType) -> TokenStream { + assert!(*self == Self::Avx2, "wide dword compaction requires AVX2"); + assert_eq!(vec_ty.n_bits(), 512, "requires two native AVX2 vectors"); + assert!( + matches!(vec_ty.scalar_bits, 32 | 64), + "requires dword/qword elements" + ); + let vec = vec_ty.rust(); + let half = vec_ty.split_operand().unwrap().rust(); + let low_dwords = if vec_ty.scalar_bits == 32 { + quote! { low_count } + } else { + quote! { low_count * 2 } + }; + let split_mask = generic_op_name("split", &vec_ty.mask_ty()); + self.kernel_method(op, vec_ty, |token| { + let body = match op.sig { + OpSig::Compress { merge } => { + let empty = if merge { + quote! { merge } + } else if vec_ty.scalar == ScalarType::Float { + quote! { #vec::splat(#token, 0.0) } + } else { + quote! { #vec::splat(#token, 0) } + }; + let all = Literal::u64_unsuffixed((1_u64 << vec_ty.len) - 1); + let compressed = quote! { + #vec { + val: crate::transmute::checked_transmute_copy(&[result_low, result_high]), + simd: #token, + } + }; + let finish = if merge { + let mask_ty = vec_ty.mask_ty().rust(); + let select = generic_op_name("select", vec_ty); + quote! { + let compressed = #compressed; + let count = low_count + high_mask.to_bitmask().count_ones() as usize; + let prefix = #mask_ty::from_bitmask(#token, (1u64 << count) - 1); + #token.#select(prefix, compressed, merge) + } + } else { + compressed + }; + quote! { + let bits = mask.to_bitmask(); + if bits == 0 { return #empty; } + if bits == #all { return values; } + let low = low.compress(low_mask); + let high = high.compress(high_mask); + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::SHIFTS[#low_dwords], + ); + let low_bits = crate::transmute::checked_transmute_copy(&low.val); + let high_bits = crate::transmute::checked_transmute_copy(&high.val); + let shifted = _mm256_permutevar8x32_epi32(high_bits, control); + // Negative controls identify the low half's selected prefix. + // The rotated high half fills the rest, spilling its tail + // into that same prefix of the second output register. + let result_low = _mm256_blendv_epi8(shifted, low_bits, control); + let result_high = _mm256_and_si256(shifted, _mm256_srai_epi32::<31>(control)); + #finish + } + } + OpSig::Expand { merge } => { + let finish = if merge { + quote! { + let (low_merge, high_merge) = merge.split(); + low.expand_merge(low_mask, low_merge) + .combine(high.expand_merge(high_mask, high_merge)) + } + } else { + quote! { low.expand(low_mask).combine(high.expand(high_mask)) } + }; + quote! { + // Assemble the packed window starting after the elements + // consumed by the low half. The same controls rotate both + // inputs; negative indices select from the low register. + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::SHIFTS[8 - (#low_dwords)], + ); + let low_bits = crate::transmute::checked_transmute_copy(&low.val); + let high_bits = crate::transmute::checked_transmute_copy(&high.val); + let from_low = _mm256_permutevar8x32_epi32(low_bits, control); + let from_high = _mm256_permutevar8x32_epi32(high_bits, control); + let packed = _mm256_blendv_epi8(from_high, from_low, control); + let high = #half { + val: crate::transmute::checked_transmute_copy(&packed), + simd: #token, + }; + #finish + } + } + _ => unreachable!("only compact operations reuse the native halves"), + }; + quote! { + let (low_mask, high_mask) = #token.#split_mask(mask); + let low_count = low_mask.to_bitmask().count_ones() as usize; + let (low, high) = values.split(); + #body + } + }) + } + /// Compact whole dwords/qwords with a single full-width AVX2 permutation. fn handle_avx2_dword_compact(&self, op: Op, vec_ty: &VecType) -> TokenStream { assert!(*self == Self::Avx2, "dword compaction requires AVX2"); diff --git a/fearless_simd_tests/tests/harness/ops/compress.rs b/fearless_simd_tests/tests/harness/ops/compress.rs index 0f889cf9a..81adc9996 100644 --- a/fearless_simd_tests/tests/harness/ops/compress.rs +++ b/fearless_simd_tests/tests/harness/ops/compress.rs @@ -899,6 +899,7 @@ fn compact_u32x16(simd: S) { .chain(single_lanes) .chain(prefixes) .chain(suffixes) + .chain(0..=all_lanes) { let mask = mask32x16::from_bitmask(simd, mask_bits); let mut expected_compress = [0; 16]; @@ -1087,6 +1088,7 @@ fn compact_i32x16(simd: S) { .chain(single_lanes) .chain(prefixes) .chain(suffixes) + .chain(0..=all_lanes) { let mask = mask32x16::from_bitmask(simd, mask_bits); let mut expected_compress = [0; 16]; @@ -1272,6 +1274,7 @@ fn compact_u64x8(simd: S) { .chain(single_lanes) .chain(prefixes) .chain(suffixes) + .chain(0..=all_lanes) { let mask = mask64x8::from_bitmask(simd, mask_bits); let mut expected_compress = [0; 8]; @@ -1460,6 +1463,7 @@ fn compact_i64x8(simd: S) { .chain(single_lanes) .chain(prefixes) .chain(suffixes) + .chain(0..=all_lanes) { let mask = mask64x8::from_bitmask(simd, mask_bits); let mut expected_compress = [0; 8]; @@ -1724,6 +1728,7 @@ fn compact_f32x16(simd: S) { .chain(single_lanes) .chain(prefixes) .chain(suffixes) + .chain(0..=all_lanes) { let mask = mask32x16::from_bitmask(simd, mask_bits); let mut expected_compress = [0; 16]; @@ -2014,6 +2019,7 @@ fn compact_f64x8(simd: S) { .chain(single_lanes) .chain(prefixes) .chain(suffixes) + .chain(0..=all_lanes) { let mask = mask64x8::from_bitmask(simd, mask_bits); let mut expected_compress = [0; 8]; From bf7d73427ff156dbfcd478eb871f209fd28b5532 Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Wed, 30 Sep 2026 12:41:44 +0100 Subject: [PATCH 8/8] Drop redundant test --- .../tests/harness/ops/compress.rs | 45 ------------------- 1 file changed, 45 deletions(-) diff --git a/fearless_simd_tests/tests/harness/ops/compress.rs b/fearless_simd_tests/tests/harness/ops/compress.rs index 81adc9996..73519f5ed 100644 --- a/fearless_simd_tests/tests/harness/ops/compress.rs +++ b/fearless_simd_tests/tests/harness/ops/compress.rs @@ -2191,48 +2191,3 @@ fn compress_u8x16_exhaustive_masks(simd: S) { ); } } - -#[simd_test] -fn compress_u32x8_exhaustive_masks(simd: S) { - let values = [ - 0, - u32::MAX, - 0x8000_0000, - 1, - 0x7fff_ffff, - 0x1234_5678, - 0xfedc_ba98, - 0x0102_0304, - ]; - let merge = [ - 0xa0b0_c001, - 0xa0b0_c002, - 0xa0b0_c003, - 0xa0b0_c004, - 0xa0b0_c005, - 0xa0b0_c006, - 0xa0b0_c007, - 0xa0b0_c008, - ]; - let values_vec = u32x8::simd_from(simd, values); - let merge_vec = u32x8::simd_from(simd, merge); - for bits in 0..=u8::MAX { - let mask = mask32x8::from_bitmask(simd, u64::from(bits)); - let mut expected = [0; 8]; - let mut expected_merge = merge; - let mut selected = 0; - for (lane, value) in values.into_iter().enumerate() { - if bits & (1 << lane) != 0 { - expected[selected] = value; - expected_merge[selected] = value; - selected += 1; - } - } - assert_eq!(*values_vec.compress(mask), expected, "mask {bits:#04x}"); - assert_eq!( - *values_vec.compress_merge(mask, merge_vec), - expected_merge, - "mask {bits:#04x}" - ); - } -}