diff --git a/fearless_simd/src/generated/avx2.rs b/fearless_simd/src/generated/avx2.rs index 85e5a553..3a636dd6 100644 --- a/fearless_simd/src/generated/avx2.rs +++ b/fearless_simd/src/generated/avx2.rs @@ -166,6 +166,43 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn compress_f32x4(self, values: f32x4, mask: mask32x4) -> f32x4 { + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn compress_merge_f32x4( + self, + values: f32x4, + mask: mask32x4, + merge: f32x4, + ) -> f32x4 { + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0); + let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control); + let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0)); + Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed)) + } + #[inline(always)] + fn expand_f32x4(self, values: f32x4, mask: mask32x4) -> f32x4 { + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn expand_merge_f32x4( + self, + values: f32x4, + mask: mask32x4, + merge: f32x4, + ) -> f32x4 { + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0); + let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control)); + self.select_f32x4(mask, expanded, merge) + } + #[inline(always)] fn neg_f32x4(self, a: f32x4) -> f32x4 { crate::kernel!( #[inline(always)] @@ -831,6 +868,56 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn compress_i8x16(self, values: i8x16, mask: mask8x16) -> i8x16 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i8x16( + self, + values: i8x16, + mask: mask8x16, + merge: i8x16, + ) -> i8x16 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i8x16(self, values: i8x16, mask: mask8x16) -> i8x16 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i8x16( + self, + values: i8x16, + mask: mask8x16, + merge: i8x16, + ) -> i8x16 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_i8x16(self, a: i8x16) -> i8x16 { crate::kernel!( #[inline(always)] @@ -1462,6 +1549,114 @@ impl Simd for Avx2 { Bytes::from_bytes(result_bytes) } #[inline(always)] + fn compress_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: u8x16, mask: mask8x16) -> u8x16 { + let mask_bits = token.to_bitmask_mask8x16(mask) as u16; + let low_mask = (usize::from(mask_bits)) & 0xff; + let high_mask = (usize::from(mask_bits)) >> 8; + let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask]; + let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask]; + let high_control = high_control | 0x0808_0808_0808_0808; + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let low_count = low_mask.count_ones() as usize; + let compacted = _mm_shuffle_epi8(values.into(), control); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count]; + let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let compressed = _mm_shuffle_epi8(compacted, splice); + compressed.simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_u8x16( + self, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + let mask_bits = token.to_bitmask_mask8x16(mask) as u16; + let low_mask = (usize::from(mask_bits)) & 0xff; + let high_mask = (usize::from(mask_bits)) >> 8; + let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask]; + let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask]; + let high_control = high_control | 0x0808_0808_0808_0808; + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let low_count = low_mask.count_ones() as usize; + let compacted = _mm_shuffle_epi8(values.into(), control); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count]; + let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let compressed = _mm_shuffle_epi8(compacted, splice); + let count = mask_bits.count_ones() as usize; + let [prefix, _] = crate::transmute::checked_transmute_copy::<_, [__m128i; 2]>( + &crate::support::COMPACT_PREFIX_MASKS[count], + ); + _mm_blendv_epi8(merge.into(), compressed, prefix).simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: u8x16, mask: mask8x16) -> u8x16 { + let mask_bits = token.to_bitmask_mask8x16(mask) as u16; + let low_mask = (usize::from(mask_bits)) & 0xff; + let high_mask = (usize::from(mask_bits)) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = high_control + high_base; + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let expanded = _mm_shuffle_epi8(values.into(), control); + expanded.simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_u8x16( + self, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + let mask_bits = token.to_bitmask_mask8x16(mask) as u16; + let low_mask = (usize::from(mask_bits)) & 0xff; + let high_mask = (usize::from(mask_bits)) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = high_control + high_base; + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let expanded = _mm_shuffle_epi8(values.into(), control); + _mm_blendv_epi8(merge.into(), expanded, mask.into()).simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn count_ones_u8x16(self, a: u8x16) -> u8x16 { crate::kernel!( #[inline(always)] @@ -2245,6 +2440,43 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn compress_i16x8(self, values: i16x8, mask: mask16x8) -> i16x8 { + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_16[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn compress_merge_i16x8( + self, + values: i16x8, + mask: mask16x8, + merge: i16x8, + ) -> i16x8 { + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_16[bits].0); + let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control); + let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0)); + Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed)) + } + #[inline(always)] + fn expand_i16x8(self, values: i16x8, mask: mask16x8) -> i16x8 { + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_16[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn expand_merge_i16x8( + self, + values: i16x8, + mask: mask16x8, + merge: i16x8, + ) -> i16x8 { + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_16[bits].0); + let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control)); + self.select_i16x8(mask, expanded, merge) + } + #[inline(always)] fn count_ones_i16x8(self, a: i16x8) -> i16x8 { crate::kernel!( #[inline(always)] @@ -2793,6 +3025,43 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn compress_u16x8(self, values: u16x8, mask: mask16x8) -> u16x8 { + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_16[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn compress_merge_u16x8( + self, + values: u16x8, + mask: mask16x8, + merge: u16x8, + ) -> u16x8 { + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_16[bits].0); + let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control); + let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0)); + Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed)) + } + #[inline(always)] + fn expand_u16x8(self, values: u16x8, mask: mask16x8) -> u16x8 { + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_16[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn expand_merge_u16x8( + self, + values: u16x8, + mask: mask16x8, + merge: u16x8, + ) -> u16x8 { + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_16[bits].0); + let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control)); + self.select_u16x8(mask, expanded, merge) + } + #[inline(always)] fn count_ones_u16x8(self, a: u16x8) -> u16x8 { crate::kernel!( #[inline(always)] @@ -3582,6 +3851,43 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn compress_i32x4(self, values: i32x4, mask: mask32x4) -> i32x4 { + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn compress_merge_i32x4( + self, + values: i32x4, + mask: mask32x4, + merge: i32x4, + ) -> i32x4 { + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0); + let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control); + let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0)); + Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed)) + } + #[inline(always)] + fn expand_i32x4(self, values: i32x4, mask: mask32x4) -> i32x4 { + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn expand_merge_i32x4( + self, + values: i32x4, + mask: mask32x4, + merge: i32x4, + ) -> i32x4 { + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0); + let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control)); + self.select_i32x4(mask, expanded, merge) + } + #[inline(always)] fn count_ones_i32x4(self, a: i32x4) -> i32x4 { crate::kernel!( #[inline(always)] @@ -4121,6 +4427,43 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn compress_u32x4(self, values: u32x4, mask: mask32x4) -> u32x4 { + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn compress_merge_u32x4( + self, + values: u32x4, + mask: mask32x4, + merge: u32x4, + ) -> u32x4 { + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0); + let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control); + let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0)); + Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed)) + } + #[inline(always)] + fn expand_u32x4(self, values: u32x4, mask: mask32x4) -> u32x4 { + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn expand_merge_u32x4( + self, + values: u32x4, + mask: mask32x4, + merge: u32x4, + ) -> u32x4 { + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0); + let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control)); + self.select_u32x4(mask, expanded, merge) + } + #[inline(always)] fn count_ones_u32x4(self, a: u32x4) -> u32x4 { crate::kernel!( #[inline(always)] @@ -4889,6 +5232,43 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn compress_f64x2(self, values: f64x2, mask: mask64x2) -> f64x2 { + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn compress_merge_f64x2( + self, + values: f64x2, + mask: mask64x2, + merge: f64x2, + ) -> f64x2 { + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0); + let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control); + let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0)); + Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed)) + } + #[inline(always)] + fn expand_f64x2(self, values: f64x2, mask: mask64x2) -> f64x2 { + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn expand_merge_f64x2( + self, + values: f64x2, + mask: mask64x2, + merge: f64x2, + ) -> f64x2 { + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0); + let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control)); + self.select_f64x2(mask, expanded, merge) + } + #[inline(always)] fn neg_f64x2(self, a: f64x2) -> f64x2 { crate::kernel!( #[inline(always)] @@ -5554,6 +5934,43 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn compress_i64x2(self, values: i64x2, mask: mask64x2) -> i64x2 { + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn compress_merge_i64x2( + self, + values: i64x2, + mask: mask64x2, + merge: i64x2, + ) -> i64x2 { + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0); + let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control); + let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0)); + Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed)) + } + #[inline(always)] + fn expand_i64x2(self, values: i64x2, mask: mask64x2) -> i64x2 { + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn expand_merge_i64x2( + self, + values: i64x2, + mask: mask64x2, + merge: i64x2, + ) -> i64x2 { + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0); + let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control)); + self.select_i64x2(mask, expanded, merge) + } + #[inline(always)] fn count_ones_i64x2(self, a: i64x2) -> i64x2 { crate::kernel!( #[inline(always)] @@ -6090,6 +6507,43 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn compress_u64x2(self, values: u64x2, mask: mask64x2) -> u64x2 { + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn compress_merge_u64x2( + self, + values: u64x2, + mask: mask64x2, + merge: u64x2, + ) -> u64x2 { + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0); + let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control); + let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0)); + Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed)) + } + #[inline(always)] + fn expand_u64x2(self, values: u64x2, mask: mask64x2) -> u64x2 { + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn expand_merge_u64x2( + self, + values: u64x2, + mask: mask64x2, + merge: u64x2, + ) -> u64x2 { + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0); + let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control)); + self.select_u64x2(mask, expanded, merge) + } + #[inline(always)] fn count_ones_u64x2(self, a: u64x2) -> u64x2 { crate::kernel!( #[inline(always)] @@ -6874,6 +7328,96 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn compress_f32x8(self, values: f32x8, mask: mask32x8) -> f32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: f32x8, mask: mask32x8) -> f32x8 { + let bits = token.to_bitmask_mask32x8(mask) as usize & 255; + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::COMPRESS_32[bits], + ); + let reordered = + _mm256_permutevar8x32_epi32(_mm256_castps_si256(values.into()), control); + let result = _mm256_andnot_si256(_mm256_srai_epi32::<31>(control), reordered); + _mm256_castsi256_ps(result).simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_f32x8( + self, + values: f32x8, + mask: mask32x8, + merge: f32x8, + ) -> f32x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: f32x8, + mask: mask32x8, + merge: f32x8, + ) -> f32x8 { + let bits = token.to_bitmask_mask32x8(mask) as usize & 255; + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::COMPRESS_32[bits], + ); + let reordered = + _mm256_permutevar8x32_epi32(_mm256_castps_si256(values.into()), control); + let result = + _mm256_blendv_epi8(reordered, _mm256_castps_si256(merge.into()), control); + _mm256_castsi256_ps(result).simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_f32x8(self, values: f32x8, mask: mask32x8) -> f32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: f32x8, mask: mask32x8) -> f32x8 { + let bits = token.to_bitmask_mask32x8(mask) as usize & 255; + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::EXPAND_32[bits], + ); + let reordered = + _mm256_permutevar8x32_epi32(_mm256_castps_si256(values.into()), control); + let result = _mm256_and_si256(reordered, mask.into()); + _mm256_castsi256_ps(result).simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_f32x8( + self, + values: f32x8, + mask: mask32x8, + merge: f32x8, + ) -> f32x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: f32x8, + mask: mask32x8, + merge: f32x8, + ) -> f32x8 { + let bits = token.to_bitmask_mask32x8(mask) as usize & 255; + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::EXPAND_32[bits], + ); + let reordered = + _mm256_permutevar8x32_epi32(_mm256_castps_si256(values.into()), control); + let result = + _mm256_blendv_epi8(_mm256_castps_si256(merge.into()), reordered, mask.into()); + _mm256_castsi256_ps(result).simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn neg_f32x8(self, a: f32x8) -> f32x8 { crate::kernel!( #[inline(always)] @@ -7430,6 +7974,56 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn compress_i8x32(self, values: i8x32, mask: mask8x32) -> i8x32 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i8x32( + self, + values: i8x32, + mask: mask8x32, + merge: i8x32, + ) -> i8x32 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i8x32(self, values: i8x32, mask: mask8x32) -> i8x32 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i8x32( + self, + values: i8x32, + mask: mask8x32, + merge: i8x32, + ) -> i8x32 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_i8x32(self, a: i8x32) -> i8x32 { crate::kernel!( #[inline(always)] @@ -8048,6 +8642,230 @@ impl Simd for Avx2 { kernel(self, a, b, indices) } #[inline(always)] + fn compress_u8x32(self, values: u8x32, mask: mask8x32) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: u8x32, mask: mask8x32) -> u8x32 { + let mask_bits = token.to_bitmask_mask8x32(mask); + let block_bits_0 = (mask_bits & 0xffff) as usize; + let low_mask_0 = (block_bits_0) & 0xff; + let high_mask_0 = (block_bits_0) >> 8; + let low_control_0 = crate::support::COMPRESS_8_CONTROLS[low_mask_0]; + let high_control_0 = crate::support::COMPRESS_8_CONTROLS[high_mask_0]; + let high_control_0 = high_control_0 | 0x0808_0808_0808_0808; + let control_0 = + _mm_set_epi64x(high_control_0.cast_signed(), low_control_0.cast_signed()); + let low_count_0 = low_mask_0.count_ones() as usize; + let block_bits_1 = (mask_bits >> 16 & 0xffff) as usize; + let low_mask_1 = (block_bits_1) & 0xff; + let high_mask_1 = (block_bits_1) >> 8; + let low_control_1 = crate::support::COMPRESS_8_CONTROLS[low_mask_1]; + let high_control_1 = crate::support::COMPRESS_8_CONTROLS[high_mask_1]; + let high_control_1 = high_control_1 | 0x0808_0808_0808_0808; + let control_1 = + _mm_set_epi64x(high_control_1.cast_signed(), low_control_1.cast_signed()); + let low_count_1 = low_mask_1.count_ones() as usize; + let control = _mm256_set_m128i(control_1, control_0); + let compacted = _mm256_shuffle_epi8(values.val.0, control); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count_0]; + let splice_0 = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count_1]; + let splice_1 = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let splice = _mm256_set_m128i(splice_1, splice_0); + let compressed = _mm256_shuffle_epi8(compacted, splice); + let splice = crate::transmute::checked_transmute_copy( + &crate::support::COMPACT_16_SPLICE_CONTROLS[block_bits_0.count_ones() as usize], + ); + let compressed_low = _mm256_permute2x128_si256::<0x80>(compressed, compressed); + let compressed_high = _mm256_permute2x128_si256::<0x11>(compressed, compressed); + let compressed_0 = + _mm256_or_si256(compressed_low, _mm256_shuffle_epi8(compressed_high, splice)); + let result_0 = compressed_0; + result_0.simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_u8x32( + self, + values: u8x32, + mask: mask8x32, + merge: u8x32, + ) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: u8x32, + mask: mask8x32, + merge: u8x32, + ) -> u8x32 { + let mask_bits = token.to_bitmask_mask8x32(mask); + let block_bits_0 = (mask_bits & 0xffff) as usize; + let low_mask_0 = (block_bits_0) & 0xff; + let high_mask_0 = (block_bits_0) >> 8; + let low_control_0 = crate::support::COMPRESS_8_CONTROLS[low_mask_0]; + let high_control_0 = crate::support::COMPRESS_8_CONTROLS[high_mask_0]; + let high_control_0 = high_control_0 | 0x0808_0808_0808_0808; + let control_0 = + _mm_set_epi64x(high_control_0.cast_signed(), low_control_0.cast_signed()); + let low_count_0 = low_mask_0.count_ones() as usize; + let block_bits_1 = (mask_bits >> 16 & 0xffff) as usize; + let low_mask_1 = (block_bits_1) & 0xff; + let high_mask_1 = (block_bits_1) >> 8; + let low_control_1 = crate::support::COMPRESS_8_CONTROLS[low_mask_1]; + let high_control_1 = crate::support::COMPRESS_8_CONTROLS[high_mask_1]; + let high_control_1 = high_control_1 | 0x0808_0808_0808_0808; + let control_1 = + _mm_set_epi64x(high_control_1.cast_signed(), low_control_1.cast_signed()); + let low_count_1 = low_mask_1.count_ones() as usize; + let control = _mm256_set_m128i(control_1, control_0); + let compacted = _mm256_shuffle_epi8(values.val.0, control); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count_0]; + let splice_0 = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count_1]; + let splice_1 = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let splice = _mm256_set_m128i(splice_1, splice_0); + let compressed = _mm256_shuffle_epi8(compacted, splice); + let splice = crate::transmute::checked_transmute_copy( + &crate::support::COMPACT_16_SPLICE_CONTROLS[block_bits_0.count_ones() as usize], + ); + let compressed_low = _mm256_permute2x128_si256::<0x80>(compressed, compressed); + let compressed_high = _mm256_permute2x128_si256::<0x11>(compressed, compressed); + let compressed_0 = + _mm256_or_si256(compressed_low, _mm256_shuffle_epi8(compressed_high, splice)); + let output_lane = (block_bits_0.count_ones() + block_bits_1.count_ones()) as usize; + let prefix = crate::transmute::checked_transmute_copy( + &crate::support::COMPACT_PREFIX_MASKS[output_lane.min(32)], + ); + let result_0 = _mm256_blendv_epi8(merge.val.0, compressed_0, prefix); + result_0.simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_u8x32(self, values: u8x32, mask: mask8x32) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: u8x32, mask: mask8x32) -> u8x32 { + let input = <[u8; 32]>::from(values); + let mut output = [0u8; 32]; + let mut input_lane = 0; + let block_bits = _mm_movemask_epi8(_mm256_castsi256_si128(mask.val.0)) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = high_control + high_base; + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = crate::transmute::checked_transmute_copy( + input[input_lane..].first_chunk::<16>().unwrap(), + ); + let expanded = _mm_shuffle_epi8(packed, control); + let result = expanded; + crate::transmute::checked_transmute_store( + result, + output[0..].first_chunk_mut::<16>().unwrap(), + ); + input_lane += block_bits.count_ones() as usize; + let block_bits = + _mm_movemask_epi8(_mm256_extracti128_si256::<1>(mask.val.0)) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = high_control + high_base; + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = crate::transmute::checked_transmute_copy( + input[input_lane..].first_chunk::<16>().unwrap(), + ); + let expanded = _mm_shuffle_epi8(packed, control); + let result = expanded; + crate::transmute::checked_transmute_store( + result, + output[16..].first_chunk_mut::<16>().unwrap(), + ); + u8x32::simd_from(token, output) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_u8x32( + self, + values: u8x32, + mask: mask8x32, + merge: u8x32, + ) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: u8x32, + mask: mask8x32, + merge: u8x32, + ) -> u8x32 { + let input = <[u8; 32]>::from(values); + let mut output = [0u8; 32]; + let mut input_lane = 0; + let block_bits = _mm_movemask_epi8(_mm256_castsi256_si128(mask.val.0)) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = high_control + high_base; + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = crate::transmute::checked_transmute_copy( + input[input_lane..].first_chunk::<16>().unwrap(), + ); + let expanded = _mm_shuffle_epi8(packed, control); + let result = _mm_blendv_epi8( + _mm256_castsi256_si128(merge.val.0), + expanded, + _mm256_castsi256_si128(mask.val.0), + ); + crate::transmute::checked_transmute_store( + result, + output[0..].first_chunk_mut::<16>().unwrap(), + ); + input_lane += block_bits.count_ones() as usize; + let block_bits = + _mm_movemask_epi8(_mm256_extracti128_si256::<1>(mask.val.0)) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = high_control + high_base; + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = crate::transmute::checked_transmute_copy( + input[input_lane..].first_chunk::<16>().unwrap(), + ); + let expanded = _mm_shuffle_epi8(packed, control); + let result = _mm_blendv_epi8( + _mm256_extracti128_si256::<1>(merge.val.0), + expanded, + _mm256_extracti128_si256::<1>(mask.val.0), + ); + crate::transmute::checked_transmute_store( + result, + output[16..].first_chunk_mut::<16>().unwrap(), + ); + u8x32::simd_from(token, output) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn count_ones_u8x32(self, a: u8x32) -> u8x32 { crate::kernel!( #[inline(always)] @@ -8788,6 +9606,56 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn compress_i16x16(self, values: i16x16, mask: mask16x16) -> i16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i16x16( + self, + values: i16x16, + mask: mask16x16, + merge: i16x16, + ) -> i16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i16x16(self, values: i16x16, mask: mask16x16) -> i16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i16x16( + self, + values: i16x16, + mask: mask16x16, + merge: i16x16, + ) -> i16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_i16x16(self, a: i16x16) -> i16x16 { crate::kernel!( #[inline(always)] @@ -9281,6 +10149,56 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn compress_u16x16(self, values: u16x16, mask: mask16x16) -> u16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u16x16( + self, + values: u16x16, + mask: mask16x16, + merge: u16x16, + ) -> u16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u16x16(self, values: u16x16, mask: mask16x16) -> u16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u16x16( + self, + values: u16x16, + mask: mask16x16, + merge: u16x16, + ) -> u16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_u16x16(self, a: u16x16) -> u16x16 { crate::kernel!( #[inline(always)] @@ -10032,6 +10950,90 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn compress_i32x8(self, values: i32x8, mask: mask32x8) -> i32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: i32x8, mask: mask32x8) -> i32x8 { + let bits = token.to_bitmask_mask32x8(mask) as usize & 255; + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::COMPRESS_32[bits], + ); + let reordered = _mm256_permutevar8x32_epi32(values.into(), control); + let result = _mm256_andnot_si256(_mm256_srai_epi32::<31>(control), reordered); + result.simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_i32x8( + self, + values: i32x8, + mask: mask32x8, + merge: i32x8, + ) -> i32x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: i32x8, + mask: mask32x8, + merge: i32x8, + ) -> i32x8 { + let bits = token.to_bitmask_mask32x8(mask) as usize & 255; + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::COMPRESS_32[bits], + ); + let reordered = _mm256_permutevar8x32_epi32(values.into(), control); + let result = _mm256_blendv_epi8(reordered, merge.into(), control); + result.simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_i32x8(self, values: i32x8, mask: mask32x8) -> i32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: i32x8, mask: mask32x8) -> i32x8 { + let bits = token.to_bitmask_mask32x8(mask) as usize & 255; + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::EXPAND_32[bits], + ); + let reordered = _mm256_permutevar8x32_epi32(values.into(), control); + let result = _mm256_and_si256(reordered, mask.into()); + result.simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_i32x8( + self, + values: i32x8, + mask: mask32x8, + merge: i32x8, + ) -> i32x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: i32x8, + mask: mask32x8, + merge: i32x8, + ) -> i32x8 { + let bits = token.to_bitmask_mask32x8(mask) as usize & 255; + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::EXPAND_32[bits], + ); + let reordered = _mm256_permutevar8x32_epi32(values.into(), control); + let result = _mm256_blendv_epi8(merge.into(), reordered, mask.into()); + result.simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn count_ones_i32x8(self, a: i32x8) -> i32x8 { crate::kernel!( #[inline(always)] @@ -10517,6 +11519,90 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn compress_u32x8(self, values: u32x8, mask: mask32x8) -> u32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: u32x8, mask: mask32x8) -> u32x8 { + let bits = token.to_bitmask_mask32x8(mask) as usize & 255; + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::COMPRESS_32[bits], + ); + let reordered = _mm256_permutevar8x32_epi32(values.into(), control); + let result = _mm256_andnot_si256(_mm256_srai_epi32::<31>(control), reordered); + result.simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_u32x8( + self, + values: u32x8, + mask: mask32x8, + merge: u32x8, + ) -> u32x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: u32x8, + mask: mask32x8, + merge: u32x8, + ) -> u32x8 { + let bits = token.to_bitmask_mask32x8(mask) as usize & 255; + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::COMPRESS_32[bits], + ); + let reordered = _mm256_permutevar8x32_epi32(values.into(), control); + let result = _mm256_blendv_epi8(reordered, merge.into(), control); + result.simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_u32x8(self, values: u32x8, mask: mask32x8) -> u32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: u32x8, mask: mask32x8) -> u32x8 { + let bits = token.to_bitmask_mask32x8(mask) as usize & 255; + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::EXPAND_32[bits], + ); + let reordered = _mm256_permutevar8x32_epi32(values.into(), control); + let result = _mm256_and_si256(reordered, mask.into()); + result.simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_u32x8( + self, + values: u32x8, + mask: mask32x8, + merge: u32x8, + ) -> u32x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: u32x8, + mask: mask32x8, + merge: u32x8, + ) -> u32x8 { + let bits = token.to_bitmask_mask32x8(mask) as usize & 255; + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::EXPAND_32[bits], + ); + let reordered = _mm256_permutevar8x32_epi32(values.into(), control); + let result = _mm256_blendv_epi8(merge.into(), reordered, mask.into()); + result.simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn count_ones_u32x8(self, a: u32x8) -> u32x8 { crate::kernel!( #[inline(always)] @@ -11246,6 +12332,96 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn compress_f64x4(self, values: f64x4, mask: mask64x4) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: f64x4, mask: mask64x4) -> f64x4 { + let bits = token.to_bitmask_mask64x4(mask) as usize & 15; + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::COMPRESS_64[bits], + ); + let reordered = + _mm256_permutevar8x32_epi32(_mm256_castpd_si256(values.into()), control); + let result = _mm256_andnot_si256(_mm256_srai_epi32::<31>(control), reordered); + _mm256_castsi256_pd(result).simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_f64x4( + self, + values: f64x4, + mask: mask64x4, + merge: f64x4, + ) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: f64x4, + mask: mask64x4, + merge: f64x4, + ) -> f64x4 { + let bits = token.to_bitmask_mask64x4(mask) as usize & 15; + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::COMPRESS_64[bits], + ); + let reordered = + _mm256_permutevar8x32_epi32(_mm256_castpd_si256(values.into()), control); + let result = + _mm256_blendv_epi8(reordered, _mm256_castpd_si256(merge.into()), control); + _mm256_castsi256_pd(result).simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_f64x4(self, values: f64x4, mask: mask64x4) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: f64x4, mask: mask64x4) -> f64x4 { + let bits = token.to_bitmask_mask64x4(mask) as usize & 15; + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::EXPAND_64[bits], + ); + let reordered = + _mm256_permutevar8x32_epi32(_mm256_castpd_si256(values.into()), control); + let result = _mm256_and_si256(reordered, mask.into()); + _mm256_castsi256_pd(result).simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_f64x4( + self, + values: f64x4, + mask: mask64x4, + merge: f64x4, + ) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: f64x4, + mask: mask64x4, + merge: f64x4, + ) -> f64x4 { + let bits = token.to_bitmask_mask64x4(mask) as usize & 15; + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::EXPAND_64[bits], + ); + let reordered = + _mm256_permutevar8x32_epi32(_mm256_castpd_si256(values.into()), control); + let result = + _mm256_blendv_epi8(_mm256_castpd_si256(merge.into()), reordered, mask.into()); + _mm256_castsi256_pd(result).simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn neg_f64x4(self, a: f64x4) -> f64x4 { crate::kernel!( #[inline(always)] @@ -11841,6 +13017,90 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn compress_i64x4(self, values: i64x4, mask: mask64x4) -> i64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: i64x4, mask: mask64x4) -> i64x4 { + let bits = token.to_bitmask_mask64x4(mask) as usize & 15; + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::COMPRESS_64[bits], + ); + let reordered = _mm256_permutevar8x32_epi32(values.into(), control); + let result = _mm256_andnot_si256(_mm256_srai_epi32::<31>(control), reordered); + result.simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_i64x4( + self, + values: i64x4, + mask: mask64x4, + merge: i64x4, + ) -> i64x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: i64x4, + mask: mask64x4, + merge: i64x4, + ) -> i64x4 { + let bits = token.to_bitmask_mask64x4(mask) as usize & 15; + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::COMPRESS_64[bits], + ); + let reordered = _mm256_permutevar8x32_epi32(values.into(), control); + let result = _mm256_blendv_epi8(reordered, merge.into(), control); + result.simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_i64x4(self, values: i64x4, mask: mask64x4) -> i64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: i64x4, mask: mask64x4) -> i64x4 { + let bits = token.to_bitmask_mask64x4(mask) as usize & 15; + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::EXPAND_64[bits], + ); + let reordered = _mm256_permutevar8x32_epi32(values.into(), control); + let result = _mm256_and_si256(reordered, mask.into()); + result.simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_i64x4( + self, + values: i64x4, + mask: mask64x4, + merge: i64x4, + ) -> i64x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: i64x4, + mask: mask64x4, + merge: i64x4, + ) -> i64x4 { + let bits = token.to_bitmask_mask64x4(mask) as usize & 15; + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::EXPAND_64[bits], + ); + let reordered = _mm256_permutevar8x32_epi32(values.into(), control); + let result = _mm256_blendv_epi8(merge.into(), reordered, mask.into()); + result.simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn count_ones_i64x4(self, a: i64x4) -> i64x4 { crate::kernel!( #[inline(always)] @@ -12344,6 +13604,90 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn compress_u64x4(self, values: u64x4, mask: mask64x4) -> u64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: u64x4, mask: mask64x4) -> u64x4 { + let bits = token.to_bitmask_mask64x4(mask) as usize & 15; + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::COMPRESS_64[bits], + ); + let reordered = _mm256_permutevar8x32_epi32(values.into(), control); + let result = _mm256_andnot_si256(_mm256_srai_epi32::<31>(control), reordered); + result.simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_u64x4( + self, + values: u64x4, + mask: mask64x4, + merge: u64x4, + ) -> u64x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: u64x4, + mask: mask64x4, + merge: u64x4, + ) -> u64x4 { + let bits = token.to_bitmask_mask64x4(mask) as usize & 15; + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::COMPRESS_64[bits], + ); + let reordered = _mm256_permutevar8x32_epi32(values.into(), control); + let result = _mm256_blendv_epi8(reordered, merge.into(), control); + result.simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_u64x4(self, values: u64x4, mask: mask64x4) -> u64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: u64x4, mask: mask64x4) -> u64x4 { + let bits = token.to_bitmask_mask64x4(mask) as usize & 15; + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::EXPAND_64[bits], + ); + let reordered = _mm256_permutevar8x32_epi32(values.into(), control); + let result = _mm256_and_si256(reordered, mask.into()); + result.simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_u64x4( + self, + values: u64x4, + mask: mask64x4, + merge: u64x4, + ) -> u64x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: u64x4, + mask: mask64x4, + merge: u64x4, + ) -> u64x4 { + let bits = token.to_bitmask_mask64x4(mask) as usize & 15; + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::EXPAND_64[bits], + ); + let reordered = _mm256_permutevar8x32_epi32(values.into(), control); + let result = _mm256_blendv_epi8(merge.into(), reordered, mask.into()); + result.simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn count_ones_u64x4(self, a: u64x4) -> u64x4 { crate::kernel!( #[inline(always)] @@ -13029,19 +14373,160 @@ impl Simd for Avx2 { }) } #[inline(always)] - fn simd_ne_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { - let (a0, a1) = self.split_f32x16(a); - let (b0, b1) = self.split_f32x16(b); - self.combine_mask32x8(self.simd_ne_f32x8(a0, b0), self.simd_ne_f32x8(a1, b1)) + fn compress_f32x16(self, values: f32x16, mask: mask32x16) -> f32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: f32x16, mask: mask32x16) -> f32x16 { + let (low_mask, high_mask) = token.split_mask32x16(mask); + let low_count = low_mask.to_bitmask().count_ones() as usize; + let (low, high) = values.split(); + let bits = mask.to_bitmask(); + if bits == 0 { + return f32x16::splat(token, 0.0); + } + if bits == 65535 { + return values; + } + let low = low.compress(low_mask); + let high = high.compress(high_mask); + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::SHIFTS[low_count], + ); + let low_bits = crate::transmute::checked_transmute_copy(&low.val); + let high_bits = crate::transmute::checked_transmute_copy(&high.val); + let shifted = _mm256_permutevar8x32_epi32(high_bits, control); + let result_low = _mm256_blendv_epi8(shifted, low_bits, control); + let result_high = _mm256_and_si256(shifted, _mm256_srai_epi32::<31>(control)); + f32x16 { + val: crate::transmute::checked_transmute_copy(&[result_low, result_high]), + simd: token, + } + } + ); + kernel(self, values, mask) } #[inline(always)] - fn is_nan_f32x16(self, a: f32x16) -> mask32x16 { - a.simd_ne(a) + fn compress_merge_f32x16( + self, + values: f32x16, + mask: mask32x16, + merge: f32x16, + ) -> f32x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: f32x16, + mask: mask32x16, + merge: f32x16, + ) -> f32x16 { + let (low_mask, high_mask) = token.split_mask32x16(mask); + let low_count = low_mask.to_bitmask().count_ones() as usize; + let (low, high) = values.split(); + let bits = mask.to_bitmask(); + if bits == 0 { + return merge; + } + if bits == 65535 { + return values; + } + let low = low.compress(low_mask); + let high = high.compress(high_mask); + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::SHIFTS[low_count], + ); + let low_bits = crate::transmute::checked_transmute_copy(&low.val); + let high_bits = crate::transmute::checked_transmute_copy(&high.val); + let shifted = _mm256_permutevar8x32_epi32(high_bits, control); + let result_low = _mm256_blendv_epi8(shifted, low_bits, control); + let result_high = _mm256_and_si256(shifted, _mm256_srai_epi32::<31>(control)); + let compressed = f32x16 { + val: crate::transmute::checked_transmute_copy(&[result_low, result_high]), + simd: token, + }; + let count = low_count + high_mask.to_bitmask().count_ones() as usize; + let prefix = mask32x16::from_bitmask(token, (1u64 << count) - 1); + token.select_f32x16(prefix, compressed, merge) + } + ); + kernel(self, values, mask, merge) } #[inline(always)] - fn is_infinite_f32x16(self, a: f32x16) -> mask32x16 { - a.abs().simd_eq(f32::INFINITY) - } + fn expand_f32x16(self, values: f32x16, mask: mask32x16) -> f32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: f32x16, mask: mask32x16) -> f32x16 { + let (low_mask, high_mask) = token.split_mask32x16(mask); + let low_count = low_mask.to_bitmask().count_ones() as usize; + let (low, high) = values.split(); + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::SHIFTS[8 - (low_count)], + ); + let low_bits = crate::transmute::checked_transmute_copy(&low.val); + let high_bits = crate::transmute::checked_transmute_copy(&high.val); + let from_low = _mm256_permutevar8x32_epi32(low_bits, control); + let from_high = _mm256_permutevar8x32_epi32(high_bits, control); + let packed = _mm256_blendv_epi8(from_high, from_low, control); + let high = f32x8 { + val: crate::transmute::checked_transmute_copy(&packed), + simd: token, + }; + low.expand(low_mask).combine(high.expand(high_mask)) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_f32x16( + self, + values: f32x16, + mask: mask32x16, + merge: f32x16, + ) -> f32x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: f32x16, + mask: mask32x16, + merge: f32x16, + ) -> f32x16 { + let (low_mask, high_mask) = token.split_mask32x16(mask); + let low_count = low_mask.to_bitmask().count_ones() as usize; + let (low, high) = values.split(); + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::SHIFTS[8 - (low_count)], + ); + let low_bits = crate::transmute::checked_transmute_copy(&low.val); + let high_bits = crate::transmute::checked_transmute_copy(&high.val); + let from_low = _mm256_permutevar8x32_epi32(low_bits, control); + let from_high = _mm256_permutevar8x32_epi32(high_bits, control); + let packed = _mm256_blendv_epi8(from_high, from_low, control); + let high = f32x8 { + val: crate::transmute::checked_transmute_copy(&packed), + simd: token, + }; + let (low_merge, high_merge) = merge.split(); + low.expand_merge(low_mask, low_merge) + .combine(high.expand_merge(high_mask, high_merge)) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn simd_ne_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { + let (a0, a1) = self.split_f32x16(a); + let (b0, b1) = self.split_f32x16(b); + self.combine_mask32x8(self.simd_ne_f32x8(a0, b0), self.simd_ne_f32x8(a1, b1)) + } + #[inline(always)] + fn is_nan_f32x16(self, a: f32x16) -> mask32x16 { + a.simd_ne(a) + } + #[inline(always)] + fn is_infinite_f32x16(self, a: f32x16) -> mask32x16 { + a.abs().simd_eq(f32::INFINITY) + } #[inline(always)] fn is_finite_f32x16(self, a: f32x16) -> mask32x16 { a.abs().simd_lt(f32::INFINITY) @@ -13096,6 +14581,56 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn compress_i8x64(self, values: i8x64, mask: mask8x64) -> i8x64 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i8x64( + self, + values: i8x64, + mask: mask8x64, + merge: i8x64, + ) -> i8x64 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i8x64(self, values: i8x64, mask: mask8x64) -> i8x64 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i8x64( + self, + values: i8x64, + mask: mask8x64, + merge: i8x64, + ) -> i8x64 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_i8x64(self, a: i8x64) -> (i8x32, i8x32) { ( i8x32 { @@ -13282,6 +14817,461 @@ impl Simd for Avx2 { kernel(self, a, b, indices) } #[inline(always)] + fn compress_u8x64(self, values: u8x64, mask: mask8x64) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: u8x64, mask: mask8x64) -> u8x64 { + let mask_bits = token.to_bitmask_mask8x64(mask); + if mask_bits == 0 || mask_bits == u64::MAX { + return if mask_bits == 0 { + u8x64::splat(token, 0) + } else { + values + }; + } + if mask_bits.is_power_of_two() { + let selected = values.as_array()[mask_bits.trailing_zeros() as usize]; + return { + let first = _mm256_set_epi64x(0, 0, 0, i64::from(selected)); + u8x64 { + val: crate::support::Aligned512([first, _mm256_setzero_si256()]), + simd: token, + } + }; + } + let block_bits_0 = (mask_bits & 0xffff) as usize; + let low_mask_0 = (block_bits_0) & 0xff; + let high_mask_0 = (block_bits_0) >> 8; + let low_control_0 = crate::support::COMPRESS_8_CONTROLS[low_mask_0]; + let high_control_0 = crate::support::COMPRESS_8_CONTROLS[high_mask_0]; + let high_control_0 = high_control_0 | 0x0808_0808_0808_0808; + let control_0 = + _mm_set_epi64x(high_control_0.cast_signed(), low_control_0.cast_signed()); + let low_count_0 = low_mask_0.count_ones() as usize; + let block_bits_1 = (mask_bits >> 16 & 0xffff) as usize; + let low_mask_1 = (block_bits_1) & 0xff; + let high_mask_1 = (block_bits_1) >> 8; + let low_control_1 = crate::support::COMPRESS_8_CONTROLS[low_mask_1]; + let high_control_1 = crate::support::COMPRESS_8_CONTROLS[high_mask_1]; + let high_control_1 = high_control_1 | 0x0808_0808_0808_0808; + let control_1 = + _mm_set_epi64x(high_control_1.cast_signed(), low_control_1.cast_signed()); + let low_count_1 = low_mask_1.count_ones() as usize; + let control = _mm256_set_m128i(control_1, control_0); + let compacted = _mm256_shuffle_epi8(values.val.0[0], control); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count_0]; + let splice_0 = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count_1]; + let splice_1 = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let splice = _mm256_set_m128i(splice_1, splice_0); + let compressed = _mm256_shuffle_epi8(compacted, splice); + let splice = crate::transmute::checked_transmute_copy( + &crate::support::COMPACT_16_SPLICE_CONTROLS[block_bits_0.count_ones() as usize], + ); + let compressed_low = _mm256_permute2x128_si256::<0x80>(compressed, compressed); + let compressed_high = _mm256_permute2x128_si256::<0x11>(compressed, compressed); + let compressed_0 = + _mm256_or_si256(compressed_low, _mm256_shuffle_epi8(compressed_high, splice)); + let block_bits_2 = (mask_bits >> 32 & 0xffff) as usize; + let low_mask_2 = (block_bits_2) & 0xff; + let high_mask_2 = (block_bits_2) >> 8; + let low_control_2 = crate::support::COMPRESS_8_CONTROLS[low_mask_2]; + let high_control_2 = crate::support::COMPRESS_8_CONTROLS[high_mask_2]; + let high_control_2 = high_control_2 | 0x0808_0808_0808_0808; + let control_2 = + _mm_set_epi64x(high_control_2.cast_signed(), low_control_2.cast_signed()); + let low_count_2 = low_mask_2.count_ones() as usize; + let block_bits_3 = (mask_bits >> 48 & 0xffff) as usize; + let low_mask_3 = (block_bits_3) & 0xff; + let high_mask_3 = (block_bits_3) >> 8; + let low_control_3 = crate::support::COMPRESS_8_CONTROLS[low_mask_3]; + let high_control_3 = crate::support::COMPRESS_8_CONTROLS[high_mask_3]; + let high_control_3 = high_control_3 | 0x0808_0808_0808_0808; + let control_3 = + _mm_set_epi64x(high_control_3.cast_signed(), low_control_3.cast_signed()); + let low_count_3 = low_mask_3.count_ones() as usize; + let control = _mm256_set_m128i(control_3, control_2); + let compacted = _mm256_shuffle_epi8(values.val.0[1], control); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count_2]; + let splice_2 = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count_3]; + let splice_3 = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let splice = _mm256_set_m128i(splice_3, splice_2); + let compressed = _mm256_shuffle_epi8(compacted, splice); + let splice = crate::transmute::checked_transmute_copy( + &crate::support::COMPACT_16_SPLICE_CONTROLS[block_bits_2.count_ones() as usize], + ); + let compressed_low = _mm256_permute2x128_si256::<0x80>(compressed, compressed); + let compressed_high = _mm256_permute2x128_si256::<0x11>(compressed, compressed); + let compressed_1 = + _mm256_or_si256(compressed_low, _mm256_shuffle_epi8(compressed_high, splice)); + let first_count = (block_bits_0.count_ones() + block_bits_1.count_ones()) as usize; + let controls = &crate::support::COMPACT_32_STITCH_CONTROLS[first_count]; + let left_same = crate::transmute::checked_transmute_copy(&controls.left_same); + let left_cross = crate::transmute::checked_transmute_copy(&controls.left_cross); + let right_same = crate::transmute::checked_transmute_copy(&controls.right_same); + let right_cross = crate::transmute::checked_transmute_copy(&controls.right_cross); + let low_to_high = _mm256_permute2x128_si256::<0x08>(compressed_1, compressed_1); + let shifted_left = _mm256_or_si256( + _mm256_shuffle_epi8(compressed_1, left_same), + _mm256_shuffle_epi8(low_to_high, left_cross), + ); + let compressed_0 = _mm256_or_si256(compressed_0, shifted_left); + let high_to_low = _mm256_permute2x128_si256::<0x81>(compressed_1, compressed_1); + let compressed_1 = _mm256_or_si256( + _mm256_shuffle_epi8(compressed_1, right_same), + _mm256_shuffle_epi8(high_to_low, right_cross), + ); + let result_0 = compressed_0; + let result_1 = compressed_1; + u8x64 { + val: crate::support::Aligned512([result_0, result_1]), + simd: token, + } + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_u8x64( + self, + values: u8x64, + mask: mask8x64, + merge: u8x64, + ) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: u8x64, + mask: mask8x64, + merge: u8x64, + ) -> u8x64 { + let mask_bits = token.to_bitmask_mask8x64(mask); + if mask_bits == 0 || mask_bits == u64::MAX { + return if mask_bits == 0 { merge } else { values }; + } + if mask_bits.is_power_of_two() { + let selected = values.as_array()[mask_bits.trailing_zeros() as usize]; + return { + let low = _mm_insert_epi8::<0>( + _mm256_castsi256_si128(merge.val.0[0]), + i32::from(selected), + ); + let first = _mm256_inserti128_si256::<0>(merge.val.0[0], low); + u8x64 { + val: crate::support::Aligned512([first, merge.val.0[1]]), + simd: token, + } + }; + } + let block_bits_0 = (mask_bits & 0xffff) as usize; + let low_mask_0 = (block_bits_0) & 0xff; + let high_mask_0 = (block_bits_0) >> 8; + let low_control_0 = crate::support::COMPRESS_8_CONTROLS[low_mask_0]; + let high_control_0 = crate::support::COMPRESS_8_CONTROLS[high_mask_0]; + let high_control_0 = high_control_0 | 0x0808_0808_0808_0808; + let control_0 = + _mm_set_epi64x(high_control_0.cast_signed(), low_control_0.cast_signed()); + let low_count_0 = low_mask_0.count_ones() as usize; + let block_bits_1 = (mask_bits >> 16 & 0xffff) as usize; + let low_mask_1 = (block_bits_1) & 0xff; + let high_mask_1 = (block_bits_1) >> 8; + let low_control_1 = crate::support::COMPRESS_8_CONTROLS[low_mask_1]; + let high_control_1 = crate::support::COMPRESS_8_CONTROLS[high_mask_1]; + let high_control_1 = high_control_1 | 0x0808_0808_0808_0808; + let control_1 = + _mm_set_epi64x(high_control_1.cast_signed(), low_control_1.cast_signed()); + let low_count_1 = low_mask_1.count_ones() as usize; + let control = _mm256_set_m128i(control_1, control_0); + let compacted = _mm256_shuffle_epi8(values.val.0[0], control); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count_0]; + let splice_0 = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count_1]; + let splice_1 = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let splice = _mm256_set_m128i(splice_1, splice_0); + let compressed = _mm256_shuffle_epi8(compacted, splice); + let splice = crate::transmute::checked_transmute_copy( + &crate::support::COMPACT_16_SPLICE_CONTROLS[block_bits_0.count_ones() as usize], + ); + let compressed_low = _mm256_permute2x128_si256::<0x80>(compressed, compressed); + let compressed_high = _mm256_permute2x128_si256::<0x11>(compressed, compressed); + let compressed_0 = + _mm256_or_si256(compressed_low, _mm256_shuffle_epi8(compressed_high, splice)); + let block_bits_2 = (mask_bits >> 32 & 0xffff) as usize; + let low_mask_2 = (block_bits_2) & 0xff; + let high_mask_2 = (block_bits_2) >> 8; + let low_control_2 = crate::support::COMPRESS_8_CONTROLS[low_mask_2]; + let high_control_2 = crate::support::COMPRESS_8_CONTROLS[high_mask_2]; + let high_control_2 = high_control_2 | 0x0808_0808_0808_0808; + let control_2 = + _mm_set_epi64x(high_control_2.cast_signed(), low_control_2.cast_signed()); + let low_count_2 = low_mask_2.count_ones() as usize; + let block_bits_3 = (mask_bits >> 48 & 0xffff) as usize; + let low_mask_3 = (block_bits_3) & 0xff; + let high_mask_3 = (block_bits_3) >> 8; + let low_control_3 = crate::support::COMPRESS_8_CONTROLS[low_mask_3]; + let high_control_3 = crate::support::COMPRESS_8_CONTROLS[high_mask_3]; + let high_control_3 = high_control_3 | 0x0808_0808_0808_0808; + let control_3 = + _mm_set_epi64x(high_control_3.cast_signed(), low_control_3.cast_signed()); + let low_count_3 = low_mask_3.count_ones() as usize; + let control = _mm256_set_m128i(control_3, control_2); + let compacted = _mm256_shuffle_epi8(values.val.0[1], control); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count_2]; + let splice_2 = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count_3]; + let splice_3 = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let splice = _mm256_set_m128i(splice_3, splice_2); + let compressed = _mm256_shuffle_epi8(compacted, splice); + let splice = crate::transmute::checked_transmute_copy( + &crate::support::COMPACT_16_SPLICE_CONTROLS[block_bits_2.count_ones() as usize], + ); + let compressed_low = _mm256_permute2x128_si256::<0x80>(compressed, compressed); + let compressed_high = _mm256_permute2x128_si256::<0x11>(compressed, compressed); + let compressed_1 = + _mm256_or_si256(compressed_low, _mm256_shuffle_epi8(compressed_high, splice)); + let first_count = (block_bits_0.count_ones() + block_bits_1.count_ones()) as usize; + let controls = &crate::support::COMPACT_32_STITCH_CONTROLS[first_count]; + let left_same = crate::transmute::checked_transmute_copy(&controls.left_same); + let left_cross = crate::transmute::checked_transmute_copy(&controls.left_cross); + let right_same = crate::transmute::checked_transmute_copy(&controls.right_same); + let right_cross = crate::transmute::checked_transmute_copy(&controls.right_cross); + let low_to_high = _mm256_permute2x128_si256::<0x08>(compressed_1, compressed_1); + let shifted_left = _mm256_or_si256( + _mm256_shuffle_epi8(compressed_1, left_same), + _mm256_shuffle_epi8(low_to_high, left_cross), + ); + let compressed_0 = _mm256_or_si256(compressed_0, shifted_left); + let high_to_low = _mm256_permute2x128_si256::<0x81>(compressed_1, compressed_1); + let compressed_1 = _mm256_or_si256( + _mm256_shuffle_epi8(compressed_1, right_same), + _mm256_shuffle_epi8(high_to_low, right_cross), + ); + let output_lane = mask_bits.count_ones() as usize; + let prefix = crate::transmute::checked_transmute_copy( + &crate::support::COMPACT_PREFIX_MASKS[output_lane.min(32)], + ); + let result_0 = _mm256_blendv_epi8(merge.val.0[0], compressed_0, prefix); + let prefix = crate::transmute::checked_transmute_copy( + &crate::support::COMPACT_PREFIX_MASKS[output_lane.saturating_sub(32).min(32)], + ); + let result_1 = _mm256_blendv_epi8(merge.val.0[1], compressed_1, prefix); + u8x64 { + val: crate::support::Aligned512([result_0, result_1]), + simd: token, + } + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_u8x64(self, values: u8x64, mask: mask8x64) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: u8x64, mask: mask8x64) -> u8x64 { + let input = <[u8; 64]>::from(values); + let mut output = [0u8; 64]; + let mut input_lane = 0; + let block_bits = _mm_movemask_epi8(_mm256_castsi256_si128(mask.val.0[0])) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = high_control + high_base; + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = crate::transmute::checked_transmute_copy( + input[input_lane..].first_chunk::<16>().unwrap(), + ); + let expanded = _mm_shuffle_epi8(packed, control); + let result = expanded; + crate::transmute::checked_transmute_store( + result, + output[0..].first_chunk_mut::<16>().unwrap(), + ); + input_lane += block_bits.count_ones() as usize; + let block_bits = + _mm_movemask_epi8(_mm256_extracti128_si256::<1>(mask.val.0[0])) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = high_control + high_base; + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = crate::transmute::checked_transmute_copy( + input[input_lane..].first_chunk::<16>().unwrap(), + ); + let expanded = _mm_shuffle_epi8(packed, control); + let result = expanded; + crate::transmute::checked_transmute_store( + result, + output[16..].first_chunk_mut::<16>().unwrap(), + ); + input_lane += block_bits.count_ones() as usize; + let block_bits = _mm_movemask_epi8(_mm256_castsi256_si128(mask.val.0[1])) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = high_control + high_base; + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = crate::transmute::checked_transmute_copy( + input[input_lane..].first_chunk::<16>().unwrap(), + ); + let expanded = _mm_shuffle_epi8(packed, control); + let result = expanded; + crate::transmute::checked_transmute_store( + result, + output[32..].first_chunk_mut::<16>().unwrap(), + ); + input_lane += block_bits.count_ones() as usize; + let block_bits = + _mm_movemask_epi8(_mm256_extracti128_si256::<1>(mask.val.0[1])) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = high_control + high_base; + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = crate::transmute::checked_transmute_copy( + input[input_lane..].first_chunk::<16>().unwrap(), + ); + let expanded = _mm_shuffle_epi8(packed, control); + let result = expanded; + crate::transmute::checked_transmute_store( + result, + output[48..].first_chunk_mut::<16>().unwrap(), + ); + u8x64::simd_from(token, output) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_u8x64( + self, + values: u8x64, + mask: mask8x64, + merge: u8x64, + ) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: u8x64, + mask: mask8x64, + merge: u8x64, + ) -> u8x64 { + let input = <[u8; 64]>::from(values); + let mut output = [0u8; 64]; + let mut input_lane = 0; + let block_bits = _mm_movemask_epi8(_mm256_castsi256_si128(mask.val.0[0])) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = high_control + high_base; + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = crate::transmute::checked_transmute_copy( + input[input_lane..].first_chunk::<16>().unwrap(), + ); + let expanded = _mm_shuffle_epi8(packed, control); + let result = _mm_blendv_epi8( + _mm256_castsi256_si128(merge.val.0[0]), + expanded, + _mm256_castsi256_si128(mask.val.0[0]), + ); + crate::transmute::checked_transmute_store( + result, + output[0..].first_chunk_mut::<16>().unwrap(), + ); + input_lane += block_bits.count_ones() as usize; + let block_bits = + _mm_movemask_epi8(_mm256_extracti128_si256::<1>(mask.val.0[0])) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = high_control + high_base; + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = crate::transmute::checked_transmute_copy( + input[input_lane..].first_chunk::<16>().unwrap(), + ); + let expanded = _mm_shuffle_epi8(packed, control); + let result = _mm_blendv_epi8( + _mm256_extracti128_si256::<1>(merge.val.0[0]), + expanded, + _mm256_extracti128_si256::<1>(mask.val.0[0]), + ); + crate::transmute::checked_transmute_store( + result, + output[16..].first_chunk_mut::<16>().unwrap(), + ); + input_lane += block_bits.count_ones() as usize; + let block_bits = _mm_movemask_epi8(_mm256_castsi256_si128(mask.val.0[1])) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = high_control + high_base; + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = crate::transmute::checked_transmute_copy( + input[input_lane..].first_chunk::<16>().unwrap(), + ); + let expanded = _mm_shuffle_epi8(packed, control); + let result = _mm_blendv_epi8( + _mm256_castsi256_si128(merge.val.0[1]), + expanded, + _mm256_castsi256_si128(mask.val.0[1]), + ); + crate::transmute::checked_transmute_store( + result, + output[32..].first_chunk_mut::<16>().unwrap(), + ); + input_lane += block_bits.count_ones() as usize; + let block_bits = + _mm_movemask_epi8(_mm256_extracti128_si256::<1>(mask.val.0[1])) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = high_control + high_base; + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = crate::transmute::checked_transmute_copy( + input[input_lane..].first_chunk::<16>().unwrap(), + ); + let expanded = _mm_shuffle_epi8(packed, control); + let result = _mm_blendv_epi8( + _mm256_extracti128_si256::<1>(merge.val.0[1]), + expanded, + _mm256_extracti128_si256::<1>(mask.val.0[1]), + ); + crate::transmute::checked_transmute_store( + result, + output[48..].first_chunk_mut::<16>().unwrap(), + ); + u8x64::simd_from(token, output) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn split_u8x64(self, a: u8x64) -> (u8x32, u8x32) { ( u8x32 { @@ -13406,6 +15396,56 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn compress_i16x32(self, values: i16x32, mask: mask16x32) -> i16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i16x32( + self, + values: i16x32, + mask: mask16x32, + merge: i16x32, + ) -> i16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i16x32(self, values: i16x32, mask: mask16x32) -> i16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i16x32( + self, + values: i16x32, + mask: mask16x32, + merge: i16x32, + ) -> i16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_i16x32(self, a: i16x32) -> (i16x16, i16x16) { ( i16x16 { @@ -13435,6 +15475,56 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn compress_u16x32(self, values: u16x32, mask: mask16x32) -> u16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u16x32( + self, + values: u16x32, + mask: mask16x32, + merge: u16x32, + ) -> u16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u16x32(self, values: u16x32, mask: mask16x32) -> u16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u16x32( + self, + values: u16x32, + mask: mask16x32, + merge: u16x32, + ) -> u16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_u16x32(self, a: u16x32) -> (u16x16, u16x16) { ( u16x16 { @@ -13534,6 +15624,147 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn compress_i32x16(self, values: i32x16, mask: mask32x16) -> i32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: i32x16, mask: mask32x16) -> i32x16 { + let (low_mask, high_mask) = token.split_mask32x16(mask); + let low_count = low_mask.to_bitmask().count_ones() as usize; + let (low, high) = values.split(); + let bits = mask.to_bitmask(); + if bits == 0 { + return i32x16::splat(token, 0); + } + if bits == 65535 { + return values; + } + let low = low.compress(low_mask); + let high = high.compress(high_mask); + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::SHIFTS[low_count], + ); + let low_bits = crate::transmute::checked_transmute_copy(&low.val); + let high_bits = crate::transmute::checked_transmute_copy(&high.val); + let shifted = _mm256_permutevar8x32_epi32(high_bits, control); + let result_low = _mm256_blendv_epi8(shifted, low_bits, control); + let result_high = _mm256_and_si256(shifted, _mm256_srai_epi32::<31>(control)); + i32x16 { + val: crate::transmute::checked_transmute_copy(&[result_low, result_high]), + simd: token, + } + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_i32x16( + self, + values: i32x16, + mask: mask32x16, + merge: i32x16, + ) -> i32x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: i32x16, + mask: mask32x16, + merge: i32x16, + ) -> i32x16 { + let (low_mask, high_mask) = token.split_mask32x16(mask); + let low_count = low_mask.to_bitmask().count_ones() as usize; + let (low, high) = values.split(); + let bits = mask.to_bitmask(); + if bits == 0 { + return merge; + } + if bits == 65535 { + return values; + } + let low = low.compress(low_mask); + let high = high.compress(high_mask); + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::SHIFTS[low_count], + ); + let low_bits = crate::transmute::checked_transmute_copy(&low.val); + let high_bits = crate::transmute::checked_transmute_copy(&high.val); + let shifted = _mm256_permutevar8x32_epi32(high_bits, control); + let result_low = _mm256_blendv_epi8(shifted, low_bits, control); + let result_high = _mm256_and_si256(shifted, _mm256_srai_epi32::<31>(control)); + let compressed = i32x16 { + val: crate::transmute::checked_transmute_copy(&[result_low, result_high]), + simd: token, + }; + let count = low_count + high_mask.to_bitmask().count_ones() as usize; + let prefix = mask32x16::from_bitmask(token, (1u64 << count) - 1); + token.select_i32x16(prefix, compressed, merge) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_i32x16(self, values: i32x16, mask: mask32x16) -> i32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: i32x16, mask: mask32x16) -> i32x16 { + let (low_mask, high_mask) = token.split_mask32x16(mask); + let low_count = low_mask.to_bitmask().count_ones() as usize; + let (low, high) = values.split(); + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::SHIFTS[8 - (low_count)], + ); + let low_bits = crate::transmute::checked_transmute_copy(&low.val); + let high_bits = crate::transmute::checked_transmute_copy(&high.val); + let from_low = _mm256_permutevar8x32_epi32(low_bits, control); + let from_high = _mm256_permutevar8x32_epi32(high_bits, control); + let packed = _mm256_blendv_epi8(from_high, from_low, control); + let high = i32x8 { + val: crate::transmute::checked_transmute_copy(&packed), + simd: token, + }; + low.expand(low_mask).combine(high.expand(high_mask)) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_i32x16( + self, + values: i32x16, + mask: mask32x16, + merge: i32x16, + ) -> i32x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: i32x16, + mask: mask32x16, + merge: i32x16, + ) -> i32x16 { + let (low_mask, high_mask) = token.split_mask32x16(mask); + let low_count = low_mask.to_bitmask().count_ones() as usize; + let (low, high) = values.split(); + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::SHIFTS[8 - (low_count)], + ); + let low_bits = crate::transmute::checked_transmute_copy(&low.val); + let high_bits = crate::transmute::checked_transmute_copy(&high.val); + let from_low = _mm256_permutevar8x32_epi32(low_bits, control); + let from_high = _mm256_permutevar8x32_epi32(high_bits, control); + let packed = _mm256_blendv_epi8(from_high, from_low, control); + let high = i32x8 { + val: crate::transmute::checked_transmute_copy(&packed), + simd: token, + }; + let (low_merge, high_merge) = merge.split(); + low.expand_merge(low_mask, low_merge) + .combine(high.expand_merge(high_mask, high_merge)) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn split_i32x16(self, a: i32x16) -> (i32x8, i32x8) { ( i32x8 { @@ -13563,6 +15794,147 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn compress_u32x16(self, values: u32x16, mask: mask32x16) -> u32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: u32x16, mask: mask32x16) -> u32x16 { + let (low_mask, high_mask) = token.split_mask32x16(mask); + let low_count = low_mask.to_bitmask().count_ones() as usize; + let (low, high) = values.split(); + let bits = mask.to_bitmask(); + if bits == 0 { + return u32x16::splat(token, 0); + } + if bits == 65535 { + return values; + } + let low = low.compress(low_mask); + let high = high.compress(high_mask); + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::SHIFTS[low_count], + ); + let low_bits = crate::transmute::checked_transmute_copy(&low.val); + let high_bits = crate::transmute::checked_transmute_copy(&high.val); + let shifted = _mm256_permutevar8x32_epi32(high_bits, control); + let result_low = _mm256_blendv_epi8(shifted, low_bits, control); + let result_high = _mm256_and_si256(shifted, _mm256_srai_epi32::<31>(control)); + u32x16 { + val: crate::transmute::checked_transmute_copy(&[result_low, result_high]), + simd: token, + } + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_u32x16( + self, + values: u32x16, + mask: mask32x16, + merge: u32x16, + ) -> u32x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: u32x16, + mask: mask32x16, + merge: u32x16, + ) -> u32x16 { + let (low_mask, high_mask) = token.split_mask32x16(mask); + let low_count = low_mask.to_bitmask().count_ones() as usize; + let (low, high) = values.split(); + let bits = mask.to_bitmask(); + if bits == 0 { + return merge; + } + if bits == 65535 { + return values; + } + let low = low.compress(low_mask); + let high = high.compress(high_mask); + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::SHIFTS[low_count], + ); + let low_bits = crate::transmute::checked_transmute_copy(&low.val); + let high_bits = crate::transmute::checked_transmute_copy(&high.val); + let shifted = _mm256_permutevar8x32_epi32(high_bits, control); + let result_low = _mm256_blendv_epi8(shifted, low_bits, control); + let result_high = _mm256_and_si256(shifted, _mm256_srai_epi32::<31>(control)); + let compressed = u32x16 { + val: crate::transmute::checked_transmute_copy(&[result_low, result_high]), + simd: token, + }; + let count = low_count + high_mask.to_bitmask().count_ones() as usize; + let prefix = mask32x16::from_bitmask(token, (1u64 << count) - 1); + token.select_u32x16(prefix, compressed, merge) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_u32x16(self, values: u32x16, mask: mask32x16) -> u32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: u32x16, mask: mask32x16) -> u32x16 { + let (low_mask, high_mask) = token.split_mask32x16(mask); + let low_count = low_mask.to_bitmask().count_ones() as usize; + let (low, high) = values.split(); + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::SHIFTS[8 - (low_count)], + ); + let low_bits = crate::transmute::checked_transmute_copy(&low.val); + let high_bits = crate::transmute::checked_transmute_copy(&high.val); + let from_low = _mm256_permutevar8x32_epi32(low_bits, control); + let from_high = _mm256_permutevar8x32_epi32(high_bits, control); + let packed = _mm256_blendv_epi8(from_high, from_low, control); + let high = u32x8 { + val: crate::transmute::checked_transmute_copy(&packed), + simd: token, + }; + low.expand(low_mask).combine(high.expand(high_mask)) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_u32x16( + self, + values: u32x16, + mask: mask32x16, + merge: u32x16, + ) -> u32x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: u32x16, + mask: mask32x16, + merge: u32x16, + ) -> u32x16 { + let (low_mask, high_mask) = token.split_mask32x16(mask); + let low_count = low_mask.to_bitmask().count_ones() as usize; + let (low, high) = values.split(); + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::SHIFTS[8 - (low_count)], + ); + let low_bits = crate::transmute::checked_transmute_copy(&low.val); + let high_bits = crate::transmute::checked_transmute_copy(&high.val); + let from_low = _mm256_permutevar8x32_epi32(low_bits, control); + let from_high = _mm256_permutevar8x32_epi32(high_bits, control); + let packed = _mm256_blendv_epi8(from_high, from_low, control); + let high = u32x8 { + val: crate::transmute::checked_transmute_copy(&packed), + simd: token, + }; + let (low_merge, high_merge) = merge.split(); + low.expand_merge(low_mask, low_merge) + .combine(high.expand_merge(high_mask, high_merge)) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn split_u32x16(self, a: u32x16) -> (u32x8, u32x8) { ( u32x8 { @@ -13673,6 +16045,147 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn compress_f64x8(self, values: f64x8, mask: mask64x8) -> f64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: f64x8, mask: mask64x8) -> f64x8 { + let (low_mask, high_mask) = token.split_mask64x8(mask); + let low_count = low_mask.to_bitmask().count_ones() as usize; + let (low, high) = values.split(); + let bits = mask.to_bitmask(); + if bits == 0 { + return f64x8::splat(token, 0.0); + } + if bits == 255 { + return values; + } + let low = low.compress(low_mask); + let high = high.compress(high_mask); + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::SHIFTS[low_count * 2], + ); + let low_bits = crate::transmute::checked_transmute_copy(&low.val); + let high_bits = crate::transmute::checked_transmute_copy(&high.val); + let shifted = _mm256_permutevar8x32_epi32(high_bits, control); + let result_low = _mm256_blendv_epi8(shifted, low_bits, control); + let result_high = _mm256_and_si256(shifted, _mm256_srai_epi32::<31>(control)); + f64x8 { + val: crate::transmute::checked_transmute_copy(&[result_low, result_high]), + simd: token, + } + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_f64x8( + self, + values: f64x8, + mask: mask64x8, + merge: f64x8, + ) -> f64x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: f64x8, + mask: mask64x8, + merge: f64x8, + ) -> f64x8 { + let (low_mask, high_mask) = token.split_mask64x8(mask); + let low_count = low_mask.to_bitmask().count_ones() as usize; + let (low, high) = values.split(); + let bits = mask.to_bitmask(); + if bits == 0 { + return merge; + } + if bits == 255 { + return values; + } + let low = low.compress(low_mask); + let high = high.compress(high_mask); + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::SHIFTS[low_count * 2], + ); + let low_bits = crate::transmute::checked_transmute_copy(&low.val); + let high_bits = crate::transmute::checked_transmute_copy(&high.val); + let shifted = _mm256_permutevar8x32_epi32(high_bits, control); + let result_low = _mm256_blendv_epi8(shifted, low_bits, control); + let result_high = _mm256_and_si256(shifted, _mm256_srai_epi32::<31>(control)); + let compressed = f64x8 { + val: crate::transmute::checked_transmute_copy(&[result_low, result_high]), + simd: token, + }; + let count = low_count + high_mask.to_bitmask().count_ones() as usize; + let prefix = mask64x8::from_bitmask(token, (1u64 << count) - 1); + token.select_f64x8(prefix, compressed, merge) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_f64x8(self, values: f64x8, mask: mask64x8) -> f64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: f64x8, mask: mask64x8) -> f64x8 { + let (low_mask, high_mask) = token.split_mask64x8(mask); + let low_count = low_mask.to_bitmask().count_ones() as usize; + let (low, high) = values.split(); + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::SHIFTS[8 - (low_count * 2)], + ); + let low_bits = crate::transmute::checked_transmute_copy(&low.val); + let high_bits = crate::transmute::checked_transmute_copy(&high.val); + let from_low = _mm256_permutevar8x32_epi32(low_bits, control); + let from_high = _mm256_permutevar8x32_epi32(high_bits, control); + let packed = _mm256_blendv_epi8(from_high, from_low, control); + let high = f64x4 { + val: crate::transmute::checked_transmute_copy(&packed), + simd: token, + }; + low.expand(low_mask).combine(high.expand(high_mask)) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_f64x8( + self, + values: f64x8, + mask: mask64x8, + merge: f64x8, + ) -> f64x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: f64x8, + mask: mask64x8, + merge: f64x8, + ) -> f64x8 { + let (low_mask, high_mask) = token.split_mask64x8(mask); + let low_count = low_mask.to_bitmask().count_ones() as usize; + let (low, high) = values.split(); + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::SHIFTS[8 - (low_count * 2)], + ); + let low_bits = crate::transmute::checked_transmute_copy(&low.val); + let high_bits = crate::transmute::checked_transmute_copy(&high.val); + let from_low = _mm256_permutevar8x32_epi32(low_bits, control); + let from_high = _mm256_permutevar8x32_epi32(high_bits, control); + let packed = _mm256_blendv_epi8(from_high, from_low, control); + let high = f64x4 { + val: crate::transmute::checked_transmute_copy(&packed), + simd: token, + }; + let (low_merge, high_merge) = merge.split(); + low.expand_merge(low_mask, low_merge) + .combine(high.expand_merge(high_mask, high_merge)) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn simd_ne_f64x8(self, a: f64x8, b: f64x8) -> mask64x8 { let (a0, a1) = self.split_f64x8(a); let (b0, b1) = self.split_f64x8(b); @@ -13740,6 +16253,147 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn compress_i64x8(self, values: i64x8, mask: mask64x8) -> i64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: i64x8, mask: mask64x8) -> i64x8 { + let (low_mask, high_mask) = token.split_mask64x8(mask); + let low_count = low_mask.to_bitmask().count_ones() as usize; + let (low, high) = values.split(); + let bits = mask.to_bitmask(); + if bits == 0 { + return i64x8::splat(token, 0); + } + if bits == 255 { + return values; + } + let low = low.compress(low_mask); + let high = high.compress(high_mask); + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::SHIFTS[low_count * 2], + ); + let low_bits = crate::transmute::checked_transmute_copy(&low.val); + let high_bits = crate::transmute::checked_transmute_copy(&high.val); + let shifted = _mm256_permutevar8x32_epi32(high_bits, control); + let result_low = _mm256_blendv_epi8(shifted, low_bits, control); + let result_high = _mm256_and_si256(shifted, _mm256_srai_epi32::<31>(control)); + i64x8 { + val: crate::transmute::checked_transmute_copy(&[result_low, result_high]), + simd: token, + } + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_i64x8( + self, + values: i64x8, + mask: mask64x8, + merge: i64x8, + ) -> i64x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: i64x8, + mask: mask64x8, + merge: i64x8, + ) -> i64x8 { + let (low_mask, high_mask) = token.split_mask64x8(mask); + let low_count = low_mask.to_bitmask().count_ones() as usize; + let (low, high) = values.split(); + let bits = mask.to_bitmask(); + if bits == 0 { + return merge; + } + if bits == 255 { + return values; + } + let low = low.compress(low_mask); + let high = high.compress(high_mask); + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::SHIFTS[low_count * 2], + ); + let low_bits = crate::transmute::checked_transmute_copy(&low.val); + let high_bits = crate::transmute::checked_transmute_copy(&high.val); + let shifted = _mm256_permutevar8x32_epi32(high_bits, control); + let result_low = _mm256_blendv_epi8(shifted, low_bits, control); + let result_high = _mm256_and_si256(shifted, _mm256_srai_epi32::<31>(control)); + let compressed = i64x8 { + val: crate::transmute::checked_transmute_copy(&[result_low, result_high]), + simd: token, + }; + let count = low_count + high_mask.to_bitmask().count_ones() as usize; + let prefix = mask64x8::from_bitmask(token, (1u64 << count) - 1); + token.select_i64x8(prefix, compressed, merge) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_i64x8(self, values: i64x8, mask: mask64x8) -> i64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: i64x8, mask: mask64x8) -> i64x8 { + let (low_mask, high_mask) = token.split_mask64x8(mask); + let low_count = low_mask.to_bitmask().count_ones() as usize; + let (low, high) = values.split(); + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::SHIFTS[8 - (low_count * 2)], + ); + let low_bits = crate::transmute::checked_transmute_copy(&low.val); + let high_bits = crate::transmute::checked_transmute_copy(&high.val); + let from_low = _mm256_permutevar8x32_epi32(low_bits, control); + let from_high = _mm256_permutevar8x32_epi32(high_bits, control); + let packed = _mm256_blendv_epi8(from_high, from_low, control); + let high = i64x4 { + val: crate::transmute::checked_transmute_copy(&packed), + simd: token, + }; + low.expand(low_mask).combine(high.expand(high_mask)) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_i64x8( + self, + values: i64x8, + mask: mask64x8, + merge: i64x8, + ) -> i64x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: i64x8, + mask: mask64x8, + merge: i64x8, + ) -> i64x8 { + let (low_mask, high_mask) = token.split_mask64x8(mask); + let low_count = low_mask.to_bitmask().count_ones() as usize; + let (low, high) = values.split(); + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::SHIFTS[8 - (low_count * 2)], + ); + let low_bits = crate::transmute::checked_transmute_copy(&low.val); + let high_bits = crate::transmute::checked_transmute_copy(&high.val); + let from_low = _mm256_permutevar8x32_epi32(low_bits, control); + let from_high = _mm256_permutevar8x32_epi32(high_bits, control); + let packed = _mm256_blendv_epi8(from_high, from_low, control); + let high = i64x4 { + val: crate::transmute::checked_transmute_copy(&packed), + simd: token, + }; + let (low_merge, high_merge) = merge.split(); + low.expand_merge(low_mask, low_merge) + .combine(high.expand_merge(high_mask, high_merge)) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn split_i64x8(self, a: i64x8) -> (i64x4, i64x4) { ( i64x4 { @@ -13769,6 +16423,147 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn compress_u64x8(self, values: u64x8, mask: mask64x8) -> u64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: u64x8, mask: mask64x8) -> u64x8 { + let (low_mask, high_mask) = token.split_mask64x8(mask); + let low_count = low_mask.to_bitmask().count_ones() as usize; + let (low, high) = values.split(); + let bits = mask.to_bitmask(); + if bits == 0 { + return u64x8::splat(token, 0); + } + if bits == 255 { + return values; + } + let low = low.compress(low_mask); + let high = high.compress(high_mask); + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::SHIFTS[low_count * 2], + ); + let low_bits = crate::transmute::checked_transmute_copy(&low.val); + let high_bits = crate::transmute::checked_transmute_copy(&high.val); + let shifted = _mm256_permutevar8x32_epi32(high_bits, control); + let result_low = _mm256_blendv_epi8(shifted, low_bits, control); + let result_high = _mm256_and_si256(shifted, _mm256_srai_epi32::<31>(control)); + u64x8 { + val: crate::transmute::checked_transmute_copy(&[result_low, result_high]), + simd: token, + } + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_u64x8( + self, + values: u64x8, + mask: mask64x8, + merge: u64x8, + ) -> u64x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: u64x8, + mask: mask64x8, + merge: u64x8, + ) -> u64x8 { + let (low_mask, high_mask) = token.split_mask64x8(mask); + let low_count = low_mask.to_bitmask().count_ones() as usize; + let (low, high) = values.split(); + let bits = mask.to_bitmask(); + if bits == 0 { + return merge; + } + if bits == 255 { + return values; + } + let low = low.compress(low_mask); + let high = high.compress(high_mask); + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::SHIFTS[low_count * 2], + ); + let low_bits = crate::transmute::checked_transmute_copy(&low.val); + let high_bits = crate::transmute::checked_transmute_copy(&high.val); + let shifted = _mm256_permutevar8x32_epi32(high_bits, control); + let result_low = _mm256_blendv_epi8(shifted, low_bits, control); + let result_high = _mm256_and_si256(shifted, _mm256_srai_epi32::<31>(control)); + let compressed = u64x8 { + val: crate::transmute::checked_transmute_copy(&[result_low, result_high]), + simd: token, + }; + let count = low_count + high_mask.to_bitmask().count_ones() as usize; + let prefix = mask64x8::from_bitmask(token, (1u64 << count) - 1); + token.select_u64x8(prefix, compressed, merge) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_u64x8(self, values: u64x8, mask: mask64x8) -> u64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, values: u64x8, mask: mask64x8) -> u64x8 { + let (low_mask, high_mask) = token.split_mask64x8(mask); + let low_count = low_mask.to_bitmask().count_ones() as usize; + let (low, high) = values.split(); + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::SHIFTS[8 - (low_count * 2)], + ); + let low_bits = crate::transmute::checked_transmute_copy(&low.val); + let high_bits = crate::transmute::checked_transmute_copy(&high.val); + let from_low = _mm256_permutevar8x32_epi32(low_bits, control); + let from_high = _mm256_permutevar8x32_epi32(high_bits, control); + let packed = _mm256_blendv_epi8(from_high, from_low, control); + let high = u64x4 { + val: crate::transmute::checked_transmute_copy(&packed), + simd: token, + }; + low.expand(low_mask).combine(high.expand(high_mask)) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_u64x8( + self, + values: u64x8, + mask: mask64x8, + merge: u64x8, + ) -> u64x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx2, + values: u64x8, + mask: mask64x8, + merge: u64x8, + ) -> u64x8 { + let (low_mask, high_mask) = token.split_mask64x8(mask); + let low_count = low_mask.to_bitmask().count_ones() as usize; + let (low, high) = values.split(); + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::SHIFTS[8 - (low_count * 2)], + ); + let low_bits = crate::transmute::checked_transmute_copy(&low.val); + let high_bits = crate::transmute::checked_transmute_copy(&high.val); + let from_low = _mm256_permutevar8x32_epi32(low_bits, control); + let from_high = _mm256_permutevar8x32_epi32(high_bits, control); + let packed = _mm256_blendv_epi8(from_high, from_low, control); + let high = u64x4 { + val: crate::transmute::checked_transmute_copy(&packed), + simd: token, + }; + let (low_merge, high_merge) = merge.split(); + low.expand_merge(low_mask, low_merge) + .combine(high.expand_merge(high_mask, high_merge)) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn split_u64x8(self, a: u64x8) -> (u64x4, u64x4) { ( u64x4 { diff --git a/fearless_simd/src/generated/avx512.rs b/fearless_simd/src/generated/avx512.rs index d24e4862..4681fe04 100644 --- a/fearless_simd/src/generated/avx512.rs +++ b/fearless_simd/src/generated/avx512.rs @@ -440,6 +440,76 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn compress_f32x4(self, values: f32x4, mask: mask32x4) -> f32x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: f32x4, + mask: mask32x4, + ) -> f32x4 { + _mm_maskz_compress_ps(u64::from((mask).val) as u8, values.into()).simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_f32x4( + self, + values: f32x4, + mask: mask32x4, + merge: f32x4, + ) -> f32x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: f32x4, + mask: mask32x4, + merge: f32x4, + ) -> f32x4 { + _mm_mask_compress_ps(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_f32x4(self, values: f32x4, mask: mask32x4) -> f32x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: f32x4, + mask: mask32x4, + ) -> f32x4 { + _mm_maskz_expand_ps(u64::from((mask).val) as u8, values.into()).simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_f32x4( + self, + values: f32x4, + mask: mask32x4, + merge: f32x4, + ) -> f32x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: f32x4, + mask: mask32x4, + merge: f32x4, + ) -> f32x4 { + _mm_mask_expand_ps(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn neg_f32x4(self, a: f32x4) -> f32x4 { crate::kernel!( #[inline(always)] @@ -1102,6 +1172,77 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn compress_i8x16(self, values: i8x16, mask: mask8x16) -> i8x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i8x16, + mask: mask8x16, + ) -> i8x16 { + _mm_maskz_compress_epi8(u64::from((mask).val) as u16, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_i8x16( + self, + values: i8x16, + mask: mask8x16, + merge: i8x16, + ) -> i8x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i8x16, + mask: mask8x16, + merge: i8x16, + ) -> i8x16 { + _mm_mask_compress_epi8(merge.into(), u64::from((mask).val) as u16, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_i8x16(self, values: i8x16, mask: mask8x16) -> i8x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i8x16, + mask: mask8x16, + ) -> i8x16 { + _mm_maskz_expand_epi8(u64::from((mask).val) as u16, values.into()).simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_i8x16( + self, + values: i8x16, + mask: mask8x16, + merge: i8x16, + ) -> i8x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i8x16, + mask: mask8x16, + merge: i8x16, + ) -> i8x16 { + _mm_mask_expand_epi8(merge.into(), u64::from((mask).val) as u16, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn count_ones_i8x16(self, a: i8x16) -> i8x16 { crate::kernel!( #[inline(always)] @@ -1762,6 +1903,77 @@ impl Simd for Avx512 { kernel(self, a, b, indices) } #[inline(always)] + fn compress_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u8x16, + mask: mask8x16, + ) -> u8x16 { + _mm_maskz_compress_epi8(u64::from((mask).val) as u16, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_u8x16( + self, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + _mm_mask_compress_epi8(merge.into(), u64::from((mask).val) as u16, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u8x16, + mask: mask8x16, + ) -> u8x16 { + _mm_maskz_expand_epi8(u64::from((mask).val) as u16, values.into()).simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_u8x16( + self, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + _mm_mask_expand_epi8(merge.into(), u64::from((mask).val) as u16, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn count_ones_u8x16(self, a: u8x16) -> u8x16 { crate::kernel!( #[inline(always)] @@ -2473,6 +2685,77 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn compress_i16x8(self, values: i16x8, mask: mask16x8) -> i16x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i16x8, + mask: mask16x8, + ) -> i16x8 { + _mm_maskz_compress_epi16(u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_i16x8( + self, + values: i16x8, + mask: mask16x8, + merge: i16x8, + ) -> i16x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i16x8, + mask: mask16x8, + merge: i16x8, + ) -> i16x8 { + _mm_mask_compress_epi16(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_i16x8(self, values: i16x8, mask: mask16x8) -> i16x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i16x8, + mask: mask16x8, + ) -> i16x8 { + _mm_maskz_expand_epi16(u64::from((mask).val) as u8, values.into()).simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_i16x8( + self, + values: i16x8, + mask: mask16x8, + merge: i16x8, + ) -> i16x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i16x8, + mask: mask16x8, + merge: i16x8, + ) -> i16x8 { + _mm_mask_expand_epi16(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn count_ones_i16x8(self, a: i16x8) -> i16x8 { crate::kernel!( #[inline(always)] @@ -3002,6 +3285,77 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn compress_u16x8(self, values: u16x8, mask: mask16x8) -> u16x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u16x8, + mask: mask16x8, + ) -> u16x8 { + _mm_maskz_compress_epi16(u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_u16x8( + self, + values: u16x8, + mask: mask16x8, + merge: u16x8, + ) -> u16x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u16x8, + mask: mask16x8, + merge: u16x8, + ) -> u16x8 { + _mm_mask_compress_epi16(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_u16x8(self, values: u16x8, mask: mask16x8) -> u16x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u16x8, + mask: mask16x8, + ) -> u16x8 { + _mm_maskz_expand_epi16(u64::from((mask).val) as u8, values.into()).simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_u16x8( + self, + values: u16x8, + mask: mask16x8, + merge: u16x8, + ) -> u16x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u16x8, + mask: mask16x8, + merge: u16x8, + ) -> u16x8 { + _mm_mask_expand_epi16(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn count_ones_u16x8(self, a: u16x8) -> u16x8 { crate::kernel!( #[inline(always)] @@ -3698,6 +4052,77 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn compress_i32x4(self, values: i32x4, mask: mask32x4) -> i32x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i32x4, + mask: mask32x4, + ) -> i32x4 { + _mm_maskz_compress_epi32(u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_i32x4( + self, + values: i32x4, + mask: mask32x4, + merge: i32x4, + ) -> i32x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i32x4, + mask: mask32x4, + merge: i32x4, + ) -> i32x4 { + _mm_mask_compress_epi32(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_i32x4(self, values: i32x4, mask: mask32x4) -> i32x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i32x4, + mask: mask32x4, + ) -> i32x4 { + _mm_maskz_expand_epi32(u64::from((mask).val) as u8, values.into()).simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_i32x4( + self, + values: i32x4, + mask: mask32x4, + merge: i32x4, + ) -> i32x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i32x4, + mask: mask32x4, + merge: i32x4, + ) -> i32x4 { + _mm_mask_expand_epi32(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn count_ones_i32x4(self, a: i32x4) -> i32x4 { crate::kernel!( #[inline(always)] @@ -4222,17 +4647,88 @@ impl Simd for Avx512 { }) } #[inline(always)] - fn count_ones_u32x4(self, a: u32x4) -> u32x4 { + fn compress_u32x4(self, values: u32x4, mask: mask32x4) -> u32x4 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx512, a: u32x4) -> u32x4 { - _mm_popcnt_epi32(a.into()).simd_into(token) + fn kernel( + token: Avx512, + values: u32x4, + mask: mask32x4, + ) -> u32x4 { + _mm_maskz_compress_epi32(u64::from((mask).val) as u8, values.into()) + .simd_into(token) } ); - kernel(self, a) + kernel(self, values, mask) } #[inline(always)] - fn count_zeros_u32x4(self, a: u32x4) -> u32x4 { + fn compress_merge_u32x4( + self, + values: u32x4, + mask: mask32x4, + merge: u32x4, + ) -> u32x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u32x4, + mask: mask32x4, + merge: u32x4, + ) -> u32x4 { + _mm_mask_compress_epi32(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_u32x4(self, values: u32x4, mask: mask32x4) -> u32x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u32x4, + mask: mask32x4, + ) -> u32x4 { + _mm_maskz_expand_epi32(u64::from((mask).val) as u8, values.into()).simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_u32x4( + self, + values: u32x4, + mask: mask32x4, + merge: u32x4, + ) -> u32x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u32x4, + mask: mask32x4, + merge: u32x4, + ) -> u32x4 { + _mm_mask_expand_epi32(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn count_ones_u32x4(self, a: u32x4) -> u32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x4) -> u32x4 { + _mm_popcnt_epi32(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn count_zeros_u32x4(self, a: u32x4) -> u32x4 { self.count_ones_u32x4(self.not_u32x4(a)) } #[inline(always)] @@ -4921,6 +5417,76 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn compress_f64x2(self, values: f64x2, mask: mask64x2) -> f64x2 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: f64x2, + mask: mask64x2, + ) -> f64x2 { + _mm_maskz_compress_pd(u64::from((mask).val) as u8, values.into()).simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_f64x2( + self, + values: f64x2, + mask: mask64x2, + merge: f64x2, + ) -> f64x2 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: f64x2, + mask: mask64x2, + merge: f64x2, + ) -> f64x2 { + _mm_mask_compress_pd(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_f64x2(self, values: f64x2, mask: mask64x2) -> f64x2 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: f64x2, + mask: mask64x2, + ) -> f64x2 { + _mm_maskz_expand_pd(u64::from((mask).val) as u8, values.into()).simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_f64x2( + self, + values: f64x2, + mask: mask64x2, + merge: f64x2, + ) -> f64x2 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: f64x2, + mask: mask64x2, + merge: f64x2, + ) -> f64x2 { + _mm_mask_expand_pd(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn neg_f64x2(self, a: f64x2) -> f64x2 { crate::kernel!( #[inline(always)] @@ -5573,6 +6139,77 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn compress_i64x2(self, values: i64x2, mask: mask64x2) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i64x2, + mask: mask64x2, + ) -> i64x2 { + _mm_maskz_compress_epi64(u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_i64x2( + self, + values: i64x2, + mask: mask64x2, + merge: i64x2, + ) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i64x2, + mask: mask64x2, + merge: i64x2, + ) -> i64x2 { + _mm_mask_compress_epi64(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_i64x2(self, values: i64x2, mask: mask64x2) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i64x2, + mask: mask64x2, + ) -> i64x2 { + _mm_maskz_expand_epi64(u64::from((mask).val) as u8, values.into()).simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_i64x2( + self, + values: i64x2, + mask: mask64x2, + merge: i64x2, + ) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i64x2, + mask: mask64x2, + merge: i64x2, + ) -> i64x2 { + _mm_mask_expand_epi64(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn count_ones_i64x2(self, a: i64x2) -> i64x2 { crate::kernel!( #[inline(always)] @@ -6058,6 +6695,77 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn compress_u64x2(self, values: u64x2, mask: mask64x2) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u64x2, + mask: mask64x2, + ) -> u64x2 { + _mm_maskz_compress_epi64(u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_u64x2( + self, + values: u64x2, + mask: mask64x2, + merge: u64x2, + ) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u64x2, + mask: mask64x2, + merge: u64x2, + ) -> u64x2 { + _mm_mask_compress_epi64(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_u64x2(self, values: u64x2, mask: mask64x2) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u64x2, + mask: mask64x2, + ) -> u64x2 { + _mm_maskz_expand_epi64(u64::from((mask).val) as u8, values.into()).simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_u64x2( + self, + values: u64x2, + mask: mask64x2, + merge: u64x2, + ) -> u64x2 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u64x2, + mask: mask64x2, + merge: u64x2, + ) -> u64x2 { + _mm_mask_expand_epi64(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn count_ones_u64x2(self, a: u64x2) -> u64x2 { crate::kernel!( #[inline(always)] @@ -6743,6 +7451,77 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn compress_f32x8(self, values: f32x8, mask: mask32x8) -> f32x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: f32x8, + mask: mask32x8, + ) -> f32x8 { + _mm256_maskz_compress_ps(u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_f32x8( + self, + values: f32x8, + mask: mask32x8, + merge: f32x8, + ) -> f32x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: f32x8, + mask: mask32x8, + merge: f32x8, + ) -> f32x8 { + _mm256_mask_compress_ps(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_f32x8(self, values: f32x8, mask: mask32x8) -> f32x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: f32x8, + mask: mask32x8, + ) -> f32x8 { + _mm256_maskz_expand_ps(u64::from((mask).val) as u8, values.into()).simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_f32x8( + self, + values: f32x8, + mask: mask32x8, + merge: f32x8, + ) -> f32x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: f32x8, + mask: mask32x8, + merge: f32x8, + ) -> f32x8 { + _mm256_mask_expand_ps(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn neg_f32x8(self, a: f32x8) -> f32x8 { crate::kernel!( #[inline(always)] @@ -7380,6 +8159,78 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn compress_i8x32(self, values: i8x32, mask: mask8x32) -> i8x32 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i8x32, + mask: mask8x32, + ) -> i8x32 { + _mm256_maskz_compress_epi8(u64::from((mask).val) as u32, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_i8x32( + self, + values: i8x32, + mask: mask8x32, + merge: i8x32, + ) -> i8x32 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i8x32, + mask: mask8x32, + merge: i8x32, + ) -> i8x32 { + _mm256_mask_compress_epi8(merge.into(), u64::from((mask).val) as u32, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_i8x32(self, values: i8x32, mask: mask8x32) -> i8x32 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i8x32, + mask: mask8x32, + ) -> i8x32 { + _mm256_maskz_expand_epi8(u64::from((mask).val) as u32, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_i8x32( + self, + values: i8x32, + mask: mask8x32, + merge: i8x32, + ) -> i8x32 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i8x32, + mask: mask8x32, + merge: i8x32, + ) -> i8x32 { + _mm256_mask_expand_epi8(merge.into(), u64::from((mask).val) as u32, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn count_ones_i8x32(self, a: i8x32) -> i8x32 { crate::kernel!( #[inline(always)] @@ -8013,13 +8864,85 @@ impl Simd for Avx512 { kernel(self, a, b, indices) } #[inline(always)] - fn count_ones_u8x32(self, a: u8x32) -> u8x32 { + fn compress_u8x32(self, values: u8x32, mask: mask8x32) -> u8x32 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx512, a: u8x32) -> u8x32 { - _mm256_popcnt_epi8(a.into()).simd_into(token) - } - ); + fn kernel( + token: Avx512, + values: u8x32, + mask: mask8x32, + ) -> u8x32 { + _mm256_maskz_compress_epi8(u64::from((mask).val) as u32, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_u8x32( + self, + values: u8x32, + mask: mask8x32, + merge: u8x32, + ) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u8x32, + mask: mask8x32, + merge: u8x32, + ) -> u8x32 { + _mm256_mask_compress_epi8(merge.into(), u64::from((mask).val) as u32, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_u8x32(self, values: u8x32, mask: mask8x32) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u8x32, + mask: mask8x32, + ) -> u8x32 { + _mm256_maskz_expand_epi8(u64::from((mask).val) as u32, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_u8x32( + self, + values: u8x32, + mask: mask8x32, + merge: u8x32, + ) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u8x32, + mask: mask8x32, + merge: u8x32, + ) -> u8x32 { + _mm256_mask_expand_epi8(merge.into(), u64::from((mask).val) as u32, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn count_ones_u8x32(self, a: u8x32) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x32) -> u8x32 { + _mm256_popcnt_epi8(a.into()).simd_into(token) + } + ); kernel(self, a) } #[inline(always)] @@ -8711,6 +9634,82 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn compress_i16x16(self, values: i16x16, mask: mask16x16) -> i16x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i16x16, + mask: mask16x16, + ) -> i16x16 { + _mm256_maskz_compress_epi16(u64::from((mask).val) as u16, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_i16x16( + self, + values: i16x16, + mask: mask16x16, + merge: i16x16, + ) -> i16x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i16x16, + mask: mask16x16, + merge: i16x16, + ) -> i16x16 { + _mm256_mask_compress_epi16( + merge.into(), + u64::from((mask).val) as u16, + values.into(), + ) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_i16x16(self, values: i16x16, mask: mask16x16) -> i16x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i16x16, + mask: mask16x16, + ) -> i16x16 { + _mm256_maskz_expand_epi16(u64::from((mask).val) as u16, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_i16x16( + self, + values: i16x16, + mask: mask16x16, + merge: i16x16, + ) -> i16x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i16x16, + mask: mask16x16, + merge: i16x16, + ) -> i16x16 { + _mm256_mask_expand_epi16(merge.into(), u64::from((mask).val) as u16, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn count_ones_i16x16(self, a: i16x16) -> i16x16 { crate::kernel!( #[inline(always)] @@ -9214,6 +10213,82 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn compress_u16x16(self, values: u16x16, mask: mask16x16) -> u16x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u16x16, + mask: mask16x16, + ) -> u16x16 { + _mm256_maskz_compress_epi16(u64::from((mask).val) as u16, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_u16x16( + self, + values: u16x16, + mask: mask16x16, + merge: u16x16, + ) -> u16x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u16x16, + mask: mask16x16, + merge: u16x16, + ) -> u16x16 { + _mm256_mask_compress_epi16( + merge.into(), + u64::from((mask).val) as u16, + values.into(), + ) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_u16x16(self, values: u16x16, mask: mask16x16) -> u16x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u16x16, + mask: mask16x16, + ) -> u16x16 { + _mm256_maskz_expand_epi16(u64::from((mask).val) as u16, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_u16x16( + self, + values: u16x16, + mask: mask16x16, + merge: u16x16, + ) -> u16x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u16x16, + mask: mask16x16, + merge: u16x16, + ) -> u16x16 { + _mm256_mask_expand_epi16(merge.into(), u64::from((mask).val) as u16, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn count_ones_u16x16(self, a: u16x16) -> u16x16 { crate::kernel!( #[inline(always)] @@ -9898,6 +10973,78 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn compress_i32x8(self, values: i32x8, mask: mask32x8) -> i32x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i32x8, + mask: mask32x8, + ) -> i32x8 { + _mm256_maskz_compress_epi32(u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_i32x8( + self, + values: i32x8, + mask: mask32x8, + merge: i32x8, + ) -> i32x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i32x8, + mask: mask32x8, + merge: i32x8, + ) -> i32x8 { + _mm256_mask_compress_epi32(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_i32x8(self, values: i32x8, mask: mask32x8) -> i32x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i32x8, + mask: mask32x8, + ) -> i32x8 { + _mm256_maskz_expand_epi32(u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_i32x8( + self, + values: i32x8, + mask: mask32x8, + merge: i32x8, + ) -> i32x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i32x8, + mask: mask32x8, + merge: i32x8, + ) -> i32x8 { + _mm256_mask_expand_epi32(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn count_ones_i32x8(self, a: i32x8) -> i32x8 { crate::kernel!( #[inline(always)] @@ -10405,6 +11552,78 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn compress_u32x8(self, values: u32x8, mask: mask32x8) -> u32x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u32x8, + mask: mask32x8, + ) -> u32x8 { + _mm256_maskz_compress_epi32(u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_u32x8( + self, + values: u32x8, + mask: mask32x8, + merge: u32x8, + ) -> u32x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u32x8, + mask: mask32x8, + merge: u32x8, + ) -> u32x8 { + _mm256_mask_compress_epi32(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_u32x8(self, values: u32x8, mask: mask32x8) -> u32x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u32x8, + mask: mask32x8, + ) -> u32x8 { + _mm256_maskz_expand_epi32(u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_u32x8( + self, + values: u32x8, + mask: mask32x8, + merge: u32x8, + ) -> u32x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u32x8, + mask: mask32x8, + merge: u32x8, + ) -> u32x8 { + _mm256_mask_expand_epi32(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn count_ones_u32x8(self, a: u32x8) -> u32x8 { crate::kernel!( #[inline(always)] @@ -11083,40 +12302,111 @@ impl Simd for Avx512 { }) } #[inline(always)] - fn neg_f64x4(self, a: f64x4) -> f64x4 { + fn compress_f64x4(self, values: f64x4, mask: mask64x4) -> f64x4 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx512, a: f64x4) -> f64x4 { - _mm256_xor_pd(a.into(), _mm256_set1_pd(-0.0)).simd_into(token) + fn kernel( + token: Avx512, + values: f64x4, + mask: mask64x4, + ) -> f64x4 { + _mm256_maskz_compress_pd(u64::from((mask).val) as u8, values.into()) + .simd_into(token) } ); - kernel(self, a) + kernel(self, values, mask) } #[inline(always)] - fn sqrt_f64x4(self, a: f64x4) -> f64x4 { + fn compress_merge_f64x4( + self, + values: f64x4, + mask: mask64x4, + merge: f64x4, + ) -> f64x4 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx512, a: f64x4) -> f64x4 { - _mm256_sqrt_pd(a.into()).simd_into(token) + fn kernel( + token: Avx512, + values: f64x4, + mask: mask64x4, + merge: f64x4, + ) -> f64x4 { + _mm256_mask_compress_pd(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) } ); - kernel(self, a) + kernel(self, values, mask, merge) } #[inline(always)] - fn approximate_recip_f64x4(self, a: f64x4) -> f64x4 { + fn expand_f64x4(self, values: f64x4, mask: mask64x4) -> f64x4 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx512, a: f64x4) -> f64x4 { - _mm256_rcp14_pd(a.into()).simd_into(token) + fn kernel( + token: Avx512, + values: f64x4, + mask: mask64x4, + ) -> f64x4 { + _mm256_maskz_expand_pd(u64::from((mask).val) as u8, values.into()).simd_into(token) } ); - kernel(self, a) + kernel(self, values, mask) } #[inline(always)] - fn add_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + fn expand_merge_f64x4( + self, + values: f64x4, + mask: mask64x4, + merge: f64x4, + ) -> f64x4 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx512, a: f64x4, b: f64x4) -> f64x4 { + fn kernel( + token: Avx512, + values: f64x4, + mask: mask64x4, + merge: f64x4, + ) -> f64x4 { + _mm256_mask_expand_pd(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn neg_f64x4(self, a: f64x4) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x4) -> f64x4 { + _mm256_xor_pd(a.into(), _mm256_set1_pd(-0.0)).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn sqrt_f64x4(self, a: f64x4) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x4) -> f64x4 { + _mm256_sqrt_pd(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn approximate_recip_f64x4(self, a: f64x4) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x4) -> f64x4 { + _mm256_rcp14_pd(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn add_f64x4(self, a: f64x4, b: f64x4) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x4, b: f64x4) -> f64x4 { _mm256_add_pd(a.into(), b.into()).simd_into(token) } ); @@ -11707,6 +12997,78 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn compress_i64x4(self, values: i64x4, mask: mask64x4) -> i64x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i64x4, + mask: mask64x4, + ) -> i64x4 { + _mm256_maskz_compress_epi64(u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_i64x4( + self, + values: i64x4, + mask: mask64x4, + merge: i64x4, + ) -> i64x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i64x4, + mask: mask64x4, + merge: i64x4, + ) -> i64x4 { + _mm256_mask_compress_epi64(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_i64x4(self, values: i64x4, mask: mask64x4) -> i64x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i64x4, + mask: mask64x4, + ) -> i64x4 { + _mm256_maskz_expand_epi64(u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_i64x4( + self, + values: i64x4, + mask: mask64x4, + merge: i64x4, + ) -> i64x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i64x4, + mask: mask64x4, + merge: i64x4, + ) -> i64x4 { + _mm256_mask_expand_epi64(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn count_ones_i64x4(self, a: i64x4) -> i64x4 { crate::kernel!( #[inline(always)] @@ -12184,6 +13546,78 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn compress_u64x4(self, values: u64x4, mask: mask64x4) -> u64x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u64x4, + mask: mask64x4, + ) -> u64x4 { + _mm256_maskz_compress_epi64(u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_u64x4( + self, + values: u64x4, + mask: mask64x4, + merge: u64x4, + ) -> u64x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u64x4, + mask: mask64x4, + merge: u64x4, + ) -> u64x4 { + _mm256_mask_compress_epi64(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_u64x4(self, values: u64x4, mask: mask64x4) -> u64x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u64x4, + mask: mask64x4, + ) -> u64x4 { + _mm256_maskz_expand_epi64(u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_u64x4( + self, + values: u64x4, + mask: mask64x4, + merge: u64x4, + ) -> u64x4 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u64x4, + mask: mask64x4, + merge: u64x4, + ) -> u64x4 { + _mm256_mask_expand_epi64(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn count_ones_u64x4(self, a: u64x4) -> u64x4 { crate::kernel!( #[inline(always)] @@ -12834,6 +14268,77 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn compress_f32x16(self, values: f32x16, mask: mask32x16) -> f32x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: f32x16, + mask: mask32x16, + ) -> f32x16 { + _mm512_maskz_compress_ps(u64::from((mask).val) as u16, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_f32x16( + self, + values: f32x16, + mask: mask32x16, + merge: f32x16, + ) -> f32x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: f32x16, + mask: mask32x16, + merge: f32x16, + ) -> f32x16 { + _mm512_mask_compress_ps(merge.into(), u64::from((mask).val) as u16, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_f32x16(self, values: f32x16, mask: mask32x16) -> f32x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: f32x16, + mask: mask32x16, + ) -> f32x16 { + _mm512_maskz_expand_ps(u64::from((mask).val) as u16, values.into()).simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_f32x16( + self, + values: f32x16, + mask: mask32x16, + merge: f32x16, + ) -> f32x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: f32x16, + mask: mask32x16, + merge: f32x16, + ) -> f32x16 { + _mm512_mask_expand_ps(merge.into(), u64::from((mask).val) as u16, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn neg_f32x16(self, a: f32x16) -> f32x16 { crate::kernel!( #[inline(always)] @@ -13496,6 +15001,78 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn compress_i8x64(self, values: i8x64, mask: mask8x64) -> i8x64 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i8x64, + mask: mask8x64, + ) -> i8x64 { + _mm512_maskz_compress_epi8(u64::from((mask).val) as u64, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_i8x64( + self, + values: i8x64, + mask: mask8x64, + merge: i8x64, + ) -> i8x64 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i8x64, + mask: mask8x64, + merge: i8x64, + ) -> i8x64 { + _mm512_mask_compress_epi8(merge.into(), u64::from((mask).val) as u64, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_i8x64(self, values: i8x64, mask: mask8x64) -> i8x64 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i8x64, + mask: mask8x64, + ) -> i8x64 { + _mm512_maskz_expand_epi8(u64::from((mask).val) as u64, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_i8x64( + self, + values: i8x64, + mask: mask8x64, + merge: i8x64, + ) -> i8x64 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i8x64, + mask: mask8x64, + merge: i8x64, + ) -> i8x64 { + _mm512_mask_expand_epi8(merge.into(), u64::from((mask).val) as u64, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn count_ones_i8x64(self, a: i8x64) -> i8x64 { crate::kernel!( #[inline(always)] @@ -14139,45 +15716,117 @@ impl Simd for Avx512 { kernel(self, a, b, indices) } #[inline(always)] - fn count_ones_u8x64(self, a: u8x64) -> u8x64 { + fn compress_u8x64(self, values: u8x64, mask: mask8x64) -> u8x64 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx512, a: u8x64) -> u8x64 { - _mm512_popcnt_epi8(a.into()).simd_into(token) + fn kernel( + token: Avx512, + values: u8x64, + mask: mask8x64, + ) -> u8x64 { + _mm512_maskz_compress_epi8(u64::from((mask).val) as u64, values.into()) + .simd_into(token) } ); - kernel(self, a) - } - #[inline(always)] - fn count_zeros_u8x64(self, a: u8x64) -> u8x64 { - self.count_ones_u8x64(self.not_u8x64(a)) + kernel(self, values, mask) } #[inline(always)] - fn add_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + fn compress_merge_u8x64( + self, + values: u8x64, + mask: mask8x64, + merge: u8x64, + ) -> u8x64 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx512, a: u8x64, b: u8x64) -> u8x64 { - _mm512_add_epi8(a.into(), b.into()).simd_into(token) + fn kernel( + token: Avx512, + values: u8x64, + mask: mask8x64, + merge: u8x64, + ) -> u8x64 { + _mm512_mask_compress_epi8(merge.into(), u64::from((mask).val) as u64, values.into()) + .simd_into(token) } ); - kernel(self, a, b) + kernel(self, values, mask, merge) } #[inline(always)] - fn saturating_add_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + fn expand_u8x64(self, values: u8x64, mask: mask8x64) -> u8x64 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx512, a: u8x64, b: u8x64) -> u8x64 { - _mm512_adds_epu8(a.into(), b.into()).simd_into(token) + fn kernel( + token: Avx512, + values: u8x64, + mask: mask8x64, + ) -> u8x64 { + _mm512_maskz_expand_epi8(u64::from((mask).val) as u64, values.into()) + .simd_into(token) } ); - kernel(self, a, b) + kernel(self, values, mask) } #[inline(always)] - fn sub_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + fn expand_merge_u8x64( + self, + values: u8x64, + mask: mask8x64, + merge: u8x64, + ) -> u8x64 { crate::kernel!( #[inline(always)] - fn kernel(token: Avx512, a: u8x64, b: u8x64) -> u8x64 { - _mm512_sub_epi8(a.into(), b.into()).simd_into(token) + fn kernel( + token: Avx512, + values: u8x64, + mask: mask8x64, + merge: u8x64, + ) -> u8x64 { + _mm512_mask_expand_epi8(merge.into(), u64::from((mask).val) as u64, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn count_ones_u8x64(self, a: u8x64) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x64) -> u8x64 { + _mm512_popcnt_epi8(a.into()).simd_into(token) + } + ); + kernel(self, a) + } + #[inline(always)] + fn count_zeros_u8x64(self, a: u8x64) -> u8x64 { + self.count_ones_u8x64(self.not_u8x64(a)) + } + #[inline(always)] + fn add_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x64, b: u8x64) -> u8x64 { + _mm512_add_epi8(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn saturating_add_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x64, b: u8x64) -> u8x64 { + _mm512_adds_epu8(a.into(), b.into()).simd_into(token) + } + ); + kernel(self, a, b) + } + #[inline(always)] + fn sub_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x64, b: u8x64) -> u8x64 { + _mm512_sub_epi8(a.into(), b.into()).simd_into(token) } ); kernel(self, a, b) @@ -14839,6 +16488,82 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn compress_i16x32(self, values: i16x32, mask: mask16x32) -> i16x32 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i16x32, + mask: mask16x32, + ) -> i16x32 { + _mm512_maskz_compress_epi16(u64::from((mask).val) as u32, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_i16x32( + self, + values: i16x32, + mask: mask16x32, + merge: i16x32, + ) -> i16x32 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i16x32, + mask: mask16x32, + merge: i16x32, + ) -> i16x32 { + _mm512_mask_compress_epi16( + merge.into(), + u64::from((mask).val) as u32, + values.into(), + ) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_i16x32(self, values: i16x32, mask: mask16x32) -> i16x32 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i16x32, + mask: mask16x32, + ) -> i16x32 { + _mm512_maskz_expand_epi16(u64::from((mask).val) as u32, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_i16x32( + self, + values: i16x32, + mask: mask16x32, + merge: i16x32, + ) -> i16x32 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i16x32, + mask: mask16x32, + merge: i16x32, + ) -> i16x32 { + _mm512_mask_expand_epi16(merge.into(), u64::from((mask).val) as u32, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn count_ones_i16x32(self, a: i16x32) -> i16x32 { crate::kernel!( #[inline(always)] @@ -15353,6 +17078,82 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn compress_u16x32(self, values: u16x32, mask: mask16x32) -> u16x32 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u16x32, + mask: mask16x32, + ) -> u16x32 { + _mm512_maskz_compress_epi16(u64::from((mask).val) as u32, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_u16x32( + self, + values: u16x32, + mask: mask16x32, + merge: u16x32, + ) -> u16x32 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u16x32, + mask: mask16x32, + merge: u16x32, + ) -> u16x32 { + _mm512_mask_compress_epi16( + merge.into(), + u64::from((mask).val) as u32, + values.into(), + ) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_u16x32(self, values: u16x32, mask: mask16x32) -> u16x32 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u16x32, + mask: mask16x32, + ) -> u16x32 { + _mm512_maskz_expand_epi16(u64::from((mask).val) as u32, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_u16x32( + self, + values: u16x32, + mask: mask16x32, + merge: u16x32, + ) -> u16x32 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u16x32, + mask: mask16x32, + merge: u16x32, + ) -> u16x32 { + _mm512_mask_expand_epi16(merge.into(), u64::from((mask).val) as u32, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn count_ones_u16x32(self, a: u16x32) -> u16x32 { crate::kernel!( #[inline(always)] @@ -16040,6 +17841,82 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn compress_i32x16(self, values: i32x16, mask: mask32x16) -> i32x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i32x16, + mask: mask32x16, + ) -> i32x16 { + _mm512_maskz_compress_epi32(u64::from((mask).val) as u16, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_i32x16( + self, + values: i32x16, + mask: mask32x16, + merge: i32x16, + ) -> i32x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i32x16, + mask: mask32x16, + merge: i32x16, + ) -> i32x16 { + _mm512_mask_compress_epi32( + merge.into(), + u64::from((mask).val) as u16, + values.into(), + ) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_i32x16(self, values: i32x16, mask: mask32x16) -> i32x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i32x16, + mask: mask32x16, + ) -> i32x16 { + _mm512_maskz_expand_epi32(u64::from((mask).val) as u16, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_i32x16( + self, + values: i32x16, + mask: mask32x16, + merge: i32x16, + ) -> i32x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i32x16, + mask: mask32x16, + merge: i32x16, + ) -> i32x16 { + _mm512_mask_expand_epi32(merge.into(), u64::from((mask).val) as u16, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn count_ones_i32x16(self, a: i32x16) -> i32x16 { crate::kernel!( #[inline(always)] @@ -16562,6 +18439,82 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn compress_u32x16(self, values: u32x16, mask: mask32x16) -> u32x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u32x16, + mask: mask32x16, + ) -> u32x16 { + _mm512_maskz_compress_epi32(u64::from((mask).val) as u16, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_u32x16( + self, + values: u32x16, + mask: mask32x16, + merge: u32x16, + ) -> u32x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u32x16, + mask: mask32x16, + merge: u32x16, + ) -> u32x16 { + _mm512_mask_compress_epi32( + merge.into(), + u64::from((mask).val) as u16, + values.into(), + ) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_u32x16(self, values: u32x16, mask: mask32x16) -> u32x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u32x16, + mask: mask32x16, + ) -> u32x16 { + _mm512_maskz_expand_epi32(u64::from((mask).val) as u16, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_u32x16( + self, + values: u32x16, + mask: mask32x16, + merge: u32x16, + ) -> u32x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u32x16, + mask: mask32x16, + merge: u32x16, + ) -> u32x16 { + _mm512_mask_expand_epi32(merge.into(), u64::from((mask).val) as u16, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn count_ones_u32x16(self, a: u32x16) -> u32x16 { crate::kernel!( #[inline(always)] @@ -17246,6 +19199,77 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn compress_f64x8(self, values: f64x8, mask: mask64x8) -> f64x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: f64x8, + mask: mask64x8, + ) -> f64x8 { + _mm512_maskz_compress_pd(u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_f64x8( + self, + values: f64x8, + mask: mask64x8, + merge: f64x8, + ) -> f64x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: f64x8, + mask: mask64x8, + merge: f64x8, + ) -> f64x8 { + _mm512_mask_compress_pd(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_f64x8(self, values: f64x8, mask: mask64x8) -> f64x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: f64x8, + mask: mask64x8, + ) -> f64x8 { + _mm512_maskz_expand_pd(u64::from((mask).val) as u8, values.into()).simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_f64x8( + self, + values: f64x8, + mask: mask64x8, + merge: f64x8, + ) -> f64x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: f64x8, + mask: mask64x8, + merge: f64x8, + ) -> f64x8 { + _mm512_mask_expand_pd(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn neg_f64x8(self, a: f64x8) -> f64x8 { crate::kernel!( #[inline(always)] @@ -17883,6 +19907,78 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn compress_i64x8(self, values: i64x8, mask: mask64x8) -> i64x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i64x8, + mask: mask64x8, + ) -> i64x8 { + _mm512_maskz_compress_epi64(u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_i64x8( + self, + values: i64x8, + mask: mask64x8, + merge: i64x8, + ) -> i64x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i64x8, + mask: mask64x8, + merge: i64x8, + ) -> i64x8 { + _mm512_mask_compress_epi64(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_i64x8(self, values: i64x8, mask: mask64x8) -> i64x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i64x8, + mask: mask64x8, + ) -> i64x8 { + _mm512_maskz_expand_epi64(u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_i64x8( + self, + values: i64x8, + mask: mask64x8, + merge: i64x8, + ) -> i64x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: i64x8, + mask: mask64x8, + merge: i64x8, + ) -> i64x8 { + _mm512_mask_expand_epi64(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn count_ones_i64x8(self, a: i64x8) -> i64x8 { crate::kernel!( #[inline(always)] @@ -18369,6 +20465,78 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn compress_u64x8(self, values: u64x8, mask: mask64x8) -> u64x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u64x8, + mask: mask64x8, + ) -> u64x8 { + _mm512_maskz_compress_epi64(u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_u64x8( + self, + values: u64x8, + mask: mask64x8, + merge: u64x8, + ) -> u64x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u64x8, + mask: mask64x8, + merge: u64x8, + ) -> u64x8 { + _mm512_mask_compress_epi64(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_u64x8(self, values: u64x8, mask: mask64x8) -> u64x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u64x8, + mask: mask64x8, + ) -> u64x8 { + _mm512_maskz_expand_epi64(u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_u64x8( + self, + values: u64x8, + mask: mask64x8, + merge: u64x8, + ) -> u64x8 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Avx512, + values: u64x8, + mask: mask64x8, + merge: u64x8, + ) -> u64x8 { + _mm512_mask_expand_epi64(merge.into(), u64::from((mask).val) as u8, values.into()) + .simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn count_ones_u64x8(self, a: u64x8) -> u64x8 { crate::kernel!( #[inline(always)] diff --git a/fearless_simd/src/generated/fallback.rs b/fearless_simd/src/generated/fallback.rs index 13cc8ac6..9edd51df 100644 --- a/fearless_simd/src/generated/fallback.rs +++ b/fearless_simd/src/generated/fallback.rs @@ -141,6 +141,56 @@ impl Simd for Fallback { dest.simd_into(self) } #[inline(always)] + fn compress_f32x4(self, values: f32x4, mask: mask32x4) -> f32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_f32x4( + self, + values: f32x4, + mask: mask32x4, + merge: f32x4, + ) -> f32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_f32x4(self, values: f32x4, mask: mask32x4) -> f32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_f32x4( + self, + values: f32x4, + mask: mask32x4, + merge: f32x4, + ) -> f32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn neg_f32x4(self, a: f32x4) -> f32x4 { [ f32::neg(a[0usize]), @@ -641,6 +691,56 @@ impl Simd for Fallback { dest.simd_into(self) } #[inline(always)] + fn compress_i8x16(self, values: i8x16, mask: mask8x16) -> i8x16 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i8x16( + self, + values: i8x16, + mask: mask8x16, + merge: i8x16, + ) -> i8x16 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i8x16(self, values: i8x16, mask: mask8x16) -> i8x16 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i8x16( + self, + values: i8x16, + mask: mask8x16, + merge: i8x16, + ) -> i8x16 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_i8x16(self, a: i8x16) -> i8x16 { [ i8::try_from(a[0usize].count_ones()).unwrap(), @@ -1679,6 +1779,53 @@ impl Simd for Fallback { Bytes::from_bytes(result) } #[inline(always)] + fn compress_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16 { + self.compress_merge_u8x16(values, mask, u8x16::splat(self, 0)) + } + #[inline(always)] + fn compress_merge_u8x16( + self, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + let mask = self.to_bitmask_mask8x16(mask); + let mut merge_padded = [0u8; 16 + 1]; + merge_padded[..16].copy_from_slice(&*merge); + let mut compacted = merge_padded; + let mut output_lane = 0; + for input_lane in 0..16 { + compacted[output_lane] = values[input_lane]; + output_lane += ((mask >> input_lane) & 1) as usize; + } + compacted[output_lane] = merge_padded[output_lane]; + let mut result = merge; + result.copy_from_slice(&compacted[..16]); + result + } + #[inline(always)] + fn expand_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16 { + self.expand_merge_u8x16(values, mask, u8x16::splat(self, 0)) + } + #[inline(always)] + fn expand_merge_u8x16( + self, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + let mask = self.to_bitmask_mask8x16(mask); + let mut result = merge; + let mut input_lane = 0; + for output_lane in 0..16 { + let bit = ((mask >> output_lane) & 1) as u8; + let keep = 0u8.wrapping_sub(bit); + result[output_lane] = (values[input_lane] & keep) | (result[output_lane] & !keep); + input_lane += usize::from(bit); + } + result + } + #[inline(always)] fn count_ones_u8x16(self, a: u8x16) -> u8x16 { [ u8::try_from(a[0usize].count_ones()).unwrap(), @@ -2982,6 +3129,56 @@ impl Simd for Fallback { dest.simd_into(self) } #[inline(always)] + fn compress_i16x8(self, values: i16x8, mask: mask16x8) -> i16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i16x8( + self, + values: i16x8, + mask: mask16x8, + merge: i16x8, + ) -> i16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i16x8(self, values: i16x8, mask: mask16x8) -> i16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i16x8( + self, + values: i16x8, + mask: mask16x8, + merge: i16x8, + ) -> i16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_i16x8(self, a: i16x8) -> i16x8 { [ i16::try_from(a[0usize].count_ones()).unwrap(), @@ -3607,6 +3804,56 @@ impl Simd for Fallback { dest.simd_into(self) } #[inline(always)] + fn compress_u16x8(self, values: u16x8, mask: mask16x8) -> u16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u16x8( + self, + values: u16x8, + mask: mask16x8, + merge: u16x8, + ) -> u16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u16x8(self, values: u16x8, mask: mask16x8) -> u16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u16x8( + self, + values: u16x8, + mask: mask16x8, + merge: u16x8, + ) -> u16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_u16x8(self, a: u16x8) -> u16x8 { [ u16::try_from(a[0usize].count_ones()).unwrap(), @@ -4525,6 +4772,56 @@ impl Simd for Fallback { dest.simd_into(self) } #[inline(always)] + fn compress_i32x4(self, values: i32x4, mask: mask32x4) -> i32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i32x4( + self, + values: i32x4, + mask: mask32x4, + merge: i32x4, + ) -> i32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i32x4(self, values: i32x4, mask: mask32x4) -> i32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i32x4( + self, + values: i32x4, + mask: mask32x4, + merge: i32x4, + ) -> i32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_i32x4(self, a: i32x4) -> i32x4 { [ a[0usize].count_ones().cast_signed(), @@ -4933,6 +5230,56 @@ impl Simd for Fallback { dest.simd_into(self) } #[inline(always)] + fn compress_u32x4(self, values: u32x4, mask: mask32x4) -> u32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u32x4( + self, + values: u32x4, + mask: mask32x4, + merge: u32x4, + ) -> u32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u32x4(self, values: u32x4, mask: mask32x4) -> u32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u32x4( + self, + values: u32x4, + mask: mask32x4, + merge: u32x4, + ) -> u32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_u32x4(self, a: u32x4) -> u32x4 { [ a[0usize].count_ones(), @@ -5539,6 +5886,56 @@ impl Simd for Fallback { dest.simd_into(self) } #[inline(always)] + fn compress_f64x2(self, values: f64x2, mask: mask64x2) -> f64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_f64x2( + self, + values: f64x2, + mask: mask64x2, + merge: f64x2, + ) -> f64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_f64x2(self, values: f64x2, mask: mask64x2) -> f64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_f64x2( + self, + values: f64x2, + mask: mask64x2, + merge: f64x2, + ) -> f64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn neg_f64x2(self, a: f64x2) -> f64x2 { [f64::neg(a[0usize]), f64::neg(a[1usize])].simd_into(self) } @@ -5893,6 +6290,56 @@ impl Simd for Fallback { dest.simd_into(self) } #[inline(always)] + fn compress_i64x2(self, values: i64x2, mask: mask64x2) -> i64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i64x2( + self, + values: i64x2, + mask: mask64x2, + merge: i64x2, + ) -> i64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i64x2(self, values: i64x2, mask: mask64x2) -> i64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i64x2( + self, + values: i64x2, + mask: mask64x2, + merge: i64x2, + ) -> i64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_i64x2(self, a: i64x2) -> i64x2 { [ i64::from(a[0usize].count_ones()), @@ -6196,6 +6643,56 @@ impl Simd for Fallback { dest.simd_into(self) } #[inline(always)] + fn compress_u64x2(self, values: u64x2, mask: mask64x2) -> u64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u64x2( + self, + values: u64x2, + mask: mask64x2, + merge: u64x2, + ) -> u64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u64x2(self, values: u64x2, mask: mask64x2) -> u64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u64x2( + self, + values: u64x2, + mask: mask64x2, + merge: u64x2, + ) -> u64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_u64x2(self, a: u64x2) -> u64x2 { [ u64::from(a[0usize].count_ones()), @@ -6648,6 +7145,56 @@ impl Simd for Fallback { dest.simd_into(self) } #[inline(always)] + fn compress_f32x8(self, values: f32x8, mask: mask32x8) -> f32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_f32x8( + self, + values: f32x8, + mask: mask32x8, + merge: f32x8, + ) -> f32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_f32x8(self, values: f32x8, mask: mask32x8) -> f32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_f32x8( + self, + values: f32x8, + mask: mask32x8, + merge: f32x8, + ) -> f32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn is_nan_f32x8(self, a: f32x8) -> mask32x8 { a.simd_ne(a) } @@ -6702,6 +7249,56 @@ impl Simd for Fallback { dest.simd_into(self) } #[inline(always)] + fn compress_i8x32(self, values: i8x32, mask: mask8x32) -> i8x32 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i8x32( + self, + values: i8x32, + mask: mask8x32, + merge: i8x32, + ) -> i8x32 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i8x32(self, values: i8x32, mask: mask8x32) -> i8x32 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i8x32( + self, + values: i8x32, + mask: mask8x32, + merge: i8x32, + ) -> i8x32 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn combine_i8x32(self, a: i8x32, b: i8x32) -> i8x64 { let mut result = [0; 64usize]; result[0..32usize].copy_from_slice(&a.val.0); @@ -6791,6 +7388,53 @@ impl Simd for Fallback { Bytes::from_bytes(result) } #[inline(always)] + fn compress_u8x32(self, values: u8x32, mask: mask8x32) -> u8x32 { + self.compress_merge_u8x32(values, mask, u8x32::splat(self, 0)) + } + #[inline(always)] + fn compress_merge_u8x32( + self, + values: u8x32, + mask: mask8x32, + merge: u8x32, + ) -> u8x32 { + let mask = self.to_bitmask_mask8x32(mask); + let mut merge_padded = [0u8; 32 + 1]; + merge_padded[..32].copy_from_slice(&*merge); + let mut compacted = merge_padded; + let mut output_lane = 0; + for input_lane in 0..32 { + compacted[output_lane] = values[input_lane]; + output_lane += ((mask >> input_lane) & 1) as usize; + } + compacted[output_lane] = merge_padded[output_lane]; + let mut result = merge; + result.copy_from_slice(&compacted[..32]); + result + } + #[inline(always)] + fn expand_u8x32(self, values: u8x32, mask: mask8x32) -> u8x32 { + self.expand_merge_u8x32(values, mask, u8x32::splat(self, 0)) + } + #[inline(always)] + fn expand_merge_u8x32( + self, + values: u8x32, + mask: mask8x32, + merge: u8x32, + ) -> u8x32 { + let mask = self.to_bitmask_mask8x32(mask); + let mut result = merge; + let mut input_lane = 0; + for output_lane in 0..32 { + let bit = ((mask >> output_lane) & 1) as u8; + let keep = 0u8.wrapping_sub(bit); + result[output_lane] = (values[input_lane] & keep) | (result[output_lane] & !keep); + input_lane += usize::from(bit); + } + result + } + #[inline(always)] fn combine_u8x32(self, a: u8x32, b: u8x32) -> u8x64 { let mut result = [0; 64usize]; result[0..32usize].copy_from_slice(&a.val.0); @@ -6869,6 +7513,56 @@ impl Simd for Fallback { dest.simd_into(self) } #[inline(always)] + fn compress_i16x16(self, values: i16x16, mask: mask16x16) -> i16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i16x16( + self, + values: i16x16, + mask: mask16x16, + merge: i16x16, + ) -> i16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i16x16(self, values: i16x16, mask: mask16x16) -> i16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i16x16( + self, + values: i16x16, + mask: mask16x16, + merge: i16x16, + ) -> i16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn combine_i16x16(self, a: i16x16, b: i16x16) -> i16x32 { let mut result = [0; 32usize]; result[0..16usize].copy_from_slice(&a.val.0); @@ -6891,18 +7585,68 @@ impl Simd for Fallback { dest.simd_into(self) } #[inline(always)] - fn combine_u16x16(self, a: u16x16, b: u16x16) -> u16x32 { - let mut result = [0; 32usize]; - result[0..16usize].copy_from_slice(&a.val.0); - result[16usize..32usize].copy_from_slice(&b.val.0); - result.simd_into(self) + fn compress_u16x16(self, values: u16x16, mask: mask16x16) -> u16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) } #[inline(always)] - fn split_u16x16(self, a: u16x16) -> (u16x8, u16x8) { - let mut b0 = [0; 8usize]; - let mut b1 = [0; 8usize]; - b0.copy_from_slice(&a.val.0[0..8usize]); - b1.copy_from_slice(&a.val.0[8usize..16usize]); + fn compress_merge_u16x16( + self, + values: u16x16, + mask: mask16x16, + merge: u16x16, + ) -> u16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u16x16(self, values: u16x16, mask: mask16x16) -> u16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u16x16( + self, + values: u16x16, + mask: mask16x16, + merge: u16x16, + ) -> u16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn combine_u16x16(self, a: u16x16, b: u16x16) -> u16x32 { + let mut result = [0; 32usize]; + result[0..16usize].copy_from_slice(&a.val.0); + result[16usize..32usize].copy_from_slice(&b.val.0); + result.simd_into(self) + } + #[inline(always)] + fn split_u16x16(self, a: u16x16) -> (u16x8, u16x8) { + let mut b0 = [0; 8usize]; + let mut b1 = [0; 8usize]; + b0.copy_from_slice(&a.val.0[0..8usize]); + b1.copy_from_slice(&a.val.0[8usize..16usize]); (b0.simd_into(self), b1.simd_into(self)) } #[inline(always)] @@ -6969,6 +7713,56 @@ impl Simd for Fallback { dest.simd_into(self) } #[inline(always)] + fn compress_i32x8(self, values: i32x8, mask: mask32x8) -> i32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i32x8( + self, + values: i32x8, + mask: mask32x8, + merge: i32x8, + ) -> i32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i32x8(self, values: i32x8, mask: mask32x8) -> i32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i32x8( + self, + values: i32x8, + mask: mask32x8, + merge: i32x8, + ) -> i32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn combine_i32x8(self, a: i32x8, b: i32x8) -> i32x16 { let mut result = [0; 16usize]; result[0..8usize].copy_from_slice(&a.val.0); @@ -6991,6 +7785,56 @@ impl Simd for Fallback { dest.simd_into(self) } #[inline(always)] + fn compress_u32x8(self, values: u32x8, mask: mask32x8) -> u32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u32x8( + self, + values: u32x8, + mask: mask32x8, + merge: u32x8, + ) -> u32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u32x8(self, values: u32x8, mask: mask32x8) -> u32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u32x8( + self, + values: u32x8, + mask: mask32x8, + merge: u32x8, + ) -> u32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn combine_u32x8(self, a: u32x8, b: u32x8) -> u32x16 { let mut result = [0; 16usize]; result[0..8usize].copy_from_slice(&a.val.0); @@ -7069,6 +7913,56 @@ impl Simd for Fallback { dest.simd_into(self) } #[inline(always)] + fn compress_f64x4(self, values: f64x4, mask: mask64x4) -> f64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_f64x4( + self, + values: f64x4, + mask: mask64x4, + merge: f64x4, + ) -> f64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_f64x4(self, values: f64x4, mask: mask64x4) -> f64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_f64x4( + self, + values: f64x4, + mask: mask64x4, + merge: f64x4, + ) -> f64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn is_nan_f64x4(self, a: f64x4) -> mask64x4 { a.simd_ne(a) } @@ -7123,6 +8017,56 @@ impl Simd for Fallback { dest.simd_into(self) } #[inline(always)] + fn compress_i64x4(self, values: i64x4, mask: mask64x4) -> i64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i64x4( + self, + values: i64x4, + mask: mask64x4, + merge: i64x4, + ) -> i64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i64x4(self, values: i64x4, mask: mask64x4) -> i64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i64x4( + self, + values: i64x4, + mask: mask64x4, + merge: i64x4, + ) -> i64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn combine_i64x4(self, a: i64x4, b: i64x4) -> i64x8 { let mut result = [0; 8usize]; result[0..4usize].copy_from_slice(&a.val.0); @@ -7145,6 +8089,56 @@ impl Simd for Fallback { dest.simd_into(self) } #[inline(always)] + fn compress_u64x4(self, values: u64x4, mask: mask64x4) -> u64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u64x4( + self, + values: u64x4, + mask: mask64x4, + merge: u64x4, + ) -> u64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u64x4(self, values: u64x4, mask: mask64x4) -> u64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u64x4( + self, + values: u64x4, + mask: mask64x4, + merge: u64x4, + ) -> u64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn combine_u64x4(self, a: u64x4, b: u64x4) -> u64x8 { let mut result = [0; 8usize]; result[0..4usize].copy_from_slice(&a.val.0); @@ -7223,6 +8217,56 @@ impl Simd for Fallback { dest.simd_into(self) } #[inline(always)] + fn compress_f32x16(self, values: f32x16, mask: mask32x16) -> f32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_f32x16( + self, + values: f32x16, + mask: mask32x16, + merge: f32x16, + ) -> f32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_f32x16(self, values: f32x16, mask: mask32x16) -> f32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_f32x16( + self, + values: f32x16, + mask: mask32x16, + merge: f32x16, + ) -> f32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn is_nan_f32x16(self, a: f32x16) -> mask32x16 { a.simd_ne(a) } @@ -7270,6 +8314,56 @@ impl Simd for Fallback { dest.simd_into(self) } #[inline(always)] + fn compress_i8x64(self, values: i8x64, mask: mask8x64) -> i8x64 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i8x64( + self, + values: i8x64, + mask: mask8x64, + merge: i8x64, + ) -> i8x64 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i8x64(self, values: i8x64, mask: mask8x64) -> i8x64 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i8x64( + self, + values: i8x64, + mask: mask8x64, + merge: i8x64, + ) -> i8x64 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_i8x64(self, a: i8x64) -> (i8x32, i8x32) { let mut b0 = [0; 32usize]; let mut b1 = [0; 32usize]; @@ -7352,6 +8446,53 @@ impl Simd for Fallback { Bytes::from_bytes(result) } #[inline(always)] + fn compress_u8x64(self, values: u8x64, mask: mask8x64) -> u8x64 { + self.compress_merge_u8x64(values, mask, u8x64::splat(self, 0)) + } + #[inline(always)] + fn compress_merge_u8x64( + self, + values: u8x64, + mask: mask8x64, + merge: u8x64, + ) -> u8x64 { + let mask = self.to_bitmask_mask8x64(mask); + let mut merge_padded = [0u8; 64 + 1]; + merge_padded[..64].copy_from_slice(&*merge); + let mut compacted = merge_padded; + let mut output_lane = 0; + for input_lane in 0..64 { + compacted[output_lane] = values[input_lane]; + output_lane += ((mask >> input_lane) & 1) as usize; + } + compacted[output_lane] = merge_padded[output_lane]; + let mut result = merge; + result.copy_from_slice(&compacted[..64]); + result + } + #[inline(always)] + fn expand_u8x64(self, values: u8x64, mask: mask8x64) -> u8x64 { + self.expand_merge_u8x64(values, mask, u8x64::splat(self, 0)) + } + #[inline(always)] + fn expand_merge_u8x64( + self, + values: u8x64, + mask: mask8x64, + merge: u8x64, + ) -> u8x64 { + let mask = self.to_bitmask_mask8x64(mask); + let mut result = merge; + let mut input_lane = 0; + for output_lane in 0..64 { + let bit = ((mask >> output_lane) & 1) as u8; + let keep = 0u8.wrapping_sub(bit); + result[output_lane] = (values[input_lane] & keep) | (result[output_lane] & !keep); + input_lane += usize::from(bit); + } + result + } + #[inline(always)] fn split_u8x64(self, a: u8x64) -> (u8x32, u8x32) { let mut b0 = [0; 32usize]; let mut b1 = [0; 32usize]; @@ -7416,6 +8557,56 @@ impl Simd for Fallback { dest.simd_into(self) } #[inline(always)] + fn compress_i16x32(self, values: i16x32, mask: mask16x32) -> i16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i16x32( + self, + values: i16x32, + mask: mask16x32, + merge: i16x32, + ) -> i16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i16x32(self, values: i16x32, mask: mask16x32) -> i16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i16x32( + self, + values: i16x32, + mask: mask16x32, + merge: i16x32, + ) -> i16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_i16x32(self, a: i16x32) -> (i16x16, i16x16) { let mut b0 = [0; 16usize]; let mut b1 = [0; 16usize]; @@ -7431,6 +8622,56 @@ impl Simd for Fallback { dest.simd_into(self) } #[inline(always)] + fn compress_u16x32(self, values: u16x32, mask: mask16x32) -> u16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u16x32( + self, + values: u16x32, + mask: mask16x32, + merge: u16x32, + ) -> u16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u16x32(self, values: u16x32, mask: mask16x32) -> u16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u16x32( + self, + values: u16x32, + mask: mask16x32, + merge: u16x32, + ) -> u16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_u16x32(self, a: u16x32) -> (u16x16, u16x16) { let mut b0 = [0; 16usize]; let mut b1 = [0; 16usize]; @@ -7495,6 +8736,56 @@ impl Simd for Fallback { dest.simd_into(self) } #[inline(always)] + fn compress_i32x16(self, values: i32x16, mask: mask32x16) -> i32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i32x16( + self, + values: i32x16, + mask: mask32x16, + merge: i32x16, + ) -> i32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i32x16(self, values: i32x16, mask: mask32x16) -> i32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i32x16( + self, + values: i32x16, + mask: mask32x16, + merge: i32x16, + ) -> i32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_i32x16(self, a: i32x16) -> (i32x8, i32x8) { let mut b0 = [0; 8usize]; let mut b1 = [0; 8usize]; @@ -7510,6 +8801,56 @@ impl Simd for Fallback { dest.simd_into(self) } #[inline(always)] + fn compress_u32x16(self, values: u32x16, mask: mask32x16) -> u32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u32x16( + self, + values: u32x16, + mask: mask32x16, + merge: u32x16, + ) -> u32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u32x16(self, values: u32x16, mask: mask32x16) -> u32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u32x16( + self, + values: u32x16, + mask: mask32x16, + merge: u32x16, + ) -> u32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_u32x16(self, a: u32x16) -> (u32x8, u32x8) { let mut b0 = [0; 8usize]; let mut b1 = [0; 8usize]; @@ -7574,6 +8915,56 @@ impl Simd for Fallback { dest.simd_into(self) } #[inline(always)] + fn compress_f64x8(self, values: f64x8, mask: mask64x8) -> f64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_f64x8( + self, + values: f64x8, + mask: mask64x8, + merge: f64x8, + ) -> f64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_f64x8(self, values: f64x8, mask: mask64x8) -> f64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_f64x8( + self, + values: f64x8, + mask: mask64x8, + merge: f64x8, + ) -> f64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn is_nan_f64x8(self, a: f64x8) -> mask64x8 { a.simd_ne(a) } @@ -7621,6 +9012,56 @@ impl Simd for Fallback { dest.simd_into(self) } #[inline(always)] + fn compress_i64x8(self, values: i64x8, mask: mask64x8) -> i64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i64x8( + self, + values: i64x8, + mask: mask64x8, + merge: i64x8, + ) -> i64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i64x8(self, values: i64x8, mask: mask64x8) -> i64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i64x8( + self, + values: i64x8, + mask: mask64x8, + merge: i64x8, + ) -> i64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_i64x8(self, a: i64x8) -> (i64x4, i64x4) { let mut b0 = [0; 4usize]; let mut b1 = [0; 4usize]; @@ -7636,6 +9077,56 @@ impl Simd for Fallback { dest.simd_into(self) } #[inline(always)] + fn compress_u64x8(self, values: u64x8, mask: mask64x8) -> u64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u64x8( + self, + values: u64x8, + mask: mask64x8, + merge: u64x8, + ) -> u64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u64x8(self, values: u64x8, mask: mask64x8) -> u64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u64x8( + self, + values: u64x8, + mask: mask64x8, + merge: u64x8, + ) -> u64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_u64x8(self, a: u64x8) -> (u64x4, u64x4) { let mut b0 = [0; 4usize]; let mut b1 = [0; 4usize]; diff --git a/fearless_simd/src/generated/neon.rs b/fearless_simd/src/generated/neon.rs index fd318d7c..931bf675 100644 --- a/fearless_simd/src/generated/neon.rs +++ b/fearless_simd/src/generated/neon.rs @@ -159,6 +159,90 @@ impl Simd for Neon { }) } #[inline(always)] + fn compress_f32x4(self, values: f32x4, mask: mask32x4) -> f32x4 { + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn compress_merge_f32x4( + self, + values: f32x4, + mask: mask32x4, + merge: f32x4, + ) -> f32x4 { + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0); + Bytes::from_bytes({ + crate::kernel!( + #[inline(always)] + fn merge_swizzle( + token: Neon, + values: u8x16, + control: u8x16, + merge: u8x16, + ) -> u8x16 { + let values: uint8x16_t = values.into(); + let control: uint8x16_t = control.into(); + let merge: uint8x16_t = merge.into(); + let result = vqtbx1q_u8(merge, values, control); + u8x16 { + val: crate::support::Aligned128(result), + simd: token, + } + } + ); + merge_swizzle( + self, + Bytes::to_bytes(values), + control, + Bytes::to_bytes(merge), + ) + }) + } + #[inline(always)] + fn expand_f32x4(self, values: f32x4, mask: mask32x4) -> f32x4 { + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn expand_merge_f32x4( + self, + values: f32x4, + mask: mask32x4, + merge: f32x4, + ) -> f32x4 { + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0); + Bytes::from_bytes({ + crate::kernel!( + #[inline(always)] + fn merge_swizzle( + token: Neon, + values: u8x16, + control: u8x16, + merge: u8x16, + ) -> u8x16 { + let values: uint8x16_t = values.into(); + let control: uint8x16_t = control.into(); + let merge: uint8x16_t = merge.into(); + let result = vqtbx1q_u8(merge, values, control); + u8x16 { + val: crate::support::Aligned128(result), + simd: token, + } + } + ); + merge_swizzle( + self, + Bytes::to_bytes(values), + control, + Bytes::to_bytes(merge), + ) + }) + } + #[inline(always)] fn neg_f32x4(self, a: f32x4) -> f32x4 { crate::kernel!( #[inline(always)] @@ -680,6 +764,56 @@ impl Simd for Neon { }) } #[inline(always)] + fn compress_i8x16(self, values: i8x16, mask: mask8x16) -> i8x16 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i8x16( + self, + values: i8x16, + mask: mask8x16, + merge: i8x16, + ) -> i8x16 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i8x16(self, values: i8x16, mask: mask8x16) -> i8x16 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i8x16( + self, + values: i8x16, + mask: mask8x16, + merge: i8x16, + ) -> i8x16 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_i8x16(self, a: i8x16) -> i8x16 { crate::kernel!( #[inline(always)] @@ -1142,6 +1276,126 @@ impl Simd for Neon { kernel(self, a, b, indices) } #[inline(always)] + fn compress_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16 { + let mask_bits = self.to_bitmask_mask8x16(mask); + let mut control = [u8::MAX; 16]; + let mut output_lane = 0; + for block in 0..16 / 8 { + let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; + let packed = crate::support::COMPRESS_8_CONTROLS[block_mask]; + let base = (block * 8) as u64 * 0x0101_0101_0101_0101; + let adjusted = packed | base; + let adjusted = adjusted.to_le_bytes(); + let write_len = core::cmp::min(8, 16 - output_lane); + control[output_lane..output_lane + write_len].copy_from_slice(&adjusted[..write_len]); + output_lane += crate::support::COMPACT_8_COUNTS[block_mask] as usize; + } + let control = u8x16::simd_from(self, control); + self.swizzle_dyn_precise_u8x16(values, control) + } + #[inline(always)] + fn compress_merge_u8x16( + self, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + let mask_bits = self.to_bitmask_mask8x16(mask); + let mut control = [u8::MAX; 16]; + let mut output_lane = 0; + for block in 0..16 / 8 { + let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; + let packed = crate::support::COMPRESS_8_CONTROLS[block_mask]; + let base = (block * 8) as u64 * 0x0101_0101_0101_0101; + let adjusted = packed | base; + let adjusted = adjusted.to_le_bytes(); + let write_len = core::cmp::min(8, 16 - output_lane); + control[output_lane..output_lane + write_len].copy_from_slice(&adjusted[..write_len]); + output_lane += crate::support::COMPACT_8_COUNTS[block_mask] as usize; + } + let control = u8x16::simd_from(self, control); + { + crate::kernel!( + #[inline(always)] + fn merge_swizzle( + token: Neon, + values: u8x16, + control: u8x16, + merge: u8x16, + ) -> u8x16 { + let values: uint8x16_t = values.into(); + let control: uint8x16_t = control.into(); + let merge: uint8x16_t = merge.into(); + let result = vqtbx1q_u8(merge, values, control); + u8x16 { + val: crate::support::Aligned128(result), + simd: token, + } + } + ); + merge_swizzle(self, values, control, merge) + } + } + #[inline(always)] + fn expand_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16 { + let mask_bits = self.to_bitmask_mask8x16(mask); + let mut control = [u8::MAX; 16]; + let mut input_lane = 0; + for block in 0..16 / 8 { + let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; + let packed = crate::support::EXPAND_8_CONTROLS[block_mask]; + let base = input_lane as u64 * 0x0101_0101_0101_0101; + let adjusted = packed + base; + let output_lane = block * 8; + control[output_lane..output_lane + 8].copy_from_slice(&adjusted.to_le_bytes()); + input_lane += crate::support::COMPACT_8_COUNTS[block_mask] as usize; + } + let control = u8x16::simd_from(self, control); + self.swizzle_dyn_precise_u8x16(values, control) + } + #[inline(always)] + fn expand_merge_u8x16( + self, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + let mask_bits = self.to_bitmask_mask8x16(mask); + let mut control = [u8::MAX; 16]; + let mut input_lane = 0; + for block in 0..16 / 8 { + let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; + let packed = crate::support::EXPAND_8_CONTROLS[block_mask]; + let base = input_lane as u64 * 0x0101_0101_0101_0101; + let adjusted = packed + base; + let output_lane = block * 8; + control[output_lane..output_lane + 8].copy_from_slice(&adjusted.to_le_bytes()); + input_lane += crate::support::COMPACT_8_COUNTS[block_mask] as usize; + } + let control = u8x16::simd_from(self, control); + { + crate::kernel!( + #[inline(always)] + fn merge_swizzle( + token: Neon, + values: u8x16, + control: u8x16, + merge: u8x16, + ) -> u8x16 { + let values: uint8x16_t = values.into(); + let control: uint8x16_t = control.into(); + let merge: uint8x16_t = merge.into(); + let result = vqtbx1q_u8(merge, values, control); + u8x16 { + val: crate::support::Aligned128(result), + simd: token, + } + } + ); + merge_swizzle(self, values, control, merge) + } + } + #[inline(always)] fn count_ones_u8x16(self, a: u8x16) -> u8x16 { crate::kernel!( #[inline(always)] @@ -1769,6 +2023,90 @@ impl Simd for Neon { }) } #[inline(always)] + fn compress_i16x8(self, values: i16x8, mask: mask16x8) -> i16x8 { + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_16[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn compress_merge_i16x8( + self, + values: i16x8, + mask: mask16x8, + merge: i16x8, + ) -> i16x8 { + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_16[bits].0); + Bytes::from_bytes({ + crate::kernel!( + #[inline(always)] + fn merge_swizzle( + token: Neon, + values: u8x16, + control: u8x16, + merge: u8x16, + ) -> u8x16 { + let values: uint8x16_t = values.into(); + let control: uint8x16_t = control.into(); + let merge: uint8x16_t = merge.into(); + let result = vqtbx1q_u8(merge, values, control); + u8x16 { + val: crate::support::Aligned128(result), + simd: token, + } + } + ); + merge_swizzle( + self, + Bytes::to_bytes(values), + control, + Bytes::to_bytes(merge), + ) + }) + } + #[inline(always)] + fn expand_i16x8(self, values: i16x8, mask: mask16x8) -> i16x8 { + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_16[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn expand_merge_i16x8( + self, + values: i16x8, + mask: mask16x8, + merge: i16x8, + ) -> i16x8 { + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_16[bits].0); + Bytes::from_bytes({ + crate::kernel!( + #[inline(always)] + fn merge_swizzle( + token: Neon, + values: u8x16, + control: u8x16, + merge: u8x16, + ) -> u8x16 { + let values: uint8x16_t = values.into(); + let control: uint8x16_t = control.into(); + let merge: uint8x16_t = merge.into(); + let result = vqtbx1q_u8(merge, values, control); + u8x16 { + val: crate::support::Aligned128(result), + simd: token, + } + } + ); + merge_swizzle( + self, + Bytes::to_bytes(values), + control, + Bytes::to_bytes(merge), + ) + }) + } + #[inline(always)] fn count_ones_i16x8(self, a: i16x8) -> i16x8 { crate::kernel!( #[inline(always)] @@ -2196,6 +2534,90 @@ impl Simd for Neon { }) } #[inline(always)] + fn compress_u16x8(self, values: u16x8, mask: mask16x8) -> u16x8 { + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_16[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn compress_merge_u16x8( + self, + values: u16x8, + mask: mask16x8, + merge: u16x8, + ) -> u16x8 { + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_16[bits].0); + Bytes::from_bytes({ + crate::kernel!( + #[inline(always)] + fn merge_swizzle( + token: Neon, + values: u8x16, + control: u8x16, + merge: u8x16, + ) -> u8x16 { + let values: uint8x16_t = values.into(); + let control: uint8x16_t = control.into(); + let merge: uint8x16_t = merge.into(); + let result = vqtbx1q_u8(merge, values, control); + u8x16 { + val: crate::support::Aligned128(result), + simd: token, + } + } + ); + merge_swizzle( + self, + Bytes::to_bytes(values), + control, + Bytes::to_bytes(merge), + ) + }) + } + #[inline(always)] + fn expand_u16x8(self, values: u16x8, mask: mask16x8) -> u16x8 { + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_16[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn expand_merge_u16x8( + self, + values: u16x8, + mask: mask16x8, + merge: u16x8, + ) -> u16x8 { + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_16[bits].0); + Bytes::from_bytes({ + crate::kernel!( + #[inline(always)] + fn merge_swizzle( + token: Neon, + values: u8x16, + control: u8x16, + merge: u8x16, + ) -> u8x16 { + let values: uint8x16_t = values.into(); + let control: uint8x16_t = control.into(); + let merge: uint8x16_t = merge.into(); + let result = vqtbx1q_u8(merge, values, control); + u8x16 { + val: crate::support::Aligned128(result), + simd: token, + } + } + ); + merge_swizzle( + self, + Bytes::to_bytes(values), + control, + Bytes::to_bytes(merge), + ) + }) + } + #[inline(always)] fn count_ones_u16x8(self, a: u16x8) -> u16x8 { crate::kernel!( #[inline(always)] @@ -2855,6 +3277,90 @@ impl Simd for Neon { }) } #[inline(always)] + fn compress_i32x4(self, values: i32x4, mask: mask32x4) -> i32x4 { + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn compress_merge_i32x4( + self, + values: i32x4, + mask: mask32x4, + merge: i32x4, + ) -> i32x4 { + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0); + Bytes::from_bytes({ + crate::kernel!( + #[inline(always)] + fn merge_swizzle( + token: Neon, + values: u8x16, + control: u8x16, + merge: u8x16, + ) -> u8x16 { + let values: uint8x16_t = values.into(); + let control: uint8x16_t = control.into(); + let merge: uint8x16_t = merge.into(); + let result = vqtbx1q_u8(merge, values, control); + u8x16 { + val: crate::support::Aligned128(result), + simd: token, + } + } + ); + merge_swizzle( + self, + Bytes::to_bytes(values), + control, + Bytes::to_bytes(merge), + ) + }) + } + #[inline(always)] + fn expand_i32x4(self, values: i32x4, mask: mask32x4) -> i32x4 { + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn expand_merge_i32x4( + self, + values: i32x4, + mask: mask32x4, + merge: i32x4, + ) -> i32x4 { + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0); + Bytes::from_bytes({ + crate::kernel!( + #[inline(always)] + fn merge_swizzle( + token: Neon, + values: u8x16, + control: u8x16, + merge: u8x16, + ) -> u8x16 { + let values: uint8x16_t = values.into(); + let control: uint8x16_t = control.into(); + let merge: uint8x16_t = merge.into(); + let result = vqtbx1q_u8(merge, values, control); + u8x16 { + val: crate::support::Aligned128(result), + simd: token, + } + } + ); + merge_swizzle( + self, + Bytes::to_bytes(values), + control, + Bytes::to_bytes(merge), + ) + }) + } + #[inline(always)] fn count_ones_i32x4(self, a: i32x4) -> i32x4 { crate::kernel!( #[inline(always)] @@ -3293,6 +3799,90 @@ impl Simd for Neon { }) } #[inline(always)] + fn compress_u32x4(self, values: u32x4, mask: mask32x4) -> u32x4 { + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn compress_merge_u32x4( + self, + values: u32x4, + mask: mask32x4, + merge: u32x4, + ) -> u32x4 { + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0); + Bytes::from_bytes({ + crate::kernel!( + #[inline(always)] + fn merge_swizzle( + token: Neon, + values: u8x16, + control: u8x16, + merge: u8x16, + ) -> u8x16 { + let values: uint8x16_t = values.into(); + let control: uint8x16_t = control.into(); + let merge: uint8x16_t = merge.into(); + let result = vqtbx1q_u8(merge, values, control); + u8x16 { + val: crate::support::Aligned128(result), + simd: token, + } + } + ); + merge_swizzle( + self, + Bytes::to_bytes(values), + control, + Bytes::to_bytes(merge), + ) + }) + } + #[inline(always)] + fn expand_u32x4(self, values: u32x4, mask: mask32x4) -> u32x4 { + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn expand_merge_u32x4( + self, + values: u32x4, + mask: mask32x4, + merge: u32x4, + ) -> u32x4 { + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0); + Bytes::from_bytes({ + crate::kernel!( + #[inline(always)] + fn merge_swizzle( + token: Neon, + values: u8x16, + control: u8x16, + merge: u8x16, + ) -> u8x16 { + let values: uint8x16_t = values.into(); + let control: uint8x16_t = control.into(); + let merge: uint8x16_t = merge.into(); + let result = vqtbx1q_u8(merge, values, control); + u8x16 { + val: crate::support::Aligned128(result), + simd: token, + } + } + ); + merge_swizzle( + self, + Bytes::to_bytes(values), + control, + Bytes::to_bytes(merge), + ) + }) + } + #[inline(always)] fn count_ones_u32x4(self, a: u32x4) -> u32x4 { crate::kernel!( #[inline(always)] @@ -3960,6 +4550,90 @@ impl Simd for Neon { }) } #[inline(always)] + fn compress_f64x2(self, values: f64x2, mask: mask64x2) -> f64x2 { + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn compress_merge_f64x2( + self, + values: f64x2, + mask: mask64x2, + merge: f64x2, + ) -> f64x2 { + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0); + Bytes::from_bytes({ + crate::kernel!( + #[inline(always)] + fn merge_swizzle( + token: Neon, + values: u8x16, + control: u8x16, + merge: u8x16, + ) -> u8x16 { + let values: uint8x16_t = values.into(); + let control: uint8x16_t = control.into(); + let merge: uint8x16_t = merge.into(); + let result = vqtbx1q_u8(merge, values, control); + u8x16 { + val: crate::support::Aligned128(result), + simd: token, + } + } + ); + merge_swizzle( + self, + Bytes::to_bytes(values), + control, + Bytes::to_bytes(merge), + ) + }) + } + #[inline(always)] + fn expand_f64x2(self, values: f64x2, mask: mask64x2) -> f64x2 { + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn expand_merge_f64x2( + self, + values: f64x2, + mask: mask64x2, + merge: f64x2, + ) -> f64x2 { + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0); + Bytes::from_bytes({ + crate::kernel!( + #[inline(always)] + fn merge_swizzle( + token: Neon, + values: u8x16, + control: u8x16, + merge: u8x16, + ) -> u8x16 { + let values: uint8x16_t = values.into(); + let control: uint8x16_t = control.into(); + let merge: uint8x16_t = merge.into(); + let result = vqtbx1q_u8(merge, values, control); + u8x16 { + val: crate::support::Aligned128(result), + simd: token, + } + } + ); + merge_swizzle( + self, + Bytes::to_bytes(values), + control, + Bytes::to_bytes(merge), + ) + }) + } + #[inline(always)] fn neg_f64x2(self, a: f64x2) -> f64x2 { crate::kernel!( #[inline(always)] @@ -4484,6 +5158,90 @@ impl Simd for Neon { }) } #[inline(always)] + fn compress_i64x2(self, values: i64x2, mask: mask64x2) -> i64x2 { + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn compress_merge_i64x2( + self, + values: i64x2, + mask: mask64x2, + merge: i64x2, + ) -> i64x2 { + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0); + Bytes::from_bytes({ + crate::kernel!( + #[inline(always)] + fn merge_swizzle( + token: Neon, + values: u8x16, + control: u8x16, + merge: u8x16, + ) -> u8x16 { + let values: uint8x16_t = values.into(); + let control: uint8x16_t = control.into(); + let merge: uint8x16_t = merge.into(); + let result = vqtbx1q_u8(merge, values, control); + u8x16 { + val: crate::support::Aligned128(result), + simd: token, + } + } + ); + merge_swizzle( + self, + Bytes::to_bytes(values), + control, + Bytes::to_bytes(merge), + ) + }) + } + #[inline(always)] + fn expand_i64x2(self, values: i64x2, mask: mask64x2) -> i64x2 { + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn expand_merge_i64x2( + self, + values: i64x2, + mask: mask64x2, + merge: i64x2, + ) -> i64x2 { + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0); + Bytes::from_bytes({ + crate::kernel!( + #[inline(always)] + fn merge_swizzle( + token: Neon, + values: u8x16, + control: u8x16, + merge: u8x16, + ) -> u8x16 { + let values: uint8x16_t = values.into(); + let control: uint8x16_t = control.into(); + let merge: uint8x16_t = merge.into(); + let result = vqtbx1q_u8(merge, values, control); + u8x16 { + val: crate::support::Aligned128(result), + simd: token, + } + } + ); + merge_swizzle( + self, + Bytes::to_bytes(values), + control, + Bytes::to_bytes(merge), + ) + }) + } + #[inline(always)] fn count_ones_i64x2(self, a: i64x2) -> i64x2 { crate::kernel!( #[inline(always)] @@ -4885,6 +5643,90 @@ impl Simd for Neon { }) } #[inline(always)] + fn compress_u64x2(self, values: u64x2, mask: mask64x2) -> u64x2 { + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn compress_merge_u64x2( + self, + values: u64x2, + mask: mask64x2, + merge: u64x2, + ) -> u64x2 { + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0); + Bytes::from_bytes({ + crate::kernel!( + #[inline(always)] + fn merge_swizzle( + token: Neon, + values: u8x16, + control: u8x16, + merge: u8x16, + ) -> u8x16 { + let values: uint8x16_t = values.into(); + let control: uint8x16_t = control.into(); + let merge: uint8x16_t = merge.into(); + let result = vqtbx1q_u8(merge, values, control); + u8x16 { + val: crate::support::Aligned128(result), + simd: token, + } + } + ); + merge_swizzle( + self, + Bytes::to_bytes(values), + control, + Bytes::to_bytes(merge), + ) + }) + } + #[inline(always)] + fn expand_u64x2(self, values: u64x2, mask: mask64x2) -> u64x2 { + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn expand_merge_u64x2( + self, + values: u64x2, + mask: mask64x2, + merge: u64x2, + ) -> u64x2 { + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0); + Bytes::from_bytes({ + crate::kernel!( + #[inline(always)] + fn merge_swizzle( + token: Neon, + values: u8x16, + control: u8x16, + merge: u8x16, + ) -> u8x16 { + let values: uint8x16_t = values.into(); + let control: uint8x16_t = control.into(); + let merge: uint8x16_t = merge.into(); + let result = vqtbx1q_u8(merge, values, control); + u8x16 { + val: crate::support::Aligned128(result), + simd: token, + } + } + ); + merge_swizzle( + self, + Bytes::to_bytes(values), + control, + Bytes::to_bytes(merge), + ) + }) + } + #[inline(always)] fn count_ones_u64x2(self, a: u64x2) -> u64x2 { crate::kernel!( #[inline(always)] @@ -5499,6 +6341,56 @@ impl Simd for Neon { }) } #[inline(always)] + fn compress_f32x8(self, values: f32x8, mask: mask32x8) -> f32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_f32x8( + self, + values: f32x8, + mask: mask32x8, + merge: f32x8, + ) -> f32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_f32x8(self, values: f32x8, mask: mask32x8) -> f32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_f32x8( + self, + values: f32x8, + mask: mask32x8, + merge: f32x8, + ) -> f32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn is_nan_f32x8(self, a: f32x8) -> mask32x8 { a.simd_ne(a) } @@ -5590,6 +6482,56 @@ impl Simd for Neon { }) } #[inline(always)] + fn compress_i8x32(self, values: i8x32, mask: mask8x32) -> i8x32 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i8x32( + self, + values: i8x32, + mask: mask8x32, + merge: i8x32, + ) -> i8x32 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i8x32(self, values: i8x32, mask: mask8x32) -> i8x32 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i8x32( + self, + values: i8x32, + mask: mask8x32, + merge: i8x32, + ) -> i8x32 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn combine_i8x32(self, a: i8x32, b: i8x32) -> i8x64 { i8x64 { val: crate::support::Aligned512(int8x16x4_t( @@ -5708,6 +6650,132 @@ impl Simd for Neon { kernel(self, a, b, indices) } #[inline(always)] + fn compress_u8x32(self, values: u8x32, mask: mask8x32) -> u8x32 { + let mask_bits = self.to_bitmask_mask8x32(mask); + let mut control = [u8::MAX; 32]; + let mut output_lane = 0; + for block in 0..32 / 8 { + let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; + let packed = crate::support::COMPRESS_8_CONTROLS[block_mask]; + let base = (block * 8) as u64 * 0x0101_0101_0101_0101; + let adjusted = packed | base; + let adjusted = adjusted.to_le_bytes(); + let write_len = core::cmp::min(8, 32 - output_lane); + control[output_lane..output_lane + write_len].copy_from_slice(&adjusted[..write_len]); + output_lane += crate::support::COMPACT_8_COUNTS[block_mask] as usize; + } + let control = u8x32::simd_from(self, control); + self.swizzle_dyn_precise_u8x32(values, control) + } + #[inline(always)] + fn compress_merge_u8x32( + self, + values: u8x32, + mask: mask8x32, + merge: u8x32, + ) -> u8x32 { + let mask_bits = self.to_bitmask_mask8x32(mask); + let mut control = [u8::MAX; 32]; + let mut output_lane = 0; + for block in 0..32 / 8 { + let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; + let packed = crate::support::COMPRESS_8_CONTROLS[block_mask]; + let base = (block * 8) as u64 * 0x0101_0101_0101_0101; + let adjusted = packed | base; + let adjusted = adjusted.to_le_bytes(); + let write_len = core::cmp::min(8, 32 - output_lane); + control[output_lane..output_lane + write_len].copy_from_slice(&adjusted[..write_len]); + output_lane += crate::support::COMPACT_8_COUNTS[block_mask] as usize; + } + let control = u8x32::simd_from(self, control); + { + crate::kernel!( + #[inline(always)] + fn merge_swizzle( + token: Neon, + values: u8x32, + control: u8x32, + merge: u8x32, + ) -> u8x32 { + let values: uint8x16x2_t = values.into(); + let control: uint8x16x2_t = control.into(); + let merge: uint8x16x2_t = merge.into(); + let result = uint8x16x2_t( + vqtbx2q_u8(merge.0, values, control.0), + vqtbx2q_u8(merge.1, values, control.1), + ); + u8x32 { + val: crate::support::Aligned256(result), + simd: token, + } + } + ); + merge_swizzle(self, values, control, merge) + } + } + #[inline(always)] + fn expand_u8x32(self, values: u8x32, mask: mask8x32) -> u8x32 { + let mask_bits = self.to_bitmask_mask8x32(mask); + let mut control = [u8::MAX; 32]; + let mut input_lane = 0; + for block in 0..32 / 8 { + let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; + let packed = crate::support::EXPAND_8_CONTROLS[block_mask]; + let base = input_lane as u64 * 0x0101_0101_0101_0101; + let adjusted = packed + base; + let output_lane = block * 8; + control[output_lane..output_lane + 8].copy_from_slice(&adjusted.to_le_bytes()); + input_lane += crate::support::COMPACT_8_COUNTS[block_mask] as usize; + } + let control = u8x32::simd_from(self, control); + self.swizzle_dyn_precise_u8x32(values, control) + } + #[inline(always)] + fn expand_merge_u8x32( + self, + values: u8x32, + mask: mask8x32, + merge: u8x32, + ) -> u8x32 { + let mask_bits = self.to_bitmask_mask8x32(mask); + let mut control = [u8::MAX; 32]; + let mut input_lane = 0; + for block in 0..32 / 8 { + let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; + let packed = crate::support::EXPAND_8_CONTROLS[block_mask]; + let base = input_lane as u64 * 0x0101_0101_0101_0101; + let adjusted = packed + base; + let output_lane = block * 8; + control[output_lane..output_lane + 8].copy_from_slice(&adjusted.to_le_bytes()); + input_lane += crate::support::COMPACT_8_COUNTS[block_mask] as usize; + } + let control = u8x32::simd_from(self, control); + { + crate::kernel!( + #[inline(always)] + fn merge_swizzle( + token: Neon, + values: u8x32, + control: u8x32, + merge: u8x32, + ) -> u8x32 { + let values: uint8x16x2_t = values.into(); + let control: uint8x16x2_t = control.into(); + let merge: uint8x16x2_t = merge.into(); + let result = uint8x16x2_t( + vqtbx2q_u8(merge.0, values, control.0), + vqtbx2q_u8(merge.1, values, control.1), + ); + u8x32 { + val: crate::support::Aligned256(result), + simd: token, + } + } + ); + merge_swizzle(self, values, control, merge) + } + } + #[inline(always)] fn combine_u8x32(self, a: u8x32, b: u8x32) -> u8x64 { u8x64 { val: crate::support::Aligned512(uint8x16x4_t( @@ -5852,6 +6920,56 @@ impl Simd for Neon { }) } #[inline(always)] + fn compress_i16x16(self, values: i16x16, mask: mask16x16) -> i16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i16x16( + self, + values: i16x16, + mask: mask16x16, + merge: i16x16, + ) -> i16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i16x16(self, values: i16x16, mask: mask16x16) -> i16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i16x16( + self, + values: i16x16, + mask: mask16x16, + merge: i16x16, + ) -> i16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn combine_i16x16(self, a: i16x16, b: i16x16) -> i16x32 { i16x32 { val: crate::support::Aligned512(int16x8x4_t( @@ -5911,6 +7029,56 @@ impl Simd for Neon { }) } #[inline(always)] + fn compress_u16x16(self, values: u16x16, mask: mask16x16) -> u16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u16x16( + self, + values: u16x16, + mask: mask16x16, + merge: u16x16, + ) -> u16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u16x16(self, values: u16x16, mask: mask16x16) -> u16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u16x16( + self, + values: u16x16, + mask: mask16x16, + merge: u16x16, + ) -> u16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn combine_u16x16(self, a: u16x16, b: u16x16) -> u16x32 { u16x32 { val: crate::support::Aligned512(uint16x8x4_t( @@ -6048,6 +7216,56 @@ impl Simd for Neon { }) } #[inline(always)] + fn compress_i32x8(self, values: i32x8, mask: mask32x8) -> i32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i32x8( + self, + values: i32x8, + mask: mask32x8, + merge: i32x8, + ) -> i32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i32x8(self, values: i32x8, mask: mask32x8) -> i32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i32x8( + self, + values: i32x8, + mask: mask32x8, + merge: i32x8, + ) -> i32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn combine_i32x8(self, a: i32x8, b: i32x8) -> i32x16 { i32x16 { val: crate::support::Aligned512(int32x4x4_t( @@ -6107,6 +7325,56 @@ impl Simd for Neon { }) } #[inline(always)] + fn compress_u32x8(self, values: u32x8, mask: mask32x8) -> u32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u32x8( + self, + values: u32x8, + mask: mask32x8, + merge: u32x8, + ) -> u32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u32x8(self, values: u32x8, mask: mask32x8) -> u32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u32x8( + self, + values: u32x8, + mask: mask32x8, + merge: u32x8, + ) -> u32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn combine_u32x8(self, a: u32x8, b: u32x8) -> u32x16 { u32x16 { val: crate::support::Aligned512(uint32x4x4_t( @@ -6244,6 +7512,56 @@ impl Simd for Neon { }) } #[inline(always)] + fn compress_f64x4(self, values: f64x4, mask: mask64x4) -> f64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_f64x4( + self, + values: f64x4, + mask: mask64x4, + merge: f64x4, + ) -> f64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_f64x4(self, values: f64x4, mask: mask64x4) -> f64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_f64x4( + self, + values: f64x4, + mask: mask64x4, + merge: f64x4, + ) -> f64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn is_nan_f64x4(self, a: f64x4) -> mask64x4 { a.simd_ne(a) } @@ -6335,6 +7653,56 @@ impl Simd for Neon { }) } #[inline(always)] + fn compress_i64x4(self, values: i64x4, mask: mask64x4) -> i64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i64x4( + self, + values: i64x4, + mask: mask64x4, + merge: i64x4, + ) -> i64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i64x4(self, values: i64x4, mask: mask64x4) -> i64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i64x4( + self, + values: i64x4, + mask: mask64x4, + merge: i64x4, + ) -> i64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn combine_i64x4(self, a: i64x4, b: i64x4) -> i64x8 { i64x8 { val: crate::support::Aligned512(int64x2x4_t( @@ -6394,6 +7762,56 @@ impl Simd for Neon { }) } #[inline(always)] + fn compress_u64x4(self, values: u64x4, mask: mask64x4) -> u64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u64x4( + self, + values: u64x4, + mask: mask64x4, + merge: u64x4, + ) -> u64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u64x4(self, values: u64x4, mask: mask64x4) -> u64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u64x4( + self, + values: u64x4, + mask: mask64x4, + merge: u64x4, + ) -> u64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn combine_u64x4(self, a: u64x4, b: u64x4) -> u64x8 { u64x8 { val: crate::support::Aligned512(uint64x2x4_t( @@ -6549,6 +7967,56 @@ impl Simd for Neon { }) } #[inline(always)] + fn compress_f32x16(self, values: f32x16, mask: mask32x16) -> f32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_f32x16( + self, + values: f32x16, + mask: mask32x16, + merge: f32x16, + ) -> f32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_f32x16(self, values: f32x16, mask: mask32x16) -> f32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_f32x16( + self, + values: f32x16, + mask: mask32x16, + merge: f32x16, + ) -> f32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn is_nan_f32x16(self, a: f32x16) -> mask32x16 { a.simd_ne(a) } @@ -6649,6 +8117,56 @@ impl Simd for Neon { }) } #[inline(always)] + fn compress_i8x64(self, values: i8x64, mask: mask8x64) -> i8x64 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i8x64( + self, + values: i8x64, + mask: mask8x64, + merge: i8x64, + ) -> i8x64 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i8x64(self, values: i8x64, mask: mask8x64) -> i8x64 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i8x64( + self, + values: i8x64, + mask: mask8x64, + merge: i8x64, + ) -> i8x64 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_i8x64(self, a: i8x64) -> (i8x32, i8x32) { ( i8x32 { @@ -6784,6 +8302,136 @@ impl Simd for Neon { kernel(self, a, b, indices) } #[inline(always)] + fn compress_u8x64(self, values: u8x64, mask: mask8x64) -> u8x64 { + let mask_bits = self.to_bitmask_mask8x64(mask); + let mut control = [u8::MAX; 64]; + let mut output_lane = 0; + for block in 0..64 / 8 { + let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; + let packed = crate::support::COMPRESS_8_CONTROLS[block_mask]; + let base = (block * 8) as u64 * 0x0101_0101_0101_0101; + let adjusted = packed | base; + let adjusted = adjusted.to_le_bytes(); + let write_len = core::cmp::min(8, 64 - output_lane); + control[output_lane..output_lane + write_len].copy_from_slice(&adjusted[..write_len]); + output_lane += crate::support::COMPACT_8_COUNTS[block_mask] as usize; + } + let control = u8x64::simd_from(self, control); + self.swizzle_dyn_precise_u8x64(values, control) + } + #[inline(always)] + fn compress_merge_u8x64( + self, + values: u8x64, + mask: mask8x64, + merge: u8x64, + ) -> u8x64 { + let mask_bits = self.to_bitmask_mask8x64(mask); + let mut control = [u8::MAX; 64]; + let mut output_lane = 0; + for block in 0..64 / 8 { + let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; + let packed = crate::support::COMPRESS_8_CONTROLS[block_mask]; + let base = (block * 8) as u64 * 0x0101_0101_0101_0101; + let adjusted = packed | base; + let adjusted = adjusted.to_le_bytes(); + let write_len = core::cmp::min(8, 64 - output_lane); + control[output_lane..output_lane + write_len].copy_from_slice(&adjusted[..write_len]); + output_lane += crate::support::COMPACT_8_COUNTS[block_mask] as usize; + } + let control = u8x64::simd_from(self, control); + { + crate::kernel!( + #[inline(always)] + fn merge_swizzle( + token: Neon, + values: u8x64, + control: u8x64, + merge: u8x64, + ) -> u8x64 { + let values: uint8x16x4_t = values.into(); + let control: uint8x16x4_t = control.into(); + let merge: uint8x16x4_t = merge.into(); + let result = uint8x16x4_t( + vqtbx4q_u8(merge.0, values, control.0), + vqtbx4q_u8(merge.1, values, control.1), + vqtbx4q_u8(merge.2, values, control.2), + vqtbx4q_u8(merge.3, values, control.3), + ); + u8x64 { + val: crate::support::Aligned512(result), + simd: token, + } + } + ); + merge_swizzle(self, values, control, merge) + } + } + #[inline(always)] + fn expand_u8x64(self, values: u8x64, mask: mask8x64) -> u8x64 { + let mask_bits = self.to_bitmask_mask8x64(mask); + let mut control = [u8::MAX; 64]; + let mut input_lane = 0; + for block in 0..64 / 8 { + let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; + let packed = crate::support::EXPAND_8_CONTROLS[block_mask]; + let base = input_lane as u64 * 0x0101_0101_0101_0101; + let adjusted = packed + base; + let output_lane = block * 8; + control[output_lane..output_lane + 8].copy_from_slice(&adjusted.to_le_bytes()); + input_lane += crate::support::COMPACT_8_COUNTS[block_mask] as usize; + } + let control = u8x64::simd_from(self, control); + self.swizzle_dyn_precise_u8x64(values, control) + } + #[inline(always)] + fn expand_merge_u8x64( + self, + values: u8x64, + mask: mask8x64, + merge: u8x64, + ) -> u8x64 { + let mask_bits = self.to_bitmask_mask8x64(mask); + let mut control = [u8::MAX; 64]; + let mut input_lane = 0; + for block in 0..64 / 8 { + let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; + let packed = crate::support::EXPAND_8_CONTROLS[block_mask]; + let base = input_lane as u64 * 0x0101_0101_0101_0101; + let adjusted = packed + base; + let output_lane = block * 8; + control[output_lane..output_lane + 8].copy_from_slice(&adjusted.to_le_bytes()); + input_lane += crate::support::COMPACT_8_COUNTS[block_mask] as usize; + } + let control = u8x64::simd_from(self, control); + { + crate::kernel!( + #[inline(always)] + fn merge_swizzle( + token: Neon, + values: u8x64, + control: u8x64, + merge: u8x64, + ) -> u8x64 { + let values: uint8x16x4_t = values.into(); + let control: uint8x16x4_t = control.into(); + let merge: uint8x16x4_t = merge.into(); + let result = uint8x16x4_t( + vqtbx4q_u8(merge.0, values, control.0), + vqtbx4q_u8(merge.1, values, control.1), + vqtbx4q_u8(merge.2, values, control.2), + vqtbx4q_u8(merge.3, values, control.3), + ); + u8x64 { + val: crate::support::Aligned512(result), + simd: token, + } + } + ); + merge_swizzle(self, values, control, merge) + } + } + #[inline(always)] fn split_u8x64(self, a: u8x64) -> (u8x32, u8x32) { ( u8x32 { @@ -6927,6 +8575,56 @@ impl Simd for Neon { }) } #[inline(always)] + fn compress_i16x32(self, values: i16x32, mask: mask16x32) -> i16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i16x32( + self, + values: i16x32, + mask: mask16x32, + merge: i16x32, + ) -> i16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i16x32(self, values: i16x32, mask: mask16x32) -> i16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i16x32( + self, + values: i16x32, + mask: mask16x32, + merge: i16x32, + ) -> i16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_i16x32(self, a: i16x32) -> (i16x16, i16x16) { ( i16x16 { @@ -6995,6 +8693,56 @@ impl Simd for Neon { }) } #[inline(always)] + fn compress_u16x32(self, values: u16x32, mask: mask16x32) -> u16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u16x32( + self, + values: u16x32, + mask: mask16x32, + merge: u16x32, + ) -> u16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u16x32(self, values: u16x32, mask: mask16x32) -> u16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u16x32( + self, + values: u16x32, + mask: mask16x32, + merge: u16x32, + ) -> u16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_u16x32(self, a: u16x32) -> (u16x16, u16x16) { ( u16x16 { @@ -7137,6 +8885,56 @@ impl Simd for Neon { }) } #[inline(always)] + fn compress_i32x16(self, values: i32x16, mask: mask32x16) -> i32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i32x16( + self, + values: i32x16, + mask: mask32x16, + merge: i32x16, + ) -> i32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i32x16(self, values: i32x16, mask: mask32x16) -> i32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i32x16( + self, + values: i32x16, + mask: mask32x16, + merge: i32x16, + ) -> i32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_i32x16(self, a: i32x16) -> (i32x8, i32x8) { ( i32x8 { @@ -7205,6 +9003,56 @@ impl Simd for Neon { }) } #[inline(always)] + fn compress_u32x16(self, values: u32x16, mask: mask32x16) -> u32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u32x16( + self, + values: u32x16, + mask: mask32x16, + merge: u32x16, + ) -> u32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u32x16(self, values: u32x16, mask: mask32x16) -> u32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u32x16( + self, + values: u32x16, + mask: mask32x16, + merge: u32x16, + ) -> u32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_u32x16(self, a: u32x16) -> (u32x8, u32x8) { ( u32x8 { @@ -7347,6 +9195,56 @@ impl Simd for Neon { }) } #[inline(always)] + fn compress_f64x8(self, values: f64x8, mask: mask64x8) -> f64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_f64x8( + self, + values: f64x8, + mask: mask64x8, + merge: f64x8, + ) -> f64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_f64x8(self, values: f64x8, mask: mask64x8) -> f64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_f64x8( + self, + values: f64x8, + mask: mask64x8, + merge: f64x8, + ) -> f64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn is_nan_f64x8(self, a: f64x8) -> mask64x8 { a.simd_ne(a) } @@ -7447,6 +9345,56 @@ impl Simd for Neon { }) } #[inline(always)] + fn compress_i64x8(self, values: i64x8, mask: mask64x8) -> i64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i64x8( + self, + values: i64x8, + mask: mask64x8, + merge: i64x8, + ) -> i64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i64x8(self, values: i64x8, mask: mask64x8) -> i64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i64x8( + self, + values: i64x8, + mask: mask64x8, + merge: i64x8, + ) -> i64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_i64x8(self, a: i64x8) -> (i64x4, i64x4) { ( i64x4 { @@ -7515,6 +9463,56 @@ impl Simd for Neon { }) } #[inline(always)] + fn compress_u64x8(self, values: u64x8, mask: mask64x8) -> u64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u64x8( + self, + values: u64x8, + mask: mask64x8, + merge: u64x8, + ) -> u64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u64x8(self, values: u64x8, mask: mask64x8) -> u64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u64x8( + self, + values: u64x8, + mask: mask64x8, + merge: u64x8, + ) -> u64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_u64x8(self, a: u64x8) -> (u64x4, u64x4) { ( u64x4 { diff --git a/fearless_simd/src/generated/simd_trait.rs b/fearless_simd/src/generated/simd_trait.rs index 2cb21f1c..e5d83f34 100644 --- a/fearless_simd/src/generated/simd_trait.rs +++ b/fearless_simd/src/generated/simd_trait.rs @@ -294,6 +294,24 @@ pub trait Simd: indices, )) } + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_f32x4(self, values: f32x4, mask: mask32x4) -> f32x4; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_f32x4( + self, + values: f32x4, + mask: mask32x4, + merge: f32x4, + ) -> f32x4; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_f32x4(self, values: f32x4, mask: mask32x4) -> f32x4; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_f32x4( + self, + values: f32x4, + mask: mask32x4, + merge: f32x4, + ) -> f32x4; #[doc = "Negate each element of the vector."] fn neg_f32x4(self, a: f32x4) -> f32x4; #[doc = "Compute the square root of each element.\n\nNegative elements other than `-0.0` will become NaN."] @@ -485,6 +503,24 @@ pub trait Simd: indices, )) } + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_i8x16(self, values: i8x16, mask: mask8x16) -> i8x16; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_i8x16( + self, + values: i8x16, + mask: mask8x16, + merge: i8x16, + ) -> i8x16; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_i8x16(self, values: i8x16, mask: mask8x16) -> i8x16; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_i8x16( + self, + values: i8x16, + mask: mask8x16, + merge: i8x16, + ) -> i8x16; #[doc = "Return the number of ones in the binary representation of each element."] fn count_ones_i8x16(self, a: i8x16) -> i8x16; #[doc = "Return the number of zeros in the binary representation of each element."] @@ -618,6 +654,24 @@ pub trait Simd: b: u8x16, indices: u8x16, ) -> u8x16; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_u8x16( + self, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_u8x16( + self, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16; #[doc = "Return the number of ones in the binary representation of each element."] fn count_ones_u8x16(self, a: u8x16) -> u8x16; #[doc = "Return the number of zeros in the binary representation of each element."] @@ -823,6 +877,24 @@ pub trait Simd: indices, )) } + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_i16x8(self, values: i16x8, mask: mask16x8) -> i16x8; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_i16x8( + self, + values: i16x8, + mask: mask16x8, + merge: i16x8, + ) -> i16x8; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_i16x8(self, values: i16x8, mask: mask16x8) -> i16x8; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_i16x8( + self, + values: i16x8, + mask: mask16x8, + merge: i16x8, + ) -> i16x8; #[doc = "Return the number of ones in the binary representation of each element."] fn count_ones_i16x8(self, a: i16x8) -> i16x8; #[doc = "Return the number of zeros in the binary representation of each element."] @@ -985,6 +1057,24 @@ pub trait Simd: indices, )) } + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_u16x8(self, values: u16x8, mask: mask16x8) -> u16x8; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_u16x8( + self, + values: u16x8, + mask: mask16x8, + merge: u16x8, + ) -> u16x8; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_u16x8(self, values: u16x8, mask: mask16x8) -> u16x8; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_u16x8( + self, + values: u16x8, + mask: mask16x8, + merge: u16x8, + ) -> u16x8; #[doc = "Return the number of ones in the binary representation of each element."] fn count_ones_u16x8(self, a: u16x8) -> u16x8; #[doc = "Return the number of zeros in the binary representation of each element."] @@ -1198,6 +1288,24 @@ pub trait Simd: indices, )) } + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_i32x4(self, values: i32x4, mask: mask32x4) -> i32x4; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_i32x4( + self, + values: i32x4, + mask: mask32x4, + merge: i32x4, + ) -> i32x4; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_i32x4(self, values: i32x4, mask: mask32x4) -> i32x4; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_i32x4( + self, + values: i32x4, + mask: mask32x4, + merge: i32x4, + ) -> i32x4; #[doc = "Return the number of ones in the binary representation of each element."] fn count_ones_i32x4(self, a: i32x4) -> i32x4; #[doc = "Return the number of zeros in the binary representation of each element."] @@ -1362,6 +1470,24 @@ pub trait Simd: indices, )) } + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_u32x4(self, values: u32x4, mask: mask32x4) -> u32x4; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_u32x4( + self, + values: u32x4, + mask: mask32x4, + merge: u32x4, + ) -> u32x4; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_u32x4(self, values: u32x4, mask: mask32x4) -> u32x4; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_u32x4( + self, + values: u32x4, + mask: mask32x4, + merge: u32x4, + ) -> u32x4; #[doc = "Return the number of ones in the binary representation of each element."] fn count_ones_u32x4(self, a: u32x4) -> u32x4; #[doc = "Return the number of zeros in the binary representation of each element."] @@ -1577,6 +1703,24 @@ pub trait Simd: indices, )) } + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_f64x2(self, values: f64x2, mask: mask64x2) -> f64x2; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_f64x2( + self, + values: f64x2, + mask: mask64x2, + merge: f64x2, + ) -> f64x2; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_f64x2(self, values: f64x2, mask: mask64x2) -> f64x2; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_f64x2( + self, + values: f64x2, + mask: mask64x2, + merge: f64x2, + ) -> f64x2; #[doc = "Negate each element of the vector."] fn neg_f64x2(self, a: f64x2) -> f64x2; #[doc = "Compute the square root of each element.\n\nNegative elements other than `-0.0` will become NaN."] @@ -1772,6 +1916,24 @@ pub trait Simd: indices, )) } + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_i64x2(self, values: i64x2, mask: mask64x2) -> i64x2; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_i64x2( + self, + values: i64x2, + mask: mask64x2, + merge: i64x2, + ) -> i64x2; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_i64x2(self, values: i64x2, mask: mask64x2) -> i64x2; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_i64x2( + self, + values: i64x2, + mask: mask64x2, + merge: i64x2, + ) -> i64x2; #[doc = "Return the number of ones in the binary representation of each element."] fn count_ones_i64x2(self, a: i64x2) -> i64x2; #[doc = "Return the number of zeros in the binary representation of each element."] @@ -1934,6 +2096,24 @@ pub trait Simd: indices, )) } + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_u64x2(self, values: u64x2, mask: mask64x2) -> u64x2; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_u64x2( + self, + values: u64x2, + mask: mask64x2, + merge: u64x2, + ) -> u64x2; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_u64x2(self, values: u64x2, mask: mask64x2) -> u64x2; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_u64x2( + self, + values: u64x2, + mask: mask64x2, + merge: u64x2, + ) -> u64x2; #[doc = "Return the number of ones in the binary representation of each element."] fn count_ones_u64x2(self, a: u64x2) -> u64x2; #[doc = "Return the number of zeros in the binary representation of each element."] @@ -2162,6 +2342,24 @@ pub trait Simd: indices, )) } + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_f32x8(self, values: f32x8, mask: mask32x8) -> f32x8; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_f32x8( + self, + values: f32x8, + mask: mask32x8, + merge: f32x8, + ) -> f32x8; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_f32x8(self, values: f32x8, mask: mask32x8) -> f32x8; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_f32x8( + self, + values: f32x8, + mask: mask32x8, + merge: f32x8, + ) -> f32x8; #[doc = "Negate each element of the vector."] #[inline(always)] fn neg_f32x8(self, a: f32x8) -> f32x8 { @@ -2616,6 +2814,24 @@ pub trait Simd: indices, )) } + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_i8x32(self, values: i8x32, mask: mask8x32) -> i8x32; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_i8x32( + self, + values: i8x32, + mask: mask8x32, + merge: i8x32, + ) -> i8x32; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_i8x32(self, values: i8x32, mask: mask8x32) -> i8x32; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_i8x32( + self, + values: i8x32, + mask: mask8x32, + merge: i8x32, + ) -> i8x32; #[doc = "Return the number of ones in the binary representation of each element."] #[inline(always)] fn count_ones_i8x32(self, a: i8x32) -> i8x32 { @@ -2939,6 +3155,24 @@ pub trait Simd: b: u8x32, indices: u8x32, ) -> u8x32; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_u8x32(self, values: u8x32, mask: mask8x32) -> u8x32; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_u8x32( + self, + values: u8x32, + mask: mask8x32, + merge: u8x32, + ) -> u8x32; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_u8x32(self, values: u8x32, mask: mask8x32) -> u8x32; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_u8x32( + self, + values: u8x32, + mask: mask8x32, + merge: u8x32, + ) -> u8x32; #[doc = "Return the number of ones in the binary representation of each element."] #[inline(always)] fn count_ones_u8x32(self, a: u8x32) -> u8x32 { @@ -3409,6 +3643,24 @@ pub trait Simd: indices, )) } + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_i16x16(self, values: i16x16, mask: mask16x16) -> i16x16; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_i16x16( + self, + values: i16x16, + mask: mask16x16, + merge: i16x16, + ) -> i16x16; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_i16x16(self, values: i16x16, mask: mask16x16) -> i16x16; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_i16x16( + self, + values: i16x16, + mask: mask16x16, + merge: i16x16, + ) -> i16x16; #[doc = "Return the number of ones in the binary representation of each element."] #[inline(always)] fn count_ones_i16x16(self, a: i16x16) -> i16x16 { @@ -3778,6 +4030,24 @@ pub trait Simd: indices, )) } + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_u16x16(self, values: u16x16, mask: mask16x16) -> u16x16; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_u16x16( + self, + values: u16x16, + mask: mask16x16, + merge: u16x16, + ) -> u16x16; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_u16x16(self, values: u16x16, mask: mask16x16) -> u16x16; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_u16x16( + self, + values: u16x16, + mask: mask16x16, + merge: u16x16, + ) -> u16x16; #[doc = "Return the number of ones in the binary representation of each element."] #[inline(always)] fn count_ones_u16x16(self, a: u16x16) -> u16x16 { @@ -4278,6 +4548,24 @@ pub trait Simd: indices, )) } + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_i32x8(self, values: i32x8, mask: mask32x8) -> i32x8; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_i32x8( + self, + values: i32x8, + mask: mask32x8, + merge: i32x8, + ) -> i32x8; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_i32x8(self, values: i32x8, mask: mask32x8) -> i32x8; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_i32x8( + self, + values: i32x8, + mask: mask32x8, + merge: i32x8, + ) -> i32x8; #[doc = "Return the number of ones in the binary representation of each element."] #[inline(always)] fn count_ones_i32x8(self, a: i32x8) -> i32x8 { @@ -4649,6 +4937,24 @@ pub trait Simd: indices, )) } + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_u32x8(self, values: u32x8, mask: mask32x8) -> u32x8; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_u32x8( + self, + values: u32x8, + mask: mask32x8, + merge: u32x8, + ) -> u32x8; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_u32x8(self, values: u32x8, mask: mask32x8) -> u32x8; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_u32x8( + self, + values: u32x8, + mask: mask32x8, + merge: u32x8, + ) -> u32x8; #[doc = "Return the number of ones in the binary representation of each element."] #[inline(always)] fn count_ones_u32x8(self, a: u32x8) -> u32x8 { @@ -5155,6 +5461,24 @@ pub trait Simd: indices, )) } + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_f64x4(self, values: f64x4, mask: mask64x4) -> f64x4; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_f64x4( + self, + values: f64x4, + mask: mask64x4, + merge: f64x4, + ) -> f64x4; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_f64x4(self, values: f64x4, mask: mask64x4) -> f64x4; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_f64x4( + self, + values: f64x4, + mask: mask64x4, + merge: f64x4, + ) -> f64x4; #[doc = "Negate each element of the vector."] #[inline(always)] fn neg_f64x4(self, a: f64x4) -> f64x4 { @@ -5628,6 +5952,24 @@ pub trait Simd: indices, )) } + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_i64x4(self, values: i64x4, mask: mask64x4) -> i64x4; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_i64x4( + self, + values: i64x4, + mask: mask64x4, + merge: i64x4, + ) -> i64x4; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_i64x4(self, values: i64x4, mask: mask64x4) -> i64x4; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_i64x4( + self, + values: i64x4, + mask: mask64x4, + merge: i64x4, + ) -> i64x4; #[doc = "Return the number of ones in the binary representation of each element."] #[inline(always)] fn count_ones_i64x4(self, a: i64x4) -> i64x4 { @@ -5991,6 +6333,24 @@ pub trait Simd: indices, )) } + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_u64x4(self, values: u64x4, mask: mask64x4) -> u64x4; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_u64x4( + self, + values: u64x4, + mask: mask64x4, + merge: u64x4, + ) -> u64x4; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_u64x4(self, values: u64x4, mask: mask64x4) -> u64x4; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_u64x4( + self, + values: u64x4, + mask: mask64x4, + merge: u64x4, + ) -> u64x4; #[doc = "Return the number of ones in the binary representation of each element."] #[inline(always)] fn count_ones_u64x4(self, a: u64x4) -> u64x4 { @@ -6482,6 +6842,24 @@ pub trait Simd: indices, )) } + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_f32x16(self, values: f32x16, mask: mask32x16) -> f32x16; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_f32x16( + self, + values: f32x16, + mask: mask32x16, + merge: f32x16, + ) -> f32x16; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_f32x16(self, values: f32x16, mask: mask32x16) -> f32x16; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_f32x16( + self, + values: f32x16, + mask: mask32x16, + merge: f32x16, + ) -> f32x16; #[doc = "Negate each element of the vector."] #[inline(always)] fn neg_f32x16(self, a: f32x16) -> f32x16 { @@ -6944,6 +7322,24 @@ pub trait Simd: indices, )) } + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_i8x64(self, values: i8x64, mask: mask8x64) -> i8x64; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_i8x64( + self, + values: i8x64, + mask: mask8x64, + merge: i8x64, + ) -> i8x64; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_i8x64(self, values: i8x64, mask: mask8x64) -> i8x64; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_i8x64( + self, + values: i8x64, + mask: mask8x64, + merge: i8x64, + ) -> i8x64; #[doc = "Return the number of ones in the binary representation of each element."] #[inline(always)] fn count_ones_i8x64(self, a: i8x64) -> i8x64 { @@ -7265,6 +7661,24 @@ pub trait Simd: b: u8x64, indices: u8x64, ) -> u8x64; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_u8x64(self, values: u8x64, mask: mask8x64) -> u8x64; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_u8x64( + self, + values: u8x64, + mask: mask8x64, + merge: u8x64, + ) -> u8x64; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_u8x64(self, values: u8x64, mask: mask8x64) -> u8x64; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_u8x64( + self, + values: u8x64, + mask: mask8x64, + merge: u8x64, + ) -> u8x64; #[doc = "Return the number of ones in the binary representation of each element."] #[inline(always)] fn count_ones_u8x64(self, a: u8x64) -> u8x64 { @@ -7731,6 +8145,24 @@ pub trait Simd: indices, )) } + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_i16x32(self, values: i16x32, mask: mask16x32) -> i16x32; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_i16x32( + self, + values: i16x32, + mask: mask16x32, + merge: i16x32, + ) -> i16x32; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_i16x32(self, values: i16x32, mask: mask16x32) -> i16x32; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_i16x32( + self, + values: i16x32, + mask: mask16x32, + merge: i16x32, + ) -> i16x32; #[doc = "Return the number of ones in the binary representation of each element."] #[inline(always)] fn count_ones_i16x32(self, a: i16x32) -> i16x32 { @@ -8104,6 +8536,24 @@ pub trait Simd: indices, )) } + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_u16x32(self, values: u16x32, mask: mask16x32) -> u16x32; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_u16x32( + self, + values: u16x32, + mask: mask16x32, + merge: u16x32, + ) -> u16x32; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_u16x32(self, values: u16x32, mask: mask16x32) -> u16x32; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_u16x32( + self, + values: u16x32, + mask: mask16x32, + merge: u16x32, + ) -> u16x32; #[doc = "Return the number of ones in the binary representation of each element."] #[inline(always)] fn count_ones_u16x32(self, a: u16x32) -> u16x32 { @@ -8613,6 +9063,24 @@ pub trait Simd: indices, )) } + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_i32x16(self, values: i32x16, mask: mask32x16) -> i32x16; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_i32x16( + self, + values: i32x16, + mask: mask32x16, + merge: i32x16, + ) -> i32x16; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_i32x16(self, values: i32x16, mask: mask32x16) -> i32x16; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_i32x16( + self, + values: i32x16, + mask: mask32x16, + merge: i32x16, + ) -> i32x16; #[doc = "Return the number of ones in the binary representation of each element."] #[inline(always)] fn count_ones_i32x16(self, a: i32x16) -> i32x16 { @@ -8986,6 +9454,24 @@ pub trait Simd: indices, )) } + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_u32x16(self, values: u32x16, mask: mask32x16) -> u32x16; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_u32x16( + self, + values: u32x16, + mask: mask32x16, + merge: u32x16, + ) -> u32x16; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_u32x16(self, values: u32x16, mask: mask32x16) -> u32x16; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_u32x16( + self, + values: u32x16, + mask: mask32x16, + merge: u32x16, + ) -> u32x16; #[doc = "Return the number of ones in the binary representation of each element."] #[inline(always)] fn count_ones_u32x16(self, a: u32x16) -> u32x16 { @@ -9488,6 +9974,24 @@ pub trait Simd: indices, )) } + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_f64x8(self, values: f64x8, mask: mask64x8) -> f64x8; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_f64x8( + self, + values: f64x8, + mask: mask64x8, + merge: f64x8, + ) -> f64x8; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_f64x8(self, values: f64x8, mask: mask64x8) -> f64x8; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_f64x8( + self, + values: f64x8, + mask: mask64x8, + merge: f64x8, + ) -> f64x8; #[doc = "Negate each element of the vector."] #[inline(always)] fn neg_f64x8(self, a: f64x8) -> f64x8 { @@ -9959,6 +10463,24 @@ pub trait Simd: indices, )) } + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_i64x8(self, values: i64x8, mask: mask64x8) -> i64x8; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_i64x8( + self, + values: i64x8, + mask: mask64x8, + merge: i64x8, + ) -> i64x8; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_i64x8(self, values: i64x8, mask: mask64x8) -> i64x8; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_i64x8( + self, + values: i64x8, + mask: mask64x8, + merge: i64x8, + ) -> i64x8; #[doc = "Return the number of ones in the binary representation of each element."] #[inline(always)] fn count_ones_i64x8(self, a: i64x8) -> i64x8 { @@ -10320,6 +10842,24 @@ pub trait Simd: indices, )) } + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_u64x8(self, values: u64x8, mask: mask64x8) -> u64x8; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge_u64x8( + self, + values: u64x8, + mask: mask64x8, + merge: u64x8, + ) -> u64x8; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_u64x8(self, values: u64x8, mask: mask64x8) -> u64x8; + #[doc = "Expand consecutive low elements from `values` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge_u64x8( + self, + values: u64x8, + mask: mask64x8, + merge: u64x8, + ) -> u64x8; #[doc = "Return the number of ones in the binary representation of each element."] #[inline(always)] fn count_ones_u64x8(self, a: u64x8) -> u64x8 { @@ -11305,6 +11845,14 @@ pub trait SimdBase: rhs: impl SimdInto, indices: impl SimdInto, ) -> Self; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress(self, mask: Self::Mask) -> Self; + #[doc = "Compact the elements selected by `mask` into consecutive low lanes, preserving their order.\n\nLanes above the number of selected elements retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self; + #[doc = "Expand consecutive low elements from `self` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes are zero (positive zero for floats).\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand(self, mask: Self::Mask) -> Self; + #[doc = "Expand consecutive low elements from `self` into the lanes selected by `mask`, preserving their order.\n\nUnselected lanes retain the corresponding values from `merge`.\n\nElements are moved without changing their bits, including floating-point NaN payloads and signed zeros."] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self; #[doc = "Return the maximum element in the vector. Integer vectors always return the exact maximum.\n\nFor floating-point vectors with no NaNs, this returns the true maximum. If any lane is NaN, the entire result is implementation-defined: it may be NaN or a numeric lane that is not the true maximum. See `reduce_max_precise` for a version that ignores quiet NaNs.\n\nIf the floating-point vector contains both positive zero and negative zero, either sign of zero may be returned."] fn reduce_max(self) -> Self::Element; #[doc = "Return the minimum element in the vector. Integer vectors always return the exact minimum.\n\nFor floating-point vectors with no NaNs, this returns the true minimum. If any lane is NaN, the entire result is implementation-defined: it may be NaN or a numeric lane that is not the true minimum. See `reduce_min_precise` for a version that ignores quiet NaNs.\n\nIf the floating-point vector contains both positive zero and negative zero, either sign of zero may be returned."] diff --git a/fearless_simd/src/generated/simd_types.rs b/fearless_simd/src/generated/simd_types.rs index f0fdf890..8081586a 100644 --- a/fearless_simd/src/generated/simd_types.rs +++ b/fearless_simd/src/generated/simd_types.rs @@ -191,6 +191,24 @@ impl SimdBase for f32x4 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_f32x4(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_f32x4(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_f32x4(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_f32x4(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_f32x4(self) } @@ -616,6 +634,24 @@ impl SimdBase for i8x16 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_i8x16(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_i8x16(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_i8x16(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_i8x16(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_i8x16(self) } @@ -953,6 +989,24 @@ impl SimdBase for u8x16 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_u8x16(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_u8x16(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_u8x16(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_u8x16(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_u8x16(self) } @@ -1402,6 +1456,24 @@ impl SimdBase for i16x8 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_i16x8(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_i16x8(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_i16x8(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_i16x8(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_i16x8(self) } @@ -1746,6 +1818,24 @@ impl SimdBase for u16x8 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_u16x8(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_u16x8(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_u16x8(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_u16x8(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_u16x8(self) } @@ -2205,6 +2295,24 @@ impl SimdBase for i32x4 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_i32x4(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_i32x4(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_i32x4(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_i32x4(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_i32x4(self) } @@ -2549,6 +2657,24 @@ impl SimdBase for u32x4 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_u32x4(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_u32x4(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_u32x4(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_u32x4(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_u32x4(self) } @@ -3020,6 +3146,24 @@ impl SimdBase for f64x2 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_f64x2(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_f64x2(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_f64x2(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_f64x2(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_f64x2(self) } @@ -3433,6 +3577,24 @@ impl SimdBase for i64x2 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_i64x2(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_i64x2(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_i64x2(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_i64x2(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_i64x2(self) } @@ -3770,6 +3932,24 @@ impl SimdBase for u64x2 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_u64x2(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_u64x2(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_u64x2(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_u64x2(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_u64x2(self) } @@ -4239,6 +4419,24 @@ impl SimdBase for f32x8 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_f32x8(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_f32x8(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_f32x8(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_f32x8(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_f32x8(self) } @@ -4675,6 +4873,24 @@ impl SimdBase for i8x32 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_i8x32(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_i8x32(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_i8x32(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_i8x32(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_i8x32(self) } @@ -5023,6 +5239,24 @@ impl SimdBase for u8x32 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_u8x32(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_u8x32(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_u8x32(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_u8x32(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_u8x32(self) } @@ -5475,6 +5709,24 @@ impl SimdBase for i16x16 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_i16x16(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_i16x16(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_i16x16(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_i16x16(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_i16x16(self) } @@ -5823,6 +6075,24 @@ impl SimdBase for u16x16 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_u16x16(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_u16x16(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_u16x16(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_u16x16(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_u16x16(self) } @@ -6290,6 +6560,24 @@ impl SimdBase for i32x8 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_i32x8(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_i32x8(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_i32x8(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_i32x8(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_i32x8(self) } @@ -6641,6 +6929,24 @@ impl SimdBase for u32x8 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_u32x8(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_u32x8(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_u32x8(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_u32x8(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_u32x8(self) } @@ -7107,6 +7413,24 @@ impl SimdBase for f64x4 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_f64x4(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_f64x4(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_f64x4(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_f64x4(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_f64x4(self) } @@ -7515,6 +7839,24 @@ impl SimdBase for i64x4 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_i64x4(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_i64x4(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_i64x4(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_i64x4(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_i64x4(self) } @@ -7847,6 +8189,24 @@ impl SimdBase for u64x4 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_u64x4(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_u64x4(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_u64x4(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_u64x4(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_u64x4(self) } @@ -8320,6 +8680,24 @@ impl SimdBase for f32x16 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_f32x16(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_f32x16(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_f32x16(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_f32x16(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_f32x16(self) } @@ -8783,6 +9161,24 @@ impl SimdBase for i8x64 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_i8x64(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_i8x64(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_i8x64(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_i8x64(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_i8x64(self) } @@ -9157,6 +9553,24 @@ impl SimdBase for u8x64 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_u8x64(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_u8x64(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_u8x64(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_u8x64(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_u8x64(self) } @@ -9619,6 +10033,24 @@ impl SimdBase for i16x32 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_i16x32(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_i16x32(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_i16x32(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_i16x32(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_i16x32(self) } @@ -9977,6 +10409,24 @@ impl SimdBase for u16x32 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_u16x32(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_u16x32(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_u16x32(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_u16x32(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_u16x32(self) } @@ -10446,6 +10896,24 @@ impl SimdBase for i32x16 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_i32x16(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_i32x16(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_i32x16(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_i32x16(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_i32x16(self) } @@ -10800,6 +11268,24 @@ impl SimdBase for u32x16 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_u32x16(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_u32x16(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_u32x16(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_u32x16(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_u32x16(self) } @@ -11273,6 +11759,24 @@ impl SimdBase for f64x8 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_f64x8(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_f64x8(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_f64x8(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_f64x8(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_f64x8(self) } @@ -11687,6 +12191,24 @@ impl SimdBase for i64x8 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_i64x8(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_i64x8(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_i64x8(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_i64x8(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_i64x8(self) } @@ -12025,6 +12547,24 @@ impl SimdBase for u64x8 { ) } #[inline(always)] + fn compress(self, mask: Self::Mask) -> Self { + self.simd.compress_u64x8(self, mask) + } + #[inline(always)] + fn compress_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .compress_merge_u64x8(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] + fn expand(self, mask: Self::Mask) -> Self { + self.simd.expand_u64x8(self, mask) + } + #[inline(always)] + fn expand_merge(self, mask: Self::Mask, merge: impl SimdInto) -> Self { + self.simd + .expand_merge_u64x8(self, mask, merge.simd_into(self.simd)) + } + #[inline(always)] fn reduce_max(self) -> Self::Element { self.simd.reduce_max_u64x8(self) } diff --git a/fearless_simd/src/generated/sse2.rs b/fearless_simd/src/generated/sse2.rs index efd1ba5d..91803e78 100644 --- a/fearless_simd/src/generated/sse2.rs +++ b/fearless_simd/src/generated/sse2.rs @@ -168,6 +168,56 @@ impl Simd for Sse2 { }) } #[inline(always)] + fn compress_f32x4(self, values: f32x4, mask: mask32x4) -> f32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_f32x4( + self, + values: f32x4, + mask: mask32x4, + merge: f32x4, + ) -> f32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_f32x4(self, values: f32x4, mask: mask32x4) -> f32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_f32x4( + self, + values: f32x4, + mask: mask32x4, + merge: f32x4, + ) -> f32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn neg_f32x4(self, a: f32x4) -> f32x4 { crate::kernel!( #[inline(always)] @@ -857,6 +907,56 @@ impl Simd for Sse2 { }) } #[inline(always)] + fn compress_i8x16(self, values: i8x16, mask: mask8x16) -> i8x16 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i8x16( + self, + values: i8x16, + mask: mask8x16, + merge: i8x16, + ) -> i8x16 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i8x16(self, values: i8x16, mask: mask8x16) -> i8x16 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i8x16( + self, + values: i8x16, + mask: mask8x16, + merge: i8x16, + ) -> i8x16 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_i8x16(self, a: i8x16) -> i8x16 { [ i8::try_from(a[0usize].count_ones()).unwrap(), @@ -1698,6 +1798,53 @@ impl Simd for Sse2 { Bytes::from_bytes(result) } #[inline(always)] + fn compress_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16 { + self.compress_merge_u8x16(values, mask, u8x16::splat(self, 0)) + } + #[inline(always)] + fn compress_merge_u8x16( + self, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + let mask = self.to_bitmask_mask8x16(mask); + let mut merge_padded = [0u8; 16 + 1]; + merge_padded[..16].copy_from_slice(&*merge); + let mut compacted = merge_padded; + let mut output_lane = 0; + for input_lane in 0..16 { + compacted[output_lane] = values[input_lane]; + output_lane += ((mask >> input_lane) & 1) as usize; + } + compacted[output_lane] = merge_padded[output_lane]; + let mut result = merge; + result.copy_from_slice(&compacted[..16]); + result + } + #[inline(always)] + fn expand_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16 { + self.expand_merge_u8x16(values, mask, u8x16::splat(self, 0)) + } + #[inline(always)] + fn expand_merge_u8x16( + self, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + let mask = self.to_bitmask_mask8x16(mask); + let mut result = merge; + let mut input_lane = 0; + for output_lane in 0..16 { + let bit = ((mask >> output_lane) & 1) as u8; + let keep = 0u8.wrapping_sub(bit); + result[output_lane] = (values[input_lane] & keep) | (result[output_lane] & !keep); + input_lane += usize::from(bit); + } + result + } + #[inline(always)] fn count_ones_u8x16(self, a: u8x16) -> u8x16 { [ u8::try_from(a[0usize].count_ones()).unwrap(), @@ -2567,6 +2714,56 @@ impl Simd for Sse2 { }) } #[inline(always)] + fn compress_i16x8(self, values: i16x8, mask: mask16x8) -> i16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i16x8( + self, + values: i16x8, + mask: mask16x8, + merge: i16x8, + ) -> i16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i16x8(self, values: i16x8, mask: mask16x8) -> i16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i16x8( + self, + values: i16x8, + mask: mask16x8, + merge: i16x8, + ) -> i16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_i16x8(self, a: i16x8) -> i16x8 { [ i16::try_from(a[0usize].count_ones()).unwrap(), @@ -3100,6 +3297,56 @@ impl Simd for Sse2 { }) } #[inline(always)] + fn compress_u16x8(self, values: u16x8, mask: mask16x8) -> u16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u16x8( + self, + values: u16x8, + mask: mask16x8, + merge: u16x8, + ) -> u16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u16x8(self, values: u16x8, mask: mask16x8) -> u16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u16x8( + self, + values: u16x8, + mask: mask16x8, + merge: u16x8, + ) -> u16x8 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_u16x8(self, a: u16x8) -> u16x8 { [ u16::try_from(a[0usize].count_ones()).unwrap(), @@ -3974,6 +4221,56 @@ impl Simd for Sse2 { }) } #[inline(always)] + fn compress_i32x4(self, values: i32x4, mask: mask32x4) -> i32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i32x4( + self, + values: i32x4, + mask: mask32x4, + merge: i32x4, + ) -> i32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i32x4(self, values: i32x4, mask: mask32x4) -> i32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i32x4( + self, + values: i32x4, + mask: mask32x4, + merge: i32x4, + ) -> i32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_i32x4(self, a: i32x4) -> i32x4 { [ a[0usize].count_ones().cast_signed(), @@ -4532,6 +4829,56 @@ impl Simd for Sse2 { }) } #[inline(always)] + fn compress_u32x4(self, values: u32x4, mask: mask32x4) -> u32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u32x4( + self, + values: u32x4, + mask: mask32x4, + merge: u32x4, + ) -> u32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u32x4(self, values: u32x4, mask: mask32x4) -> u32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u32x4( + self, + values: u32x4, + mask: mask32x4, + merge: u32x4, + ) -> u32x4 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_u32x4(self, a: u32x4) -> u32x4 { [ a[0usize].count_ones(), @@ -5366,6 +5713,56 @@ impl Simd for Sse2 { }) } #[inline(always)] + fn compress_f64x2(self, values: f64x2, mask: mask64x2) -> f64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_f64x2( + self, + values: f64x2, + mask: mask64x2, + merge: f64x2, + ) -> f64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_f64x2(self, values: f64x2, mask: mask64x2) -> f64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_f64x2( + self, + values: f64x2, + mask: mask64x2, + merge: f64x2, + ) -> f64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn neg_f64x2(self, a: f64x2) -> f64x2 { crate::kernel!( #[inline(always)] @@ -5940,6 +6337,56 @@ impl Simd for Sse2 { }) } #[inline(always)] + fn compress_i64x2(self, values: i64x2, mask: mask64x2) -> i64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i64x2( + self, + values: i64x2, + mask: mask64x2, + merge: i64x2, + ) -> i64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i64x2(self, values: i64x2, mask: mask64x2) -> i64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i64x2( + self, + values: i64x2, + mask: mask64x2, + merge: i64x2, + ) -> i64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_i64x2(self, a: i64x2) -> i64x2 { [ i64::from(a[0usize].count_ones()), @@ -6393,6 +6840,56 @@ impl Simd for Sse2 { }) } #[inline(always)] + fn compress_u64x2(self, values: u64x2, mask: mask64x2) -> u64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u64x2( + self, + values: u64x2, + mask: mask64x2, + merge: u64x2, + ) -> u64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u64x2(self, values: u64x2, mask: mask64x2) -> u64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u64x2( + self, + values: u64x2, + mask: mask64x2, + merge: u64x2, + ) -> u64x2 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_u64x2(self, a: u64x2) -> u64x2 { [ u64::from(a[0usize].count_ones()), @@ -7034,6 +7531,56 @@ impl Simd for Sse2 { }) } #[inline(always)] + fn compress_f32x8(self, values: f32x8, mask: mask32x8) -> f32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_f32x8( + self, + values: f32x8, + mask: mask32x8, + merge: f32x8, + ) -> f32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_f32x8(self, values: f32x8, mask: mask32x8) -> f32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_f32x8( + self, + values: f32x8, + mask: mask32x8, + merge: f32x8, + ) -> f32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn simd_ne_f32x8(self, a: f32x8, b: f32x8) -> mask32x8 { let (a0, a1) = self.split_f32x8(a); let (b0, b1) = self.split_f32x8(b); @@ -7108,6 +7655,56 @@ impl Simd for Sse2 { }) } #[inline(always)] + fn compress_i8x32(self, values: i8x32, mask: mask8x32) -> i8x32 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i8x32( + self, + values: i8x32, + mask: mask8x32, + merge: i8x32, + ) -> i8x32 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i8x32(self, values: i8x32, mask: mask8x32) -> i8x32 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i8x32( + self, + values: i8x32, + mask: mask8x32, + merge: i8x32, + ) -> i8x32 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn combine_i8x32(self, a: i8x32, b: i8x32) -> i8x64 { i8x64 { val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), @@ -7211,6 +7808,53 @@ impl Simd for Sse2 { Bytes::from_bytes(result) } #[inline(always)] + fn compress_u8x32(self, values: u8x32, mask: mask8x32) -> u8x32 { + self.compress_merge_u8x32(values, mask, u8x32::splat(self, 0)) + } + #[inline(always)] + fn compress_merge_u8x32( + self, + values: u8x32, + mask: mask8x32, + merge: u8x32, + ) -> u8x32 { + let mask = self.to_bitmask_mask8x32(mask); + let mut merge_padded = [0u8; 32 + 1]; + merge_padded[..32].copy_from_slice(&*merge); + let mut compacted = merge_padded; + let mut output_lane = 0; + for input_lane in 0..32 { + compacted[output_lane] = values[input_lane]; + output_lane += ((mask >> input_lane) & 1) as usize; + } + compacted[output_lane] = merge_padded[output_lane]; + let mut result = merge; + result.copy_from_slice(&compacted[..32]); + result + } + #[inline(always)] + fn expand_u8x32(self, values: u8x32, mask: mask8x32) -> u8x32 { + self.expand_merge_u8x32(values, mask, u8x32::splat(self, 0)) + } + #[inline(always)] + fn expand_merge_u8x32( + self, + values: u8x32, + mask: mask8x32, + merge: u8x32, + ) -> u8x32 { + let mask = self.to_bitmask_mask8x32(mask); + let mut result = merge; + let mut input_lane = 0; + for output_lane in 0..32 { + let bit = ((mask >> output_lane) & 1) as u8; + let keep = 0u8.wrapping_sub(bit); + result[output_lane] = (values[input_lane] & keep) | (result[output_lane] & !keep); + input_lane += usize::from(bit); + } + result + } + #[inline(always)] fn combine_u8x32(self, a: u8x32, b: u8x32) -> u8x64 { u8x64 { val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), @@ -7308,6 +7952,56 @@ impl Simd for Sse2 { }) } #[inline(always)] + fn compress_i16x16(self, values: i16x16, mask: mask16x16) -> i16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i16x16( + self, + values: i16x16, + mask: mask16x16, + merge: i16x16, + ) -> i16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i16x16(self, values: i16x16, mask: mask16x16) -> i16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i16x16( + self, + values: i16x16, + mask: mask16x16, + merge: i16x16, + ) -> i16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn combine_i16x16(self, a: i16x16, b: i16x16) -> i16x32 { i16x32 { val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), @@ -7344,20 +8038,70 @@ impl Simd for Sse2 { }) } #[inline(always)] - fn combine_u16x16(self, a: u16x16, b: u16x16) -> u16x32 { - u16x32 { - val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), + fn compress_u16x16(self, values: u16x16, mask: mask16x16) -> u16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), simd: self, - } + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) } #[inline(always)] - fn split_u16x16(self, a: u16x16) -> (u16x8, u16x8) { - ( - u16x8 { - val: crate::support::Aligned128(a.val.0[0]), - simd: self, - }, - u16x8 { + fn compress_merge_u16x16( + self, + values: u16x16, + mask: mask16x16, + merge: u16x16, + ) -> u16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u16x16(self, values: u16x16, mask: mask16x16) -> u16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u16x16( + self, + values: u16x16, + mask: mask16x16, + merge: u16x16, + ) -> u16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn combine_u16x16(self, a: u16x16, b: u16x16) -> u16x32 { + u16x32 { + val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), + simd: self, + } + } + #[inline(always)] + fn split_u16x16(self, a: u16x16) -> (u16x8, u16x8) { + ( + u16x8 { + val: crate::support::Aligned128(a.val.0[0]), + simd: self, + }, + u16x8 { val: crate::support::Aligned128(a.val.0[1]), simd: self, }, @@ -7454,6 +8198,56 @@ impl Simd for Sse2 { }) } #[inline(always)] + fn compress_i32x8(self, values: i32x8, mask: mask32x8) -> i32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i32x8( + self, + values: i32x8, + mask: mask32x8, + merge: i32x8, + ) -> i32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i32x8(self, values: i32x8, mask: mask32x8) -> i32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i32x8( + self, + values: i32x8, + mask: mask32x8, + merge: i32x8, + ) -> i32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn combine_i32x8(self, a: i32x8, b: i32x8) -> i32x16 { i32x16 { val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), @@ -7490,6 +8284,56 @@ impl Simd for Sse2 { }) } #[inline(always)] + fn compress_u32x8(self, values: u32x8, mask: mask32x8) -> u32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u32x8( + self, + values: u32x8, + mask: mask32x8, + merge: u32x8, + ) -> u32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u32x8(self, values: u32x8, mask: mask32x8) -> u32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u32x8( + self, + values: u32x8, + mask: mask32x8, + merge: u32x8, + ) -> u32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn combine_u32x8(self, a: u32x8, b: u32x8) -> u32x16 { u32x16 { val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), @@ -7587,6 +8431,56 @@ impl Simd for Sse2 { }) } #[inline(always)] + fn compress_f64x4(self, values: f64x4, mask: mask64x4) -> f64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_f64x4( + self, + values: f64x4, + mask: mask64x4, + merge: f64x4, + ) -> f64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_f64x4(self, values: f64x4, mask: mask64x4) -> f64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_f64x4( + self, + values: f64x4, + mask: mask64x4, + merge: f64x4, + ) -> f64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn simd_ne_f64x4(self, a: f64x4, b: f64x4) -> mask64x4 { let (a0, a1) = self.split_f64x4(a); let (b0, b1) = self.split_f64x4(b); @@ -7661,6 +8555,56 @@ impl Simd for Sse2 { }) } #[inline(always)] + fn compress_i64x4(self, values: i64x4, mask: mask64x4) -> i64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i64x4( + self, + values: i64x4, + mask: mask64x4, + merge: i64x4, + ) -> i64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i64x4(self, values: i64x4, mask: mask64x4) -> i64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i64x4( + self, + values: i64x4, + mask: mask64x4, + merge: i64x4, + ) -> i64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn combine_i64x4(self, a: i64x4, b: i64x4) -> i64x8 { i64x8 { val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), @@ -7697,6 +8641,56 @@ impl Simd for Sse2 { }) } #[inline(always)] + fn compress_u64x4(self, values: u64x4, mask: mask64x4) -> u64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u64x4( + self, + values: u64x4, + mask: mask64x4, + merge: u64x4, + ) -> u64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u64x4(self, values: u64x4, mask: mask64x4) -> u64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u64x4( + self, + values: u64x4, + mask: mask64x4, + merge: u64x4, + ) -> u64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn combine_u64x4(self, a: u64x4, b: u64x4) -> u64x8 { u64x8 { val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), @@ -7794,6 +8788,56 @@ impl Simd for Sse2 { }) } #[inline(always)] + fn compress_f32x16(self, values: f32x16, mask: mask32x16) -> f32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_f32x16( + self, + values: f32x16, + mask: mask32x16, + merge: f32x16, + ) -> f32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_f32x16(self, values: f32x16, mask: mask32x16) -> f32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_f32x16( + self, + values: f32x16, + mask: mask32x16, + merge: f32x16, + ) -> f32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn simd_ne_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { let (a0, a1) = self.split_f32x16(a); let (b0, b1) = self.split_f32x16(b); @@ -7861,6 +8905,56 @@ impl Simd for Sse2 { }) } #[inline(always)] + fn compress_i8x64(self, values: i8x64, mask: mask8x64) -> i8x64 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i8x64( + self, + values: i8x64, + mask: mask8x64, + merge: i8x64, + ) -> i8x64 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i8x64(self, values: i8x64, mask: mask8x64) -> i8x64 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i8x64( + self, + values: i8x64, + mask: mask8x64, + merge: i8x64, + ) -> i8x64 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_i8x64(self, a: i8x64) -> (i8x32, i8x32) { ( i8x32 { @@ -7957,6 +9051,53 @@ impl Simd for Sse2 { Bytes::from_bytes(result) } #[inline(always)] + fn compress_u8x64(self, values: u8x64, mask: mask8x64) -> u8x64 { + self.compress_merge_u8x64(values, mask, u8x64::splat(self, 0)) + } + #[inline(always)] + fn compress_merge_u8x64( + self, + values: u8x64, + mask: mask8x64, + merge: u8x64, + ) -> u8x64 { + let mask = self.to_bitmask_mask8x64(mask); + let mut merge_padded = [0u8; 64 + 1]; + merge_padded[..64].copy_from_slice(&*merge); + let mut compacted = merge_padded; + let mut output_lane = 0; + for input_lane in 0..64 { + compacted[output_lane] = values[input_lane]; + output_lane += ((mask >> input_lane) & 1) as usize; + } + compacted[output_lane] = merge_padded[output_lane]; + let mut result = merge; + result.copy_from_slice(&compacted[..64]); + result + } + #[inline(always)] + fn expand_u8x64(self, values: u8x64, mask: mask8x64) -> u8x64 { + self.expand_merge_u8x64(values, mask, u8x64::splat(self, 0)) + } + #[inline(always)] + fn expand_merge_u8x64( + self, + values: u8x64, + mask: mask8x64, + merge: u8x64, + ) -> u8x64 { + let mask = self.to_bitmask_mask8x64(mask); + let mut result = merge; + let mut input_lane = 0; + for output_lane in 0..64 { + let bit = ((mask >> output_lane) & 1) as u8; + let keep = 0u8.wrapping_sub(bit); + result[output_lane] = (values[input_lane] & keep) | (result[output_lane] & !keep); + input_lane += usize::from(bit); + } + result + } + #[inline(always)] fn split_u8x64(self, a: u8x64) -> (u8x32, u8x32) { ( u8x32 { @@ -8040,6 +9181,56 @@ impl Simd for Sse2 { }) } #[inline(always)] + fn compress_i16x32(self, values: i16x32, mask: mask16x32) -> i16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i16x32( + self, + values: i16x32, + mask: mask16x32, + merge: i16x32, + ) -> i16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i16x32(self, values: i16x32, mask: mask16x32) -> i16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i16x32( + self, + values: i16x32, + mask: mask16x32, + merge: i16x32, + ) -> i16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_i16x32(self, a: i16x32) -> (i16x16, i16x16) { ( i16x16 { @@ -8069,6 +9260,56 @@ impl Simd for Sse2 { }) } #[inline(always)] + fn compress_u16x32(self, values: u16x32, mask: mask16x32) -> u16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u16x32( + self, + values: u16x32, + mask: mask16x32, + merge: u16x32, + ) -> u16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u16x32(self, values: u16x32, mask: mask16x32) -> u16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u16x32( + self, + values: u16x32, + mask: mask16x32, + merge: u16x32, + ) -> u16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_u16x32(self, a: u16x32) -> (u16x16, u16x16) { ( u16x16 { @@ -8168,6 +9409,56 @@ impl Simd for Sse2 { }) } #[inline(always)] + fn compress_i32x16(self, values: i32x16, mask: mask32x16) -> i32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i32x16( + self, + values: i32x16, + mask: mask32x16, + merge: i32x16, + ) -> i32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i32x16(self, values: i32x16, mask: mask32x16) -> i32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i32x16( + self, + values: i32x16, + mask: mask32x16, + merge: i32x16, + ) -> i32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_i32x16(self, a: i32x16) -> (i32x8, i32x8) { ( i32x8 { @@ -8197,6 +9488,56 @@ impl Simd for Sse2 { }) } #[inline(always)] + fn compress_u32x16(self, values: u32x16, mask: mask32x16) -> u32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u32x16( + self, + values: u32x16, + mask: mask32x16, + merge: u32x16, + ) -> u32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u32x16(self, values: u32x16, mask: mask32x16) -> u32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u32x16( + self, + values: u32x16, + mask: mask32x16, + merge: u32x16, + ) -> u32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_u32x16(self, a: u32x16) -> (u32x8, u32x8) { ( u32x8 { @@ -8280,6 +9621,56 @@ impl Simd for Sse2 { }) } #[inline(always)] + fn compress_f64x8(self, values: f64x8, mask: mask64x8) -> f64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_f64x8( + self, + values: f64x8, + mask: mask64x8, + merge: f64x8, + ) -> f64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_f64x8(self, values: f64x8, mask: mask64x8) -> f64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_f64x8( + self, + values: f64x8, + mask: mask64x8, + merge: f64x8, + ) -> f64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn simd_ne_f64x8(self, a: f64x8, b: f64x8) -> mask64x8 { let (a0, a1) = self.split_f64x8(a); let (b0, b1) = self.split_f64x8(b); @@ -8347,6 +9738,56 @@ impl Simd for Sse2 { }) } #[inline(always)] + fn compress_i64x8(self, values: i64x8, mask: mask64x8) -> i64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i64x8( + self, + values: i64x8, + mask: mask64x8, + merge: i64x8, + ) -> i64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i64x8(self, values: i64x8, mask: mask64x8) -> i64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i64x8( + self, + values: i64x8, + mask: mask64x8, + merge: i64x8, + ) -> i64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_i64x8(self, a: i64x8) -> (i64x4, i64x4) { ( i64x4 { @@ -8376,6 +9817,56 @@ impl Simd for Sse2 { }) } #[inline(always)] + fn compress_u64x8(self, values: u64x8, mask: mask64x8) -> u64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u64x8( + self, + values: u64x8, + mask: mask64x8, + merge: u64x8, + ) -> u64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u64x8(self, values: u64x8, mask: mask64x8) -> u64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u64x8( + self, + values: u64x8, + mask: mask64x8, + merge: u64x8, + ) -> u64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_u64x8(self, a: u64x8) -> (u64x4, u64x4) { ( u64x4 { diff --git a/fearless_simd/src/generated/sse4_2.rs b/fearless_simd/src/generated/sse4_2.rs index d220cdad..3e4c7577 100644 --- a/fearless_simd/src/generated/sse4_2.rs +++ b/fearless_simd/src/generated/sse4_2.rs @@ -170,6 +170,43 @@ impl Simd for Sse4_2 { }) } #[inline(always)] + fn compress_f32x4(self, values: f32x4, mask: mask32x4) -> f32x4 { + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn compress_merge_f32x4( + self, + values: f32x4, + mask: mask32x4, + merge: f32x4, + ) -> f32x4 { + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0); + let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control); + let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0)); + Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed)) + } + #[inline(always)] + fn expand_f32x4(self, values: f32x4, mask: mask32x4) -> f32x4 { + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn expand_merge_f32x4( + self, + values: f32x4, + mask: mask32x4, + merge: f32x4, + ) -> f32x4 { + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0); + let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control)); + self.select_f32x4(mask, expanded, merge) + } + #[inline(always)] fn neg_f32x4(self, a: f32x4) -> f32x4 { crate::kernel!( #[inline(always)] @@ -820,6 +857,56 @@ impl Simd for Sse4_2 { }) } #[inline(always)] + fn compress_i8x16(self, values: i8x16, mask: mask8x16) -> i8x16 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i8x16( + self, + values: i8x16, + mask: mask8x16, + merge: i8x16, + ) -> i8x16 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i8x16(self, values: i8x16, mask: mask8x16) -> i8x16 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i8x16( + self, + values: i8x16, + mask: mask8x16, + merge: i8x16, + ) -> i8x16 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_i8x16(self, a: i8x16) -> i8x16 { crate::kernel!( #[inline(always)] @@ -1456,6 +1543,122 @@ impl Simd for Sse4_2 { Bytes::from_bytes(result_bytes) } #[inline(always)] + fn compress_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Sse4_2, + values: u8x16, + mask: mask8x16, + ) -> u8x16 { + let mask_bits = token.to_bitmask_mask8x16(mask) as u16; + let low_mask = (usize::from(mask_bits)) & 0xff; + let high_mask = (usize::from(mask_bits)) >> 8; + let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask]; + let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask]; + let high_control = high_control | 0x0808_0808_0808_0808; + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let low_count = low_mask.count_ones() as usize; + let compacted = _mm_shuffle_epi8(values.into(), control); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count]; + let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let compressed = _mm_shuffle_epi8(compacted, splice); + compressed.simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_u8x16( + self, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Sse4_2, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + let mask_bits = token.to_bitmask_mask8x16(mask) as u16; + let low_mask = (usize::from(mask_bits)) & 0xff; + let high_mask = (usize::from(mask_bits)) >> 8; + let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask]; + let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask]; + let high_control = high_control | 0x0808_0808_0808_0808; + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let low_count = low_mask.count_ones() as usize; + let compacted = _mm_shuffle_epi8(values.into(), control); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count]; + let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let compressed = _mm_shuffle_epi8(compacted, splice); + let count = mask_bits.count_ones() as usize; + let [prefix, _] = crate::transmute::checked_transmute_copy::<_, [__m128i; 2]>( + &crate::support::COMPACT_PREFIX_MASKS[count], + ); + _mm_blendv_epi8(merge.into(), compressed, prefix).simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Sse4_2, + values: u8x16, + mask: mask8x16, + ) -> u8x16 { + let mask_bits = token.to_bitmask_mask8x16(mask) as u16; + let low_mask = (usize::from(mask_bits)) & 0xff; + let high_mask = (usize::from(mask_bits)) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = high_control + high_base; + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let expanded = _mm_shuffle_epi8(values.into(), control); + expanded.simd_into(token) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_u8x16( + self, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Sse4_2, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + let mask_bits = token.to_bitmask_mask8x16(mask) as u16; + let low_mask = (usize::from(mask_bits)) & 0xff; + let high_mask = (usize::from(mask_bits)) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = high_control + high_base; + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let expanded = _mm_shuffle_epi8(values.into(), control); + _mm_blendv_epi8(merge.into(), expanded, mask.into()).simd_into(token) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn count_ones_u8x16(self, a: u8x16) -> u8x16 { crate::kernel!( #[inline(always)] @@ -2241,6 +2444,43 @@ impl Simd for Sse4_2 { }) } #[inline(always)] + fn compress_i16x8(self, values: i16x8, mask: mask16x8) -> i16x8 { + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_16[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn compress_merge_i16x8( + self, + values: i16x8, + mask: mask16x8, + merge: i16x8, + ) -> i16x8 { + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_16[bits].0); + let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control); + let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0)); + Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed)) + } + #[inline(always)] + fn expand_i16x8(self, values: i16x8, mask: mask16x8) -> i16x8 { + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_16[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn expand_merge_i16x8( + self, + values: i16x8, + mask: mask16x8, + merge: i16x8, + ) -> i16x8 { + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_16[bits].0); + let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control)); + self.select_i16x8(mask, expanded, merge) + } + #[inline(always)] fn count_ones_i16x8(self, a: i16x8) -> i16x8 { crate::kernel!( #[inline(always)] @@ -2790,6 +3030,43 @@ impl Simd for Sse4_2 { }) } #[inline(always)] + fn compress_u16x8(self, values: u16x8, mask: mask16x8) -> u16x8 { + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_16[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn compress_merge_u16x8( + self, + values: u16x8, + mask: mask16x8, + merge: u16x8, + ) -> u16x8 { + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_16[bits].0); + let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control); + let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0)); + Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed)) + } + #[inline(always)] + fn expand_u16x8(self, values: u16x8, mask: mask16x8) -> u16x8 { + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_16[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn expand_merge_u16x8( + self, + values: u16x8, + mask: mask16x8, + merge: u16x8, + ) -> u16x8 { + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_16[bits].0); + let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control)); + self.select_u16x8(mask, expanded, merge) + } + #[inline(always)] fn count_ones_u16x8(self, a: u16x8) -> u16x8 { crate::kernel!( #[inline(always)] @@ -3577,6 +3854,43 @@ impl Simd for Sse4_2 { }) } #[inline(always)] + fn compress_i32x4(self, values: i32x4, mask: mask32x4) -> i32x4 { + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn compress_merge_i32x4( + self, + values: i32x4, + mask: mask32x4, + merge: i32x4, + ) -> i32x4 { + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0); + let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control); + let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0)); + Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed)) + } + #[inline(always)] + fn expand_i32x4(self, values: i32x4, mask: mask32x4) -> i32x4 { + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn expand_merge_i32x4( + self, + values: i32x4, + mask: mask32x4, + merge: i32x4, + ) -> i32x4 { + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0); + let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control)); + self.select_i32x4(mask, expanded, merge) + } + #[inline(always)] fn count_ones_i32x4(self, a: i32x4) -> i32x4 { crate::kernel!( #[inline(always)] @@ -4113,6 +4427,43 @@ impl Simd for Sse4_2 { }) } #[inline(always)] + fn compress_u32x4(self, values: u32x4, mask: mask32x4) -> u32x4 { + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn compress_merge_u32x4( + self, + values: u32x4, + mask: mask32x4, + merge: u32x4, + ) -> u32x4 { + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0); + let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control); + let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0)); + Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed)) + } + #[inline(always)] + fn expand_u32x4(self, values: u32x4, mask: mask32x4) -> u32x4 { + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn expand_merge_u32x4( + self, + values: u32x4, + mask: mask32x4, + merge: u32x4, + ) -> u32x4 { + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0); + let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control)); + self.select_u32x4(mask, expanded, merge) + } + #[inline(always)] fn count_ones_u32x4(self, a: u32x4) -> u32x4 { crate::kernel!( #[inline(always)] @@ -4875,6 +5226,43 @@ impl Simd for Sse4_2 { }) } #[inline(always)] + fn compress_f64x2(self, values: f64x2, mask: mask64x2) -> f64x2 { + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn compress_merge_f64x2( + self, + values: f64x2, + mask: mask64x2, + merge: f64x2, + ) -> f64x2 { + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0); + let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control); + let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0)); + Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed)) + } + #[inline(always)] + fn expand_f64x2(self, values: f64x2, mask: mask64x2) -> f64x2 { + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn expand_merge_f64x2( + self, + values: f64x2, + mask: mask64x2, + merge: f64x2, + ) -> f64x2 { + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0); + let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control)); + self.select_f64x2(mask, expanded, merge) + } + #[inline(always)] fn neg_f64x2(self, a: f64x2) -> f64x2 { crate::kernel!( #[inline(always)] @@ -5465,6 +5853,43 @@ impl Simd for Sse4_2 { }) } #[inline(always)] + fn compress_i64x2(self, values: i64x2, mask: mask64x2) -> i64x2 { + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn compress_merge_i64x2( + self, + values: i64x2, + mask: mask64x2, + merge: i64x2, + ) -> i64x2 { + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0); + let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control); + let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0)); + Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed)) + } + #[inline(always)] + fn expand_i64x2(self, values: i64x2, mask: mask64x2) -> i64x2 { + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn expand_merge_i64x2( + self, + values: i64x2, + mask: mask64x2, + merge: i64x2, + ) -> i64x2 { + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0); + let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control)); + self.select_i64x2(mask, expanded, merge) + } + #[inline(always)] fn count_ones_i64x2(self, a: i64x2) -> i64x2 { crate::kernel!( #[inline(always)] @@ -6001,6 +6426,43 @@ impl Simd for Sse4_2 { }) } #[inline(always)] + fn compress_u64x2(self, values: u64x2, mask: mask64x2) -> u64x2 { + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn compress_merge_u64x2( + self, + values: u64x2, + mask: mask64x2, + merge: u64x2, + ) -> u64x2 { + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0); + let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control); + let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0)); + Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed)) + } + #[inline(always)] + fn expand_u64x2(self, values: u64x2, mask: mask64x2) -> u64x2 { + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn expand_merge_u64x2( + self, + values: u64x2, + mask: mask64x2, + merge: u64x2, + ) -> u64x2 { + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0); + let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control)); + self.select_u64x2(mask, expanded, merge) + } + #[inline(always)] fn count_ones_u64x2(self, a: u64x2) -> u64x2 { crate::kernel!( #[inline(always)] @@ -6736,6 +7198,56 @@ impl Simd for Sse4_2 { }) } #[inline(always)] + fn compress_f32x8(self, values: f32x8, mask: mask32x8) -> f32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_f32x8( + self, + values: f32x8, + mask: mask32x8, + merge: f32x8, + ) -> f32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_f32x8(self, values: f32x8, mask: mask32x8) -> f32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_f32x8( + self, + values: f32x8, + mask: mask32x8, + merge: f32x8, + ) -> f32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn simd_ne_f32x8(self, a: f32x8, b: f32x8) -> mask32x8 { let (a0, a1) = self.split_f32x8(a); let (b0, b1) = self.split_f32x8(b); @@ -6810,6 +7322,56 @@ impl Simd for Sse4_2 { }) } #[inline(always)] + fn compress_i8x32(self, values: i8x32, mask: mask8x32) -> i8x32 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i8x32( + self, + values: i8x32, + mask: mask8x32, + merge: i8x32, + ) -> i8x32 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i8x32(self, values: i8x32, mask: mask8x32) -> i8x32 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i8x32( + self, + values: i8x32, + mask: mask8x32, + merge: i8x32, + ) -> i8x32 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn combine_i8x32(self, a: i8x32, b: i8x32) -> i8x64 { i8x64 { val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), @@ -6902,6 +7464,251 @@ impl Simd for Sse4_2 { Bytes::from_bytes(result_bytes) } #[inline(always)] + fn compress_u8x32(self, values: u8x32, mask: mask8x32) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Sse4_2, + values: u8x32, + mask: mask8x32, + ) -> u8x32 { + let mask_bits = token.to_bitmask_mask8x32(mask); + let mut output = [0u8; 32]; + let mut output_lane = 0; + let block_bits = (mask_bits & 0xffff) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask]; + let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask]; + let high_control = high_control | 0x0808_0808_0808_0808; + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let low_count = low_mask.count_ones() as usize; + let compacted = _mm_shuffle_epi8(values.val.0[0], control); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count]; + let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let compressed = _mm_shuffle_epi8(compacted, splice); + crate::transmute::checked_transmute_store( + compressed, + output[output_lane..].first_chunk_mut::<16>().unwrap(), + ); + output_lane += block_bits.count_ones() as usize; + let block_bits = (mask_bits >> 16 & 0xffff) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask]; + let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask]; + let high_control = high_control | 0x0808_0808_0808_0808; + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let low_count = low_mask.count_ones() as usize; + let compacted = _mm_shuffle_epi8(values.val.0[1], control); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count]; + let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let compressed = _mm_shuffle_epi8(compacted, splice); + crate::transmute::checked_transmute_store( + compressed, + output[output_lane..].first_chunk_mut::<16>().unwrap(), + ); + u8x32::simd_from(token, output) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_u8x32( + self, + values: u8x32, + mask: mask8x32, + merge: u8x32, + ) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Sse4_2, + values: u8x32, + mask: mask8x32, + merge: u8x32, + ) -> u8x32 { + let mask_bits = token.to_bitmask_mask8x32(mask); + let mut output = [0u8; 32]; + let mut output_lane = 0; + let block_bits = (mask_bits & 0xffff) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask]; + let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask]; + let high_control = high_control | 0x0808_0808_0808_0808; + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let low_count = low_mask.count_ones() as usize; + let compacted = _mm_shuffle_epi8(values.val.0[0], control); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count]; + let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let compressed = _mm_shuffle_epi8(compacted, splice); + crate::transmute::checked_transmute_store( + compressed, + output[output_lane..].first_chunk_mut::<16>().unwrap(), + ); + output_lane += block_bits.count_ones() as usize; + let block_bits = (mask_bits >> 16 & 0xffff) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask]; + let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask]; + let high_control = high_control | 0x0808_0808_0808_0808; + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let low_count = low_mask.count_ones() as usize; + let compacted = _mm_shuffle_epi8(values.val.0[1], control); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count]; + let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let compressed = _mm_shuffle_epi8(compacted, splice); + crate::transmute::checked_transmute_store( + compressed, + output[output_lane..].first_chunk_mut::<16>().unwrap(), + ); + output_lane += block_bits.count_ones() as usize; + let remaining = output_lane.saturating_sub(0).min(16); + let [prefix, _] = crate::transmute::checked_transmute_copy::<_, [__m128i; 2]>( + &crate::support::COMPACT_PREFIX_MASKS[remaining], + ); + let compressed = crate::transmute::checked_transmute_copy( + output[0..].first_chunk::<16>().unwrap(), + ); + let blended = _mm_blendv_epi8(merge.val.0[0], compressed, prefix); + crate::transmute::checked_transmute_store( + blended, + output[0..].first_chunk_mut::<16>().unwrap(), + ); + let remaining = output_lane.saturating_sub(16).min(16); + let [prefix, _] = crate::transmute::checked_transmute_copy::<_, [__m128i; 2]>( + &crate::support::COMPACT_PREFIX_MASKS[remaining], + ); + let compressed = crate::transmute::checked_transmute_copy( + output[16..].first_chunk::<16>().unwrap(), + ); + let blended = _mm_blendv_epi8(merge.val.0[1], compressed, prefix); + crate::transmute::checked_transmute_store( + blended, + output[16..].first_chunk_mut::<16>().unwrap(), + ); + u8x32::simd_from(token, output) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_u8x32(self, values: u8x32, mask: mask8x32) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Sse4_2, + values: u8x32, + mask: mask8x32, + ) -> u8x32 { + let input = <[u8; 32]>::from(values); + let mut output = [0u8; 32]; + let mut input_lane = 0; + let block_bits = _mm_movemask_epi8(mask.val.0[0]) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = high_control + high_base; + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = crate::transmute::checked_transmute_copy( + input[input_lane..].first_chunk::<16>().unwrap(), + ); + let expanded = _mm_shuffle_epi8(packed, control); + let result = expanded; + crate::transmute::checked_transmute_store( + result, + output[0..].first_chunk_mut::<16>().unwrap(), + ); + input_lane += block_bits.count_ones() as usize; + let block_bits = _mm_movemask_epi8(mask.val.0[1]) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = high_control + high_base; + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = crate::transmute::checked_transmute_copy( + input[input_lane..].first_chunk::<16>().unwrap(), + ); + let expanded = _mm_shuffle_epi8(packed, control); + let result = expanded; + crate::transmute::checked_transmute_store( + result, + output[16..].first_chunk_mut::<16>().unwrap(), + ); + u8x32::simd_from(token, output) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_u8x32( + self, + values: u8x32, + mask: mask8x32, + merge: u8x32, + ) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Sse4_2, + values: u8x32, + mask: mask8x32, + merge: u8x32, + ) -> u8x32 { + let input = <[u8; 32]>::from(values); + let mut output = [0u8; 32]; + let mut input_lane = 0; + let block_bits = _mm_movemask_epi8(mask.val.0[0]) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = high_control + high_base; + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = crate::transmute::checked_transmute_copy( + input[input_lane..].first_chunk::<16>().unwrap(), + ); + let expanded = _mm_shuffle_epi8(packed, control); + let result = _mm_blendv_epi8(merge.val.0[0], expanded, mask.val.0[0]); + crate::transmute::checked_transmute_store( + result, + output[0..].first_chunk_mut::<16>().unwrap(), + ); + input_lane += block_bits.count_ones() as usize; + let block_bits = _mm_movemask_epi8(mask.val.0[1]) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = high_control + high_base; + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = crate::transmute::checked_transmute_copy( + input[input_lane..].first_chunk::<16>().unwrap(), + ); + let expanded = _mm_shuffle_epi8(packed, control); + let result = _mm_blendv_epi8(merge.val.0[1], expanded, mask.val.0[1]); + crate::transmute::checked_transmute_store( + result, + output[16..].first_chunk_mut::<16>().unwrap(), + ); + u8x32::simd_from(token, output) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn combine_u8x32(self, a: u8x32, b: u8x32) -> u8x64 { u8x64 { val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), @@ -6999,6 +7806,56 @@ impl Simd for Sse4_2 { }) } #[inline(always)] + fn compress_i16x16(self, values: i16x16, mask: mask16x16) -> i16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i16x16( + self, + values: i16x16, + mask: mask16x16, + merge: i16x16, + ) -> i16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i16x16(self, values: i16x16, mask: mask16x16) -> i16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i16x16( + self, + values: i16x16, + mask: mask16x16, + merge: i16x16, + ) -> i16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn combine_i16x16(self, a: i16x16, b: i16x16) -> i16x32 { i16x32 { val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), @@ -7035,17 +7892,67 @@ impl Simd for Sse4_2 { }) } #[inline(always)] - fn combine_u16x16(self, a: u16x16, b: u16x16) -> u16x32 { - u16x32 { - val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), + fn compress_u16x16(self, values: u16x16, mask: mask16x16) -> u16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), simd: self, - } + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) } #[inline(always)] - fn split_u16x16(self, a: u16x16) -> (u16x8, u16x8) { - ( - u16x8 { - val: crate::support::Aligned128(a.val.0[0]), + fn compress_merge_u16x16( + self, + values: u16x16, + mask: mask16x16, + merge: u16x16, + ) -> u16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u16x16(self, values: u16x16, mask: mask16x16) -> u16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u16x16( + self, + values: u16x16, + mask: mask16x16, + merge: u16x16, + ) -> u16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn combine_u16x16(self, a: u16x16, b: u16x16) -> u16x32 { + u16x32 { + val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), + simd: self, + } + } + #[inline(always)] + fn split_u16x16(self, a: u16x16) -> (u16x8, u16x8) { + ( + u16x8 { + val: crate::support::Aligned128(a.val.0[0]), simd: self, }, u16x8 { @@ -7145,6 +8052,56 @@ impl Simd for Sse4_2 { }) } #[inline(always)] + fn compress_i32x8(self, values: i32x8, mask: mask32x8) -> i32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i32x8( + self, + values: i32x8, + mask: mask32x8, + merge: i32x8, + ) -> i32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i32x8(self, values: i32x8, mask: mask32x8) -> i32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i32x8( + self, + values: i32x8, + mask: mask32x8, + merge: i32x8, + ) -> i32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn combine_i32x8(self, a: i32x8, b: i32x8) -> i32x16 { i32x16 { val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), @@ -7181,6 +8138,56 @@ impl Simd for Sse4_2 { }) } #[inline(always)] + fn compress_u32x8(self, values: u32x8, mask: mask32x8) -> u32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u32x8( + self, + values: u32x8, + mask: mask32x8, + merge: u32x8, + ) -> u32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u32x8(self, values: u32x8, mask: mask32x8) -> u32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u32x8( + self, + values: u32x8, + mask: mask32x8, + merge: u32x8, + ) -> u32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn combine_u32x8(self, a: u32x8, b: u32x8) -> u32x16 { u32x16 { val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), @@ -7278,6 +8285,56 @@ impl Simd for Sse4_2 { }) } #[inline(always)] + fn compress_f64x4(self, values: f64x4, mask: mask64x4) -> f64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_f64x4( + self, + values: f64x4, + mask: mask64x4, + merge: f64x4, + ) -> f64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_f64x4(self, values: f64x4, mask: mask64x4) -> f64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_f64x4( + self, + values: f64x4, + mask: mask64x4, + merge: f64x4, + ) -> f64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn simd_ne_f64x4(self, a: f64x4, b: f64x4) -> mask64x4 { let (a0, a1) = self.split_f64x4(a); let (b0, b1) = self.split_f64x4(b); @@ -7352,6 +8409,56 @@ impl Simd for Sse4_2 { }) } #[inline(always)] + fn compress_i64x4(self, values: i64x4, mask: mask64x4) -> i64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i64x4( + self, + values: i64x4, + mask: mask64x4, + merge: i64x4, + ) -> i64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i64x4(self, values: i64x4, mask: mask64x4) -> i64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i64x4( + self, + values: i64x4, + mask: mask64x4, + merge: i64x4, + ) -> i64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn combine_i64x4(self, a: i64x4, b: i64x4) -> i64x8 { i64x8 { val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), @@ -7388,6 +8495,56 @@ impl Simd for Sse4_2 { }) } #[inline(always)] + fn compress_u64x4(self, values: u64x4, mask: mask64x4) -> u64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u64x4( + self, + values: u64x4, + mask: mask64x4, + merge: u64x4, + ) -> u64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u64x4(self, values: u64x4, mask: mask64x4) -> u64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u64x4( + self, + values: u64x4, + mask: mask64x4, + merge: u64x4, + ) -> u64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn combine_u64x4(self, a: u64x4, b: u64x4) -> u64x8 { u64x8 { val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]), @@ -7485,6 +8642,56 @@ impl Simd for Sse4_2 { }) } #[inline(always)] + fn compress_f32x16(self, values: f32x16, mask: mask32x16) -> f32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_f32x16( + self, + values: f32x16, + mask: mask32x16, + merge: f32x16, + ) -> f32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_f32x16(self, values: f32x16, mask: mask32x16) -> f32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_f32x16( + self, + values: f32x16, + mask: mask32x16, + merge: f32x16, + ) -> f32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn simd_ne_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { let (a0, a1) = self.split_f32x16(a); let (b0, b1) = self.split_f32x16(b); @@ -7552,6 +8759,56 @@ impl Simd for Sse4_2 { }) } #[inline(always)] + fn compress_i8x64(self, values: i8x64, mask: mask8x64) -> i8x64 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i8x64( + self, + values: i8x64, + mask: mask8x64, + merge: i8x64, + ) -> i8x64 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i8x64(self, values: i8x64, mask: mask8x64) -> i8x64 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i8x64( + self, + values: i8x64, + mask: mask8x64, + merge: i8x64, + ) -> i8x64 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_i8x64(self, a: i8x64) -> (i8x32, i8x32) { ( i8x32 { @@ -7637,6 +8894,419 @@ impl Simd for Sse4_2 { Bytes::from_bytes(result_bytes) } #[inline(always)] + fn compress_u8x64(self, values: u8x64, mask: mask8x64) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Sse4_2, + values: u8x64, + mask: mask8x64, + ) -> u8x64 { + let mask_bits = token.to_bitmask_mask8x64(mask); + let mut output = [0u8; 64]; + let mut output_lane = 0; + let block_bits = (mask_bits & 0xffff) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask]; + let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask]; + let high_control = high_control | 0x0808_0808_0808_0808; + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let low_count = low_mask.count_ones() as usize; + let compacted = _mm_shuffle_epi8(values.val.0[0], control); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count]; + let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let compressed = _mm_shuffle_epi8(compacted, splice); + crate::transmute::checked_transmute_store( + compressed, + output[output_lane..].first_chunk_mut::<16>().unwrap(), + ); + output_lane += block_bits.count_ones() as usize; + let block_bits = (mask_bits >> 16 & 0xffff) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask]; + let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask]; + let high_control = high_control | 0x0808_0808_0808_0808; + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let low_count = low_mask.count_ones() as usize; + let compacted = _mm_shuffle_epi8(values.val.0[1], control); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count]; + let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let compressed = _mm_shuffle_epi8(compacted, splice); + crate::transmute::checked_transmute_store( + compressed, + output[output_lane..].first_chunk_mut::<16>().unwrap(), + ); + output_lane += block_bits.count_ones() as usize; + let block_bits = (mask_bits >> 32 & 0xffff) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask]; + let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask]; + let high_control = high_control | 0x0808_0808_0808_0808; + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let low_count = low_mask.count_ones() as usize; + let compacted = _mm_shuffle_epi8(values.val.0[2], control); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count]; + let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let compressed = _mm_shuffle_epi8(compacted, splice); + crate::transmute::checked_transmute_store( + compressed, + output[output_lane..].first_chunk_mut::<16>().unwrap(), + ); + output_lane += block_bits.count_ones() as usize; + let block_bits = (mask_bits >> 48 & 0xffff) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask]; + let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask]; + let high_control = high_control | 0x0808_0808_0808_0808; + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let low_count = low_mask.count_ones() as usize; + let compacted = _mm_shuffle_epi8(values.val.0[3], control); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count]; + let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let compressed = _mm_shuffle_epi8(compacted, splice); + crate::transmute::checked_transmute_store( + compressed, + output[output_lane..].first_chunk_mut::<16>().unwrap(), + ); + u8x64::simd_from(token, output) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn compress_merge_u8x64( + self, + values: u8x64, + mask: mask8x64, + merge: u8x64, + ) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Sse4_2, + values: u8x64, + mask: mask8x64, + merge: u8x64, + ) -> u8x64 { + let mask_bits = token.to_bitmask_mask8x64(mask); + let mut output = [0u8; 64]; + let mut output_lane = 0; + let block_bits = (mask_bits & 0xffff) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask]; + let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask]; + let high_control = high_control | 0x0808_0808_0808_0808; + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let low_count = low_mask.count_ones() as usize; + let compacted = _mm_shuffle_epi8(values.val.0[0], control); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count]; + let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let compressed = _mm_shuffle_epi8(compacted, splice); + crate::transmute::checked_transmute_store( + compressed, + output[output_lane..].first_chunk_mut::<16>().unwrap(), + ); + output_lane += block_bits.count_ones() as usize; + let block_bits = (mask_bits >> 16 & 0xffff) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask]; + let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask]; + let high_control = high_control | 0x0808_0808_0808_0808; + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let low_count = low_mask.count_ones() as usize; + let compacted = _mm_shuffle_epi8(values.val.0[1], control); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count]; + let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let compressed = _mm_shuffle_epi8(compacted, splice); + crate::transmute::checked_transmute_store( + compressed, + output[output_lane..].first_chunk_mut::<16>().unwrap(), + ); + output_lane += block_bits.count_ones() as usize; + let block_bits = (mask_bits >> 32 & 0xffff) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask]; + let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask]; + let high_control = high_control | 0x0808_0808_0808_0808; + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let low_count = low_mask.count_ones() as usize; + let compacted = _mm_shuffle_epi8(values.val.0[2], control); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count]; + let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let compressed = _mm_shuffle_epi8(compacted, splice); + crate::transmute::checked_transmute_store( + compressed, + output[output_lane..].first_chunk_mut::<16>().unwrap(), + ); + output_lane += block_bits.count_ones() as usize; + let block_bits = (mask_bits >> 48 & 0xffff) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask]; + let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask]; + let high_control = high_control | 0x0808_0808_0808_0808; + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let low_count = low_mask.count_ones() as usize; + let compacted = _mm_shuffle_epi8(values.val.0[3], control); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count]; + let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let compressed = _mm_shuffle_epi8(compacted, splice); + crate::transmute::checked_transmute_store( + compressed, + output[output_lane..].first_chunk_mut::<16>().unwrap(), + ); + output_lane += block_bits.count_ones() as usize; + let remaining = output_lane.saturating_sub(0).min(16); + let [prefix, _] = crate::transmute::checked_transmute_copy::<_, [__m128i; 2]>( + &crate::support::COMPACT_PREFIX_MASKS[remaining], + ); + let compressed = crate::transmute::checked_transmute_copy( + output[0..].first_chunk::<16>().unwrap(), + ); + let blended = _mm_blendv_epi8(merge.val.0[0], compressed, prefix); + crate::transmute::checked_transmute_store( + blended, + output[0..].first_chunk_mut::<16>().unwrap(), + ); + let remaining = output_lane.saturating_sub(16).min(16); + let [prefix, _] = crate::transmute::checked_transmute_copy::<_, [__m128i; 2]>( + &crate::support::COMPACT_PREFIX_MASKS[remaining], + ); + let compressed = crate::transmute::checked_transmute_copy( + output[16..].first_chunk::<16>().unwrap(), + ); + let blended = _mm_blendv_epi8(merge.val.0[1], compressed, prefix); + crate::transmute::checked_transmute_store( + blended, + output[16..].first_chunk_mut::<16>().unwrap(), + ); + let remaining = output_lane.saturating_sub(32).min(16); + let [prefix, _] = crate::transmute::checked_transmute_copy::<_, [__m128i; 2]>( + &crate::support::COMPACT_PREFIX_MASKS[remaining], + ); + let compressed = crate::transmute::checked_transmute_copy( + output[32..].first_chunk::<16>().unwrap(), + ); + let blended = _mm_blendv_epi8(merge.val.0[2], compressed, prefix); + crate::transmute::checked_transmute_store( + blended, + output[32..].first_chunk_mut::<16>().unwrap(), + ); + let remaining = output_lane.saturating_sub(48).min(16); + let [prefix, _] = crate::transmute::checked_transmute_copy::<_, [__m128i; 2]>( + &crate::support::COMPACT_PREFIX_MASKS[remaining], + ); + let compressed = crate::transmute::checked_transmute_copy( + output[48..].first_chunk::<16>().unwrap(), + ); + let blended = _mm_blendv_epi8(merge.val.0[3], compressed, prefix); + crate::transmute::checked_transmute_store( + blended, + output[48..].first_chunk_mut::<16>().unwrap(), + ); + u8x64::simd_from(token, output) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] + fn expand_u8x64(self, values: u8x64, mask: mask8x64) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Sse4_2, + values: u8x64, + mask: mask8x64, + ) -> u8x64 { + let input = <[u8; 64]>::from(values); + let mut output = [0u8; 64]; + let mut input_lane = 0; + let block_bits = _mm_movemask_epi8(mask.val.0[0]) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = high_control + high_base; + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = crate::transmute::checked_transmute_copy( + input[input_lane..].first_chunk::<16>().unwrap(), + ); + let expanded = _mm_shuffle_epi8(packed, control); + let result = expanded; + crate::transmute::checked_transmute_store( + result, + output[0..].first_chunk_mut::<16>().unwrap(), + ); + input_lane += block_bits.count_ones() as usize; + let block_bits = _mm_movemask_epi8(mask.val.0[1]) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = high_control + high_base; + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = crate::transmute::checked_transmute_copy( + input[input_lane..].first_chunk::<16>().unwrap(), + ); + let expanded = _mm_shuffle_epi8(packed, control); + let result = expanded; + crate::transmute::checked_transmute_store( + result, + output[16..].first_chunk_mut::<16>().unwrap(), + ); + input_lane += block_bits.count_ones() as usize; + let block_bits = _mm_movemask_epi8(mask.val.0[2]) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = high_control + high_base; + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = crate::transmute::checked_transmute_copy( + input[input_lane..].first_chunk::<16>().unwrap(), + ); + let expanded = _mm_shuffle_epi8(packed, control); + let result = expanded; + crate::transmute::checked_transmute_store( + result, + output[32..].first_chunk_mut::<16>().unwrap(), + ); + input_lane += block_bits.count_ones() as usize; + let block_bits = _mm_movemask_epi8(mask.val.0[3]) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = high_control + high_base; + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = crate::transmute::checked_transmute_copy( + input[input_lane..].first_chunk::<16>().unwrap(), + ); + let expanded = _mm_shuffle_epi8(packed, control); + let result = expanded; + crate::transmute::checked_transmute_store( + result, + output[48..].first_chunk_mut::<16>().unwrap(), + ); + u8x64::simd_from(token, output) + } + ); + kernel(self, values, mask) + } + #[inline(always)] + fn expand_merge_u8x64( + self, + values: u8x64, + mask: mask8x64, + merge: u8x64, + ) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel( + token: Sse4_2, + values: u8x64, + mask: mask8x64, + merge: u8x64, + ) -> u8x64 { + let input = <[u8; 64]>::from(values); + let mut output = [0u8; 64]; + let mut input_lane = 0; + let block_bits = _mm_movemask_epi8(mask.val.0[0]) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = high_control + high_base; + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = crate::transmute::checked_transmute_copy( + input[input_lane..].first_chunk::<16>().unwrap(), + ); + let expanded = _mm_shuffle_epi8(packed, control); + let result = _mm_blendv_epi8(merge.val.0[0], expanded, mask.val.0[0]); + crate::transmute::checked_transmute_store( + result, + output[0..].first_chunk_mut::<16>().unwrap(), + ); + input_lane += block_bits.count_ones() as usize; + let block_bits = _mm_movemask_epi8(mask.val.0[1]) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = high_control + high_base; + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = crate::transmute::checked_transmute_copy( + input[input_lane..].first_chunk::<16>().unwrap(), + ); + let expanded = _mm_shuffle_epi8(packed, control); + let result = _mm_blendv_epi8(merge.val.0[1], expanded, mask.val.0[1]); + crate::transmute::checked_transmute_store( + result, + output[16..].first_chunk_mut::<16>().unwrap(), + ); + input_lane += block_bits.count_ones() as usize; + let block_bits = _mm_movemask_epi8(mask.val.0[2]) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = high_control + high_base; + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = crate::transmute::checked_transmute_copy( + input[input_lane..].first_chunk::<16>().unwrap(), + ); + let expanded = _mm_shuffle_epi8(packed, control); + let result = _mm_blendv_epi8(merge.val.0[2], expanded, mask.val.0[2]); + crate::transmute::checked_transmute_store( + result, + output[32..].first_chunk_mut::<16>().unwrap(), + ); + input_lane += block_bits.count_ones() as usize; + let block_bits = _mm_movemask_epi8(mask.val.0[3]) as usize; + let low_mask = (block_bits) & 0xff; + let high_mask = (block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + let high_control = high_control + high_base; + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + let packed = crate::transmute::checked_transmute_copy( + input[input_lane..].first_chunk::<16>().unwrap(), + ); + let expanded = _mm_shuffle_epi8(packed, control); + let result = _mm_blendv_epi8(merge.val.0[3], expanded, mask.val.0[3]); + crate::transmute::checked_transmute_store( + result, + output[48..].first_chunk_mut::<16>().unwrap(), + ); + u8x64::simd_from(token, output) + } + ); + kernel(self, values, mask, merge) + } + #[inline(always)] fn split_u8x64(self, a: u8x64) -> (u8x32, u8x32) { ( u8x32 { @@ -7767,6 +9437,56 @@ impl Simd for Sse4_2 { }) } #[inline(always)] + fn compress_i16x32(self, values: i16x32, mask: mask16x32) -> i16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i16x32( + self, + values: i16x32, + mask: mask16x32, + merge: i16x32, + ) -> i16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i16x32(self, values: i16x32, mask: mask16x32) -> i16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i16x32( + self, + values: i16x32, + mask: mask16x32, + merge: i16x32, + ) -> i16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_i16x32(self, a: i16x32) -> (i16x16, i16x16) { ( i16x16 { @@ -7796,6 +9516,56 @@ impl Simd for Sse4_2 { }) } #[inline(always)] + fn compress_u16x32(self, values: u16x32, mask: mask16x32) -> u16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u16x32( + self, + values: u16x32, + mask: mask16x32, + merge: u16x32, + ) -> u16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u16x32(self, values: u16x32, mask: mask16x32) -> u16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u16x32( + self, + values: u16x32, + mask: mask16x32, + merge: u16x32, + ) -> u16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_u16x32(self, a: u16x32) -> (u16x16, u16x16) { ( u16x16 { @@ -7895,6 +9665,56 @@ impl Simd for Sse4_2 { }) } #[inline(always)] + fn compress_i32x16(self, values: i32x16, mask: mask32x16) -> i32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i32x16( + self, + values: i32x16, + mask: mask32x16, + merge: i32x16, + ) -> i32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i32x16(self, values: i32x16, mask: mask32x16) -> i32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i32x16( + self, + values: i32x16, + mask: mask32x16, + merge: i32x16, + ) -> i32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_i32x16(self, a: i32x16) -> (i32x8, i32x8) { ( i32x8 { @@ -7924,6 +9744,56 @@ impl Simd for Sse4_2 { }) } #[inline(always)] + fn compress_u32x16(self, values: u32x16, mask: mask32x16) -> u32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u32x16( + self, + values: u32x16, + mask: mask32x16, + merge: u32x16, + ) -> u32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u32x16(self, values: u32x16, mask: mask32x16) -> u32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u32x16( + self, + values: u32x16, + mask: mask32x16, + merge: u32x16, + ) -> u32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_u32x16(self, a: u32x16) -> (u32x8, u32x8) { ( u32x8 { @@ -8007,6 +9877,56 @@ impl Simd for Sse4_2 { }) } #[inline(always)] + fn compress_f64x8(self, values: f64x8, mask: mask64x8) -> f64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_f64x8( + self, + values: f64x8, + mask: mask64x8, + merge: f64x8, + ) -> f64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_f64x8(self, values: f64x8, mask: mask64x8) -> f64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_f64x8( + self, + values: f64x8, + mask: mask64x8, + merge: f64x8, + ) -> f64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn simd_ne_f64x8(self, a: f64x8, b: f64x8) -> mask64x8 { let (a0, a1) = self.split_f64x8(a); let (b0, b1) = self.split_f64x8(b); @@ -8074,6 +9994,56 @@ impl Simd for Sse4_2 { }) } #[inline(always)] + fn compress_i64x8(self, values: i64x8, mask: mask64x8) -> i64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i64x8( + self, + values: i64x8, + mask: mask64x8, + merge: i64x8, + ) -> i64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i64x8(self, values: i64x8, mask: mask64x8) -> i64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i64x8( + self, + values: i64x8, + mask: mask64x8, + merge: i64x8, + ) -> i64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_i64x8(self, a: i64x8) -> (i64x4, i64x4) { ( i64x4 { @@ -8103,6 +10073,56 @@ impl Simd for Sse4_2 { }) } #[inline(always)] + fn compress_u64x8(self, values: u64x8, mask: mask64x8) -> u64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u64x8( + self, + values: u64x8, + mask: mask64x8, + merge: u64x8, + ) -> u64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u64x8(self, values: u64x8, mask: mask64x8) -> u64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u64x8( + self, + values: u64x8, + mask: mask64x8, + merge: u64x8, + ) -> u64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn split_u64x8(self, a: u64x8) -> (u64x4, u64x4) { ( u64x4 { diff --git a/fearless_simd/src/generated/wasm.rs b/fearless_simd/src/generated/wasm.rs index f52da8ab..940d655e 100644 --- a/fearless_simd/src/generated/wasm.rs +++ b/fearless_simd/src/generated/wasm.rs @@ -147,6 +147,43 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn compress_f32x4(self, values: f32x4, mask: mask32x4) -> f32x4 { + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn compress_merge_f32x4( + self, + values: f32x4, + mask: mask32x4, + merge: f32x4, + ) -> f32x4 { + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0); + let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control); + let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0)); + Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed)) + } + #[inline(always)] + fn expand_f32x4(self, values: f32x4, mask: mask32x4) -> f32x4 { + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn expand_merge_f32x4( + self, + values: f32x4, + mask: mask32x4, + merge: f32x4, + ) -> f32x4 { + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0); + let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control)); + self.select_f32x4(mask, expanded, merge) + } + #[inline(always)] fn neg_f32x4(self, a: f32x4) -> f32x4 { f32x4_neg(a.into()).simd_into(self) } @@ -531,6 +568,56 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn compress_i8x16(self, values: i8x16, mask: mask8x16) -> i8x16 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i8x16( + self, + values: i8x16, + mask: mask8x16, + merge: i8x16, + ) -> i8x16 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i8x16(self, values: i8x16, mask: mask8x16) -> i8x16 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i8x16( + self, + values: i8x16, + mask: mask8x16, + merge: i8x16, + ) -> i8x16 { + let mask = mask8x16 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x16( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn count_ones_i8x16(self, a: i8x16) -> i8x16 { i8x16_popcnt(a.into()).simd_into(self) } @@ -962,6 +1049,94 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn compress_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16 { + let mask_bits = self.to_bitmask_mask8x16(mask); + let mut control = [u8::MAX; 16]; + let mut output_lane = 0; + for block in 0..16 / 8 { + let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; + let packed = crate::support::COMPRESS_8_CONTROLS[block_mask]; + let base = (block * 8) as u64 * 0x0101_0101_0101_0101; + let adjusted = packed | base; + let adjusted = adjusted.to_le_bytes(); + let write_len = core::cmp::min(8, 16 - output_lane); + control[output_lane..output_lane + write_len].copy_from_slice(&adjusted[..write_len]); + output_lane += block_mask.count_ones() as usize; + } + let control = u8x16::simd_from(self, control); + self.swizzle_dyn_precise_u8x16(values, control) + } + #[inline(always)] + fn compress_merge_u8x16( + self, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + let mask_bits = self.to_bitmask_mask8x16(mask); + let mut control = [u8::MAX; 16]; + let mut output_lane = 0; + for block in 0..16 / 8 { + let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; + let packed = crate::support::COMPRESS_8_CONTROLS[block_mask]; + let base = (block * 8) as u64 * 0x0101_0101_0101_0101; + let adjusted = packed | base; + let adjusted = adjusted.to_le_bytes(); + let write_len = core::cmp::min(8, 16 - output_lane); + control[output_lane..output_lane + write_len].copy_from_slice(&adjusted[..write_len]); + output_lane += block_mask.count_ones() as usize; + } + let control = u8x16::simd_from(self, control); + let compressed = self.swizzle_dyn_precise_u8x16(values, control); + let prefix_bits = if output_lane == 64 { + u64::MAX + } else { + (1u64 << output_lane) - 1 + }; + let prefix_mask = self.from_bitmask_mask8x16(prefix_bits); + self.select_u8x16(prefix_mask, compressed, merge) + } + #[inline(always)] + fn expand_u8x16(self, values: u8x16, mask: mask8x16) -> u8x16 { + let mask_bits = self.to_bitmask_mask8x16(mask); + let mut control = [u8::MAX; 16]; + let mut input_lane = 0; + for block in 0..16 / 8 { + let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; + let packed = crate::support::EXPAND_8_CONTROLS[block_mask]; + let base = input_lane as u64 * 0x0101_0101_0101_0101; + let adjusted = packed + base; + let output_lane = block * 8; + control[output_lane..output_lane + 8].copy_from_slice(&adjusted.to_le_bytes()); + input_lane += block_mask.count_ones() as usize; + } + let control = u8x16::simd_from(self, control); + self.swizzle_dyn_precise_u8x16(values, control) + } + #[inline(always)] + fn expand_merge_u8x16( + self, + values: u8x16, + mask: mask8x16, + merge: u8x16, + ) -> u8x16 { + let mask_bits = self.to_bitmask_mask8x16(mask); + let mut control = [u8::MAX; 16]; + let mut input_lane = 0; + for block in 0..16 / 8 { + let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; + let packed = crate::support::EXPAND_8_CONTROLS[block_mask]; + let base = input_lane as u64 * 0x0101_0101_0101_0101; + let adjusted = packed + base; + let output_lane = block * 8; + control[output_lane..output_lane + 8].copy_from_slice(&adjusted.to_le_bytes()); + input_lane += block_mask.count_ones() as usize; + } + let control = u8x16::simd_from(self, control); + let expanded = self.swizzle_dyn_precise_u8x16(values, control); + self.select_u8x16(mask, expanded, merge) + } + #[inline(always)] fn count_ones_u8x16(self, a: u8x16) -> u8x16 { i8x16_popcnt(a.into()).simd_into(self) } @@ -1478,6 +1653,43 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn compress_i16x8(self, values: i16x8, mask: mask16x8) -> i16x8 { + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_16[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn compress_merge_i16x8( + self, + values: i16x8, + mask: mask16x8, + merge: i16x8, + ) -> i16x8 { + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_16[bits].0); + let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control); + let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0)); + Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed)) + } + #[inline(always)] + fn expand_i16x8(self, values: i16x8, mask: mask16x8) -> i16x8 { + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_16[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn expand_merge_i16x8( + self, + values: i16x8, + mask: mask16x8, + merge: i16x8, + ) -> i16x8 { + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_16[bits].0); + let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control)); + self.select_i16x8(mask, expanded, merge) + } + #[inline(always)] fn count_ones_i16x8(self, a: i16x8) -> i16x8 { u16x8_extadd_pairwise_u8x16(i8x16_popcnt(a.into())).simd_into(self) } @@ -1773,6 +1985,43 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn compress_u16x8(self, values: u16x8, mask: mask16x8) -> u16x8 { + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_16[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn compress_merge_u16x8( + self, + values: u16x8, + mask: mask16x8, + merge: u16x8, + ) -> u16x8 { + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_16[bits].0); + let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control); + let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0)); + Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed)) + } + #[inline(always)] + fn expand_u16x8(self, values: u16x8, mask: mask16x8) -> u16x8 { + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_16[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn expand_merge_u16x8( + self, + values: u16x8, + mask: mask16x8, + merge: u16x8, + ) -> u16x8 { + let bits = self.to_bitmask_mask16x8(mask) as usize & 255; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_16[bits].0); + let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control)); + self.select_u16x8(mask, expanded, merge) + } + #[inline(always)] fn count_ones_u16x8(self, a: u16x8) -> u16x8 { u16x8_extadd_pairwise_u8x16(i8x16_popcnt(a.into())).simd_into(self) } @@ -2212,6 +2461,43 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn compress_i32x4(self, values: i32x4, mask: mask32x4) -> i32x4 { + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn compress_merge_i32x4( + self, + values: i32x4, + mask: mask32x4, + merge: i32x4, + ) -> i32x4 { + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0); + let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control); + let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0)); + Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed)) + } + #[inline(always)] + fn expand_i32x4(self, values: i32x4, mask: mask32x4) -> i32x4 { + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn expand_merge_i32x4( + self, + values: i32x4, + mask: mask32x4, + merge: i32x4, + ) -> i32x4 { + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0); + let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control)); + self.select_i32x4(mask, expanded, merge) + } + #[inline(always)] fn count_ones_i32x4(self, a: i32x4) -> i32x4 { u32x4_extadd_pairwise_u16x8(u16x8_extadd_pairwise_u8x16(i8x16_popcnt(a.into()))) .simd_into(self) @@ -2502,6 +2788,43 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn compress_u32x4(self, values: u32x4, mask: mask32x4) -> u32x4 { + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn compress_merge_u32x4( + self, + values: u32x4, + mask: mask32x4, + merge: u32x4, + ) -> u32x4 { + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0); + let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control); + let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0)); + Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed)) + } + #[inline(always)] + fn expand_u32x4(self, values: u32x4, mask: mask32x4) -> u32x4 { + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn expand_merge_u32x4( + self, + values: u32x4, + mask: mask32x4, + merge: u32x4, + ) -> u32x4 { + let bits = self.to_bitmask_mask32x4(mask) as usize & 15; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0); + let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control)); + self.select_u32x4(mask, expanded, merge) + } + #[inline(always)] fn count_ones_u32x4(self, a: u32x4) -> u32x4 { u32x4_extadd_pairwise_u16x8(u16x8_extadd_pairwise_u8x16(i8x16_popcnt(a.into()))) .simd_into(self) @@ -2930,6 +3253,43 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn compress_f64x2(self, values: f64x2, mask: mask64x2) -> f64x2 { + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn compress_merge_f64x2( + self, + values: f64x2, + mask: mask64x2, + merge: f64x2, + ) -> f64x2 { + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0); + let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control); + let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0)); + Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed)) + } + #[inline(always)] + fn expand_f64x2(self, values: f64x2, mask: mask64x2) -> f64x2 { + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn expand_merge_f64x2( + self, + values: f64x2, + mask: mask64x2, + merge: f64x2, + ) -> f64x2 { + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0); + let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control)); + self.select_f64x2(mask, expanded, merge) + } + #[inline(always)] fn neg_f64x2(self, a: f64x2) -> f64x2 { f64x2_neg(a.into()).simd_into(self) } @@ -3292,6 +3652,43 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn compress_i64x2(self, values: i64x2, mask: mask64x2) -> i64x2 { + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn compress_merge_i64x2( + self, + values: i64x2, + mask: mask64x2, + merge: i64x2, + ) -> i64x2 { + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0); + let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control); + let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0)); + Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed)) + } + #[inline(always)] + fn expand_i64x2(self, values: i64x2, mask: mask64x2) -> i64x2 { + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn expand_merge_i64x2( + self, + values: i64x2, + mask: mask64x2, + merge: i64x2, + ) -> i64x2 { + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0); + let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control)); + self.select_i64x2(mask, expanded, merge) + } + #[inline(always)] fn count_ones_i64x2(self, a: i64x2) -> i64x2 { { let counts = @@ -3576,6 +3973,43 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn compress_u64x2(self, values: u64x2, mask: mask64x2) -> u64x2 { + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn compress_merge_u64x2( + self, + values: u64x2, + mask: mask64x2, + merge: u64x2, + ) -> u64x2 { + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0); + let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control); + let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0)); + Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed)) + } + #[inline(always)] + fn expand_u64x2(self, values: u64x2, mask: mask64x2) -> u64x2 { + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0); + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + #[inline(always)] + fn expand_merge_u64x2( + self, + values: u64x2, + mask: mask64x2, + merge: u64x2, + ) -> u64x2 { + let bits = self.to_bitmask_mask64x2(mask) as usize & 3; + let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0); + let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control)); + self.select_u64x2(mask, expanded, merge) + } + #[inline(always)] fn count_ones_u64x2(self, a: u64x2) -> u64x2 { { let counts = @@ -3998,6 +4432,56 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn compress_f32x8(self, values: f32x8, mask: mask32x8) -> f32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_f32x8( + self, + values: f32x8, + mask: mask32x8, + merge: f32x8, + ) -> f32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_f32x8(self, values: f32x8, mask: mask32x8) -> f32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_f32x8( + self, + values: f32x8, + mask: mask32x8, + merge: f32x8, + ) -> f32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn simd_ne_f32x8(self, a: f32x8, b: f32x8) -> mask32x8 { let (a0, a1) = self.split_f32x8(a); let (b0, b1) = self.split_f32x8(b); @@ -4068,6 +4552,56 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn compress_i8x32(self, values: i8x32, mask: mask8x32) -> i8x32 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i8x32( + self, + values: i8x32, + mask: mask8x32, + merge: i8x32, + ) -> i8x32 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i8x32(self, values: i8x32, mask: mask8x32) -> i8x32 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i8x32( + self, + values: i8x32, + mask: mask8x32, + merge: i8x32, + ) -> i8x32 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn simd_ne_i8x32(self, a: i8x32, b: i8x32) -> mask8x32 { let (a0, a1) = self.split_i8x32(a); let (b0, b1) = self.split_i8x32(b); @@ -4179,6 +4713,132 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn compress_u8x32(self, values: u8x32, mask: mask8x32) -> u8x32 { + let mask_bits = self.to_bitmask_mask8x32(mask); + let mut output = [0u8; 32]; + let mut output_lane = 0; + for block in 0..32 / 16 { + let block_bits = ((mask_bits >> (block * 16)) & 0xffff) as usize; + let low_mask = block_bits & 0xff; + let high_mask = block_bits >> 8; + let low_count = low_mask.count_ones() as usize; + let low = crate::support::COMPRESS_8_CONTROLS[low_mask]; + let high = crate::support::COMPRESS_8_CONTROLS[high_mask]; + let high = high | 0x0808_0808_0808_0808; + let mut control = [u8::MAX; 16]; + control[..8].copy_from_slice(&low.to_le_bytes()); + control[low_count..low_count + 8].copy_from_slice(&high.to_le_bytes()); + let input = u8x16::from_slice(self, &values.as_array()[block * 16..block * 16 + 16]); + let control = u8x16::simd_from(self, control); + let compacted: [u8; 16] = self.swizzle_dyn_precise_u8x16(input, control).into(); + let write_len = core::cmp::min(16, 32 - output_lane); + output[output_lane..output_lane + write_len].copy_from_slice(&compacted[..write_len]); + output_lane += low_count + high_mask.count_ones() as usize; + } + u8x32::simd_from(self, output) + } + #[inline(always)] + fn compress_merge_u8x32( + self, + values: u8x32, + mask: mask8x32, + merge: u8x32, + ) -> u8x32 { + let mask_bits = self.to_bitmask_mask8x32(mask); + let mut output = [0u8; 32]; + let mut output_lane = 0; + for block in 0..32 / 16 { + let block_bits = ((mask_bits >> (block * 16)) & 0xffff) as usize; + let low_mask = block_bits & 0xff; + let high_mask = block_bits >> 8; + let low_count = low_mask.count_ones() as usize; + let low = crate::support::COMPRESS_8_CONTROLS[low_mask]; + let high = crate::support::COMPRESS_8_CONTROLS[high_mask]; + let high = high | 0x0808_0808_0808_0808; + let mut control = [u8::MAX; 16]; + control[..8].copy_from_slice(&low.to_le_bytes()); + control[low_count..low_count + 8].copy_from_slice(&high.to_le_bytes()); + let input = u8x16::from_slice(self, &values.as_array()[block * 16..block * 16 + 16]); + let control = u8x16::simd_from(self, control); + let compacted: [u8; 16] = self.swizzle_dyn_precise_u8x16(input, control).into(); + let write_len = core::cmp::min(16, 32 - output_lane); + output[output_lane..output_lane + write_len].copy_from_slice(&compacted[..write_len]); + output_lane += low_count + high_mask.count_ones() as usize; + } + let compressed = u8x32::simd_from(self, output); + let prefix_bits = if output_lane == 64 { + u64::MAX + } else { + (1u64 << output_lane) - 1 + }; + let prefix_mask = self.from_bitmask_mask8x32(prefix_bits); + self.select_u8x32(prefix_mask, compressed, merge) + } + #[inline(always)] + fn expand_u8x32(self, values: u8x32, mask: mask8x32) -> u8x32 { + let mask_bits = self.to_bitmask_mask8x32(mask); + let values: [u8; 32] = values.into(); + let mut output = [0u8; 32]; + let mut input_lane = 0; + for block in 0..32 / 16 { + let block_bits = ((mask_bits >> (block * 16)) & 0xffff) as usize; + let low_mask = block_bits & 0xff; + let high_mask = block_bits >> 8; + let low_count = low_mask.count_ones() as usize; + let low = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count as u64 * 0x0101_0101_0101_0101; + let high = high + high_base; + let mut control = [0u8; 16]; + control[..8].copy_from_slice(&low.to_le_bytes()); + control[8..].copy_from_slice(&high.to_le_bytes()); + let mut input = [0u8; 16]; + let read_len = core::cmp::min(16, 32 - input_lane); + input[..read_len].copy_from_slice(&values[input_lane..input_lane + read_len]); + let input = u8x16::simd_from(self, input); + let control = u8x16::simd_from(self, control); + let expanded: [u8; 16] = self.swizzle_dyn_precise_u8x16(input, control).into(); + output[block * 16..block * 16 + 16].copy_from_slice(&expanded); + input_lane += low_count + high_mask.count_ones() as usize; + } + u8x32::simd_from(self, output) + } + #[inline(always)] + fn expand_merge_u8x32( + self, + values: u8x32, + mask: mask8x32, + merge: u8x32, + ) -> u8x32 { + let mask_bits = self.to_bitmask_mask8x32(mask); + let values: [u8; 32] = values.into(); + let mut output = [0u8; 32]; + let mut input_lane = 0; + for block in 0..32 / 16 { + let block_bits = ((mask_bits >> (block * 16)) & 0xffff) as usize; + let low_mask = block_bits & 0xff; + let high_mask = block_bits >> 8; + let low_count = low_mask.count_ones() as usize; + let low = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count as u64 * 0x0101_0101_0101_0101; + let high = high + high_base; + let mut control = [0u8; 16]; + control[..8].copy_from_slice(&low.to_le_bytes()); + control[8..].copy_from_slice(&high.to_le_bytes()); + let mut input = [0u8; 16]; + let read_len = core::cmp::min(16, 32 - input_lane); + input[..read_len].copy_from_slice(&values[input_lane..input_lane + read_len]); + let input = u8x16::simd_from(self, input); + let control = u8x16::simd_from(self, control); + let expanded: [u8; 16] = self.swizzle_dyn_precise_u8x16(input, control).into(); + output[block * 16..block * 16 + 16].copy_from_slice(&expanded); + input_lane += low_count + high_mask.count_ones() as usize; + } + let expanded = u8x32::simd_from(self, output); + self.select_u8x32(mask, expanded, merge) + } + #[inline(always)] fn simd_ne_u8x32(self, a: u8x32, b: u8x32) -> mask8x32 { let (a0, a1) = self.split_u8x32(a); let (b0, b1) = self.split_u8x32(b); @@ -4287,6 +4947,56 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn compress_i16x16(self, values: i16x16, mask: mask16x16) -> i16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i16x16( + self, + values: i16x16, + mask: mask16x16, + merge: i16x16, + ) -> i16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i16x16(self, values: i16x16, mask: mask16x16) -> i16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i16x16( + self, + values: i16x16, + mask: mask16x16, + merge: i16x16, + ) -> i16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn simd_ne_i16x16(self, a: i16x16, b: i16x16) -> mask16x16 { let (a0, a1) = self.split_i16x16(a); let (b0, b1) = self.split_i16x16(b); @@ -4328,6 +5038,56 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn compress_u16x16(self, values: u16x16, mask: mask16x16) -> u16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u16x16( + self, + values: u16x16, + mask: mask16x16, + merge: u16x16, + ) -> u16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u16x16(self, values: u16x16, mask: mask16x16) -> u16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u16x16( + self, + values: u16x16, + mask: mask16x16, + merge: u16x16, + ) -> u16x16 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn simd_ne_u16x16(self, a: u16x16, b: u16x16) -> mask16x16 { let (a0, a1) = self.split_u16x16(a); let (b0, b1) = self.split_u16x16(b); @@ -4436,6 +5196,56 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn compress_i32x8(self, values: i32x8, mask: mask32x8) -> i32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i32x8( + self, + values: i32x8, + mask: mask32x8, + merge: i32x8, + ) -> i32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i32x8(self, values: i32x8, mask: mask32x8) -> i32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i32x8( + self, + values: i32x8, + mask: mask32x8, + merge: i32x8, + ) -> i32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn simd_ne_i32x8(self, a: i32x8, b: i32x8) -> mask32x8 { let (a0, a1) = self.split_i32x8(a); let (b0, b1) = self.split_i32x8(b); @@ -4477,6 +5287,56 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn compress_u32x8(self, values: u32x8, mask: mask32x8) -> u32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u32x8( + self, + values: u32x8, + mask: mask32x8, + merge: u32x8, + ) -> u32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u32x8(self, values: u32x8, mask: mask32x8) -> u32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u32x8( + self, + values: u32x8, + mask: mask32x8, + merge: u32x8, + ) -> u32x8 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn simd_ne_u32x8(self, a: u32x8, b: u32x8) -> mask32x8 { let (a0, a1) = self.split_u32x8(a); let (b0, b1) = self.split_u32x8(b); @@ -4585,6 +5445,56 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn compress_f64x4(self, values: f64x4, mask: mask64x4) -> f64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_f64x4( + self, + values: f64x4, + mask: mask64x4, + merge: f64x4, + ) -> f64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_f64x4(self, values: f64x4, mask: mask64x4) -> f64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_f64x4( + self, + values: f64x4, + mask: mask64x4, + merge: f64x4, + ) -> f64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn simd_ne_f64x4(self, a: f64x4, b: f64x4) -> mask64x4 { let (a0, a1) = self.split_f64x4(a); let (b0, b1) = self.split_f64x4(b); @@ -4658,6 +5568,56 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn compress_i64x4(self, values: i64x4, mask: mask64x4) -> i64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i64x4( + self, + values: i64x4, + mask: mask64x4, + merge: i64x4, + ) -> i64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i64x4(self, values: i64x4, mask: mask64x4) -> i64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i64x4( + self, + values: i64x4, + mask: mask64x4, + merge: i64x4, + ) -> i64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn simd_ne_i64x4(self, a: i64x4, b: i64x4) -> mask64x4 { let (a0, a1) = self.split_i64x4(a); let (b0, b1) = self.split_i64x4(b); @@ -4699,6 +5659,56 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn compress_u64x4(self, values: u64x4, mask: mask64x4) -> u64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u64x4( + self, + values: u64x4, + mask: mask64x4, + merge: u64x4, + ) -> u64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u64x4(self, values: u64x4, mask: mask64x4) -> u64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u64x4( + self, + values: u64x4, + mask: mask64x4, + merge: u64x4, + ) -> u64x4 { + let mask = mask8x32 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x32( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn simd_ne_u64x4(self, a: u64x4, b: u64x4) -> mask64x4 { let (a0, a1) = self.split_u64x4(a); let (b0, b1) = self.split_u64x4(b); @@ -4807,6 +5817,56 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn compress_f32x16(self, values: f32x16, mask: mask32x16) -> f32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_f32x16( + self, + values: f32x16, + mask: mask32x16, + merge: f32x16, + ) -> f32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_f32x16(self, values: f32x16, mask: mask32x16) -> f32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_f32x16( + self, + values: f32x16, + mask: mask32x16, + merge: f32x16, + ) -> f32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn simd_ne_f32x16(self, a: f32x16, b: f32x16) -> mask32x16 { let (a0, a1) = self.split_f32x16(a); let (b0, b1) = self.split_f32x16(b); @@ -4870,6 +5930,56 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn compress_i8x64(self, values: i8x64, mask: mask8x64) -> i8x64 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i8x64( + self, + values: i8x64, + mask: mask8x64, + merge: i8x64, + ) -> i8x64 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i8x64(self, values: i8x64, mask: mask8x64) -> i8x64 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i8x64( + self, + values: i8x64, + mask: mask8x64, + merge: i8x64, + ) -> i8x64 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn simd_ne_i8x64(self, a: i8x64, b: i8x64) -> mask8x64 { let (a0, a1) = self.split_i8x64(a); let (b0, b1) = self.split_i8x64(b); @@ -4947,6 +6057,132 @@ impl Simd for WasmSimd128 { Bytes::from_bytes(result_bytes) } #[inline(always)] + fn compress_u8x64(self, values: u8x64, mask: mask8x64) -> u8x64 { + let mask_bits = self.to_bitmask_mask8x64(mask); + let mut output = [0u8; 64]; + let mut output_lane = 0; + for block in 0..64 / 16 { + let block_bits = ((mask_bits >> (block * 16)) & 0xffff) as usize; + let low_mask = block_bits & 0xff; + let high_mask = block_bits >> 8; + let low_count = low_mask.count_ones() as usize; + let low = crate::support::COMPRESS_8_CONTROLS[low_mask]; + let high = crate::support::COMPRESS_8_CONTROLS[high_mask]; + let high = high | 0x0808_0808_0808_0808; + let mut control = [u8::MAX; 16]; + control[..8].copy_from_slice(&low.to_le_bytes()); + control[low_count..low_count + 8].copy_from_slice(&high.to_le_bytes()); + let input = u8x16::from_slice(self, &values.as_array()[block * 16..block * 16 + 16]); + let control = u8x16::simd_from(self, control); + let compacted: [u8; 16] = self.swizzle_dyn_precise_u8x16(input, control).into(); + let write_len = core::cmp::min(16, 64 - output_lane); + output[output_lane..output_lane + write_len].copy_from_slice(&compacted[..write_len]); + output_lane += low_count + high_mask.count_ones() as usize; + } + u8x64::simd_from(self, output) + } + #[inline(always)] + fn compress_merge_u8x64( + self, + values: u8x64, + mask: mask8x64, + merge: u8x64, + ) -> u8x64 { + let mask_bits = self.to_bitmask_mask8x64(mask); + let mut output = [0u8; 64]; + let mut output_lane = 0; + for block in 0..64 / 16 { + let block_bits = ((mask_bits >> (block * 16)) & 0xffff) as usize; + let low_mask = block_bits & 0xff; + let high_mask = block_bits >> 8; + let low_count = low_mask.count_ones() as usize; + let low = crate::support::COMPRESS_8_CONTROLS[low_mask]; + let high = crate::support::COMPRESS_8_CONTROLS[high_mask]; + let high = high | 0x0808_0808_0808_0808; + let mut control = [u8::MAX; 16]; + control[..8].copy_from_slice(&low.to_le_bytes()); + control[low_count..low_count + 8].copy_from_slice(&high.to_le_bytes()); + let input = u8x16::from_slice(self, &values.as_array()[block * 16..block * 16 + 16]); + let control = u8x16::simd_from(self, control); + let compacted: [u8; 16] = self.swizzle_dyn_precise_u8x16(input, control).into(); + let write_len = core::cmp::min(16, 64 - output_lane); + output[output_lane..output_lane + write_len].copy_from_slice(&compacted[..write_len]); + output_lane += low_count + high_mask.count_ones() as usize; + } + let compressed = u8x64::simd_from(self, output); + let prefix_bits = if output_lane == 64 { + u64::MAX + } else { + (1u64 << output_lane) - 1 + }; + let prefix_mask = self.from_bitmask_mask8x64(prefix_bits); + self.select_u8x64(prefix_mask, compressed, merge) + } + #[inline(always)] + fn expand_u8x64(self, values: u8x64, mask: mask8x64) -> u8x64 { + let mask_bits = self.to_bitmask_mask8x64(mask); + let values: [u8; 64] = values.into(); + let mut output = [0u8; 64]; + let mut input_lane = 0; + for block in 0..64 / 16 { + let block_bits = ((mask_bits >> (block * 16)) & 0xffff) as usize; + let low_mask = block_bits & 0xff; + let high_mask = block_bits >> 8; + let low_count = low_mask.count_ones() as usize; + let low = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count as u64 * 0x0101_0101_0101_0101; + let high = high + high_base; + let mut control = [0u8; 16]; + control[..8].copy_from_slice(&low.to_le_bytes()); + control[8..].copy_from_slice(&high.to_le_bytes()); + let mut input = [0u8; 16]; + let read_len = core::cmp::min(16, 64 - input_lane); + input[..read_len].copy_from_slice(&values[input_lane..input_lane + read_len]); + let input = u8x16::simd_from(self, input); + let control = u8x16::simd_from(self, control); + let expanded: [u8; 16] = self.swizzle_dyn_precise_u8x16(input, control).into(); + output[block * 16..block * 16 + 16].copy_from_slice(&expanded); + input_lane += low_count + high_mask.count_ones() as usize; + } + u8x64::simd_from(self, output) + } + #[inline(always)] + fn expand_merge_u8x64( + self, + values: u8x64, + mask: mask8x64, + merge: u8x64, + ) -> u8x64 { + let mask_bits = self.to_bitmask_mask8x64(mask); + let values: [u8; 64] = values.into(); + let mut output = [0u8; 64]; + let mut input_lane = 0; + for block in 0..64 / 16 { + let block_bits = ((mask_bits >> (block * 16)) & 0xffff) as usize; + let low_mask = block_bits & 0xff; + let high_mask = block_bits >> 8; + let low_count = low_mask.count_ones() as usize; + let low = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count as u64 * 0x0101_0101_0101_0101; + let high = high + high_base; + let mut control = [0u8; 16]; + control[..8].copy_from_slice(&low.to_le_bytes()); + control[8..].copy_from_slice(&high.to_le_bytes()); + let mut input = [0u8; 16]; + let read_len = core::cmp::min(16, 64 - input_lane); + input[..read_len].copy_from_slice(&values[input_lane..input_lane + read_len]); + let input = u8x16::simd_from(self, input); + let control = u8x16::simd_from(self, control); + let expanded: [u8; 16] = self.swizzle_dyn_precise_u8x16(input, control).into(); + output[block * 16..block * 16 + 16].copy_from_slice(&expanded); + input_lane += low_count + high_mask.count_ones() as usize; + } + let expanded = u8x64::simd_from(self, output); + self.select_u8x64(mask, expanded, merge) + } + #[inline(always)] fn simd_ne_u8x64(self, a: u8x64, b: u8x64) -> mask8x64 { let (a0, a1) = self.split_u8x64(a); let (b0, b1) = self.split_u8x64(b); @@ -5041,6 +6277,56 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn compress_i16x32(self, values: i16x32, mask: mask16x32) -> i16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i16x32( + self, + values: i16x32, + mask: mask16x32, + merge: i16x32, + ) -> i16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i16x32(self, values: i16x32, mask: mask16x32) -> i16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i16x32( + self, + values: i16x32, + mask: mask16x32, + merge: i16x32, + ) -> i16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn simd_ne_i16x32(self, a: i16x32, b: i16x32) -> mask16x32 { let (a0, a1) = self.split_i16x32(a); let (b0, b1) = self.split_i16x32(b); @@ -5075,6 +6361,56 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn compress_u16x32(self, values: u16x32, mask: mask16x32) -> u16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u16x32( + self, + values: u16x32, + mask: mask16x32, + merge: u16x32, + ) -> u16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u16x32(self, values: u16x32, mask: mask16x32) -> u16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u16x32( + self, + values: u16x32, + mask: mask16x32, + merge: u16x32, + ) -> u16x32 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn simd_ne_u16x32(self, a: u16x32, b: u16x32) -> mask16x32 { let (a0, a1) = self.split_u16x32(a); let (b0, b1) = self.split_u16x32(b); @@ -5172,6 +6508,56 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn compress_i32x16(self, values: i32x16, mask: mask32x16) -> i32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i32x16( + self, + values: i32x16, + mask: mask32x16, + merge: i32x16, + ) -> i32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i32x16(self, values: i32x16, mask: mask32x16) -> i32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i32x16( + self, + values: i32x16, + mask: mask32x16, + merge: i32x16, + ) -> i32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn simd_ne_i32x16(self, a: i32x16, b: i32x16) -> mask32x16 { let (a0, a1) = self.split_i32x16(a); let (b0, b1) = self.split_i32x16(b); @@ -5206,6 +6592,56 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn compress_u32x16(self, values: u32x16, mask: mask32x16) -> u32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u32x16( + self, + values: u32x16, + mask: mask32x16, + merge: u32x16, + ) -> u32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u32x16(self, values: u32x16, mask: mask32x16) -> u32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u32x16( + self, + values: u32x16, + mask: mask32x16, + merge: u32x16, + ) -> u32x16 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn simd_ne_u32x16(self, a: u32x16, b: u32x16) -> mask32x16 { let (a0, a1) = self.split_u32x16(a); let (b0, b1) = self.split_u32x16(b); @@ -5300,6 +6736,56 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn compress_f64x8(self, values: f64x8, mask: mask64x8) -> f64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_f64x8( + self, + values: f64x8, + mask: mask64x8, + merge: f64x8, + ) -> f64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_f64x8(self, values: f64x8, mask: mask64x8) -> f64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_f64x8( + self, + values: f64x8, + mask: mask64x8, + merge: f64x8, + ) -> f64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn simd_ne_f64x8(self, a: f64x8, b: f64x8) -> mask64x8 { let (a0, a1) = self.split_f64x8(a); let (b0, b1) = self.split_f64x8(b); @@ -5366,6 +6852,56 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn compress_i64x8(self, values: i64x8, mask: mask64x8) -> i64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_i64x8( + self, + values: i64x8, + mask: mask64x8, + merge: i64x8, + ) -> i64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_i64x8(self, values: i64x8, mask: mask64x8) -> i64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_i64x8( + self, + values: i64x8, + mask: mask64x8, + merge: i64x8, + ) -> i64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn simd_ne_i64x8(self, a: i64x8, b: i64x8) -> mask64x8 { let (a0, a1) = self.split_i64x8(a); let (b0, b1) = self.split_i64x8(b); @@ -5400,6 +6936,56 @@ impl Simd for WasmSimd128 { }) } #[inline(always)] + fn compress_u64x8(self, values: u64x8, mask: mask64x8) -> u64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn compress_merge_u64x8( + self, + values: u64x8, + mask: mask64x8, + merge: u64x8, + ) -> u64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.compress_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] + fn expand_u64x8(self, values: u64x8, mask: mask64x8) -> u64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask)) + } + #[inline(always)] + fn expand_merge_u64x8( + self, + values: u64x8, + mask: mask64x8, + merge: u64x8, + ) -> u64x8 { + let mask = mask8x64 { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.expand_merge_u8x64( + Bytes::to_bytes(values), + mask, + Bytes::to_bytes(merge), + )) + } + #[inline(always)] fn simd_ne_u64x8(self, a: u64x8, b: u64x8) -> mask64x8 { let (a0, a1) = self.split_u64x8(a); let (b0, b1) = self.split_u64x8(b); diff --git a/fearless_simd/src/support.rs b/fearless_simd/src/support.rs index 2c298326..5a87cae3 100644 --- a/fearless_simd/src/support.rs +++ b/fearless_simd/src/support.rs @@ -63,3 +63,344 @@ pub(crate) fn cross_block_slide_blocks_at( let hi_block = if hi_idx < N { a[hi_idx] } else { b[hi_idx - N] }; [lo_block, hi_block] } + +/// Byte-shuffle controls for whole 16-, 32-, or 64-bit elements in a 128-bit vector. +#[allow( + dead_code, + reason = "Used only by backends with native byte-table shuffles" +)] +pub(crate) mod compact_128 { + use super::Aligned128; + + #[allow( + clippy::cast_possible_truncation, + reason = "Byte indices are less than 16" + )] + const fn controls( + element_bytes: usize, + expand: bool, + ) -> [Aligned128<[u8; 16]>; N] { + assert!( + matches!(element_bytes, 2 | 4 | 8), + "unsupported element size" + ); + let lanes = 16 / element_bytes; + assert!( + N == 1 << lanes, + "one control row is required per selection mask" + ); + let mut table = [Aligned128([0xff; 16]); N]; + let mut mask = 0; + while mask < N { + let mut selected = 0; + let mut lane = 0; + while lane < lanes { + if mask & (1 << lane) != 0 { + let source = if expand { selected } else { lane }; + let destination = if expand { lane } else { selected }; + let mut byte = 0; + while byte < element_bytes { + table[mask].0[destination * element_bytes + byte] = + (source * element_bytes + byte) as u8; + byte += 1; + } + selected += 1; + } + lane += 1; + } + mask += 1; + } + table + } + + pub(crate) const COMPRESS_16: [Aligned128<[u8; 16]>; 256] = controls(2, false); + pub(crate) const EXPAND_16: [Aligned128<[u8; 16]>; 256] = controls(2, true); + pub(crate) const COMPRESS_32: [Aligned128<[u8; 16]>; 16] = controls(4, false); + pub(crate) const EXPAND_32: [Aligned128<[u8; 16]>; 16] = controls(4, true); + pub(crate) const COMPRESS_64: [Aligned128<[u8; 16]>; 4] = controls(8, false); + pub(crate) const EXPAND_64: [Aligned128<[u8; 16]>; 4] = controls(8, true); +} + +/// Dword-permutation controls for whole 32- or 64-bit elements in an AVX2 vector. +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +pub(crate) mod compact_256 { + use super::Aligned256; + + #[allow( + clippy::cast_possible_truncation, + clippy::cast_possible_wrap, + reason = "Dword indices are less than eight" + )] + const fn controls( + element_dwords: usize, + expand: bool, + ) -> [Aligned256<[i32; 8]>; N] { + assert!(matches!(element_dwords, 1 | 2), "unsupported element size"); + let lanes = 8 / element_dwords; + assert!( + N == 1 << lanes, + "one control row is required per selection mask" + ); + // VPERMD only uses the low three index bits. The -1 sentinel separately + // identifies inactive lanes for zero/merge fill after the permutation. + let mut table = [Aligned256([-1; 8]); N]; + let mut mask = 0; + while mask < N { + let mut selected = 0; + let mut lane = 0; + while lane < lanes { + if mask & (1 << lane) != 0 { + let source = if expand { selected } else { lane }; + let destination = if expand { lane } else { selected }; + let mut dword = 0; + while dword < element_dwords { + // Adjacent dwords of a qword always move together. + table[mask].0[destination * element_dwords + dword] = + (source * element_dwords + dword) as i32; + dword += 1; + } + selected += 1; + } + lane += 1; + } + mask += 1; + } + table + } + + // Full dword controls avoid the extra widening/validity work of packed byte + // controls, particularly on Zen 1/3. Signed, unsigned and float ops share them. + pub(crate) static COMPRESS_32: [Aligned256<[i32; 8]>; 256] = controls(1, false); + pub(crate) static EXPAND_32: [Aligned256<[i32; 8]>; 256] = controls(1, true); + pub(crate) static COMPRESS_64: [Aligned256<[i32; 8]>; 16] = controls(2, false); + pub(crate) static EXPAND_64: [Aligned256<[i32; 8]>; 16] = controls(2, true); + + #[allow( + clippy::cast_possible_truncation, + clippy::cast_possible_wrap, + reason = "Dword indices and offsets are at most eight" + )] + const fn shifts() -> [Aligned256<[i32; 8]>; 9] { + let mut table = [Aligned256([0; 8]); 9]; + let mut offset = 0; + while offset <= 8 { + let mut lane = 0; + while lane < 8 { + table[offset].0[lane] = lane as i32 - offset as i32; + lane += 1; + } + offset += 1; + } + table + } + + // VPERMD uses the low three bits to rotate by the offset. Negative controls + // (-8..=-1) have every byte's sign bit set, so the same row is a blend mask + // for the prefix crossing between two YMM registers. Qwords use even offsets. + pub(crate) static SHIFTS: [Aligned256<[i32; 8]>; 9] = shifts(); +} + +const fn compact_control_table(expand: bool) -> [u64; 256] { + // Expansion adds a lane offset to each byte of the packed control. Use 0x80 + // for inactive lanes so these additions preserve the zeroing bit without + // carrying into adjacent bytes. Every supported shuffle treats it as zero. + let empty = if expand { + 0x8080_8080_8080_8080 + } else { + u64::MAX + }; + let mut table = [empty; 256]; + let mut mask = 0_usize; + while mask < table.len() { + let mut control = empty; + let mut selected = 0_usize; + let mut lane = 0_usize; + while lane < 8 { + if mask & (1 << lane) != 0 { + let output_lane = if expand { lane } else { selected }; + control &= !(0xff_u64 << (output_lane * 8)); + let index = if expand { selected } else { lane }; + control |= (index as u64) << (output_lane * 8); + selected += 1; + } + lane += 1; + } + table[mask] = control; + mask += 1; + } + table +} + +const fn compact_count_table() -> [u8; 256] { + let mut table = [0; 256]; + let mut mask = 0_usize; + while mask < table.len() { + let mut bits = mask; + let mut count = 0; + while bits != 0 { + if bits & 1 != 0 { + count += 1; + } + bits >>= 1; + } + table[mask] = count; + mask += 1; + } + table +} + +const fn compact_splice_control_table() -> [u128; 9] { + let mut table = [u128::MAX; 9]; + let mut low_count = 0; + while low_count <= 8 { + let mut control = u128::MAX; + let mut lane = 0; + while lane < low_count { + control &= !(0xff_u128 << (lane * 8)); + control |= (lane as u128) << (lane * 8); + lane += 1; + } + lane = 0; + while lane < 8 { + let output_lane = low_count + lane; + if output_lane < 16 { + control &= !(0xff_u128 << (output_lane * 8)); + control |= ((lane + 8) as u128) << (output_lane * 8); + } + lane += 1; + } + table[low_count] = control; + low_count += 1; + } + table +} + +#[allow( + clippy::cast_possible_truncation, + reason = "lane is bounded to 0..16 by the table-construction loop" +)] +const fn compact_16_splice_control_table() -> [Aligned256<[u8; 32]>; 17] { + let mut table = [Aligned256([0x80; 32]); 17]; + let mut low_count = 0; + while low_count <= 16 { + let mut lane = 0; + while lane < 16 { + let output_lane = low_count + lane; + if output_lane < 32 { + table[low_count].0[output_lane] = lane as u8; + } + lane += 1; + } + low_count += 1; + } + table +} + +const fn compact_prefix_mask_table() -> [Aligned256<[u8; 32]>; 33] { + let mut table = [Aligned256([0; 32]); 33]; + let mut count = 0; + while count <= 32 { + let mut lane = 0; + while lane < count { + table[count].0[lane] = u8::MAX; + lane += 1; + } + count += 1; + } + table +} + +#[derive(Clone, Copy)] +#[repr(C, align(32))] +pub(crate) struct Compact32StitchControls { + pub(crate) left_same: [u8; 32], + pub(crate) left_cross: [u8; 32], + pub(crate) right_same: [u8; 32], + pub(crate) right_cross: [u8; 32], +} + +/// Controls for concatenating two compacted 32-byte vectors entirely in registers. +/// +/// For a first-vector length `count`, `left_*` shifts the second vector left by `count` +/// bytes and `right_*` shifts it right by `32 - count` bytes. Separate same-lane and +/// cross-lane controls account for AVX2 `vpshufb` operating independently in each +/// 128-bit lane. +#[allow( + clippy::cast_possible_truncation, + reason = "shuffle indices are reduced modulo 16" +)] +const fn compact_32_stitch_control_table() -> [Compact32StitchControls; 33] { + const EMPTY: Compact32StitchControls = Compact32StitchControls { + left_same: [0x80; 32], + left_cross: [0x80; 32], + right_same: [0x80; 32], + right_cross: [0x80; 32], + }; + + let mut table = [EMPTY; 33]; + let mut count = 0; + while count <= 32 { + let mut lane = 0; + while lane < 32 { + if lane >= count { + let source = lane - count; + if lane / 16 == source / 16 { + table[count].left_same[lane] = (source % 16) as u8; + } else { + table[count].left_cross[lane] = (source % 16) as u8; + } + } + + let source = lane + (32 - count); + if source < 32 { + if lane / 16 == source / 16 { + table[count].right_same[lane] = (source % 16) as u8; + } else { + table[count].right_cross[lane] = (source % 16) as u8; + } + } + lane += 1; + } + count += 1; + } + table +} + +/// Eight-byte shuffle controls indexed by an eight-bit selection mask. +#[allow( + dead_code, + reason = "Used only by SIMD backends with byte-table shuffles" +)] +pub(crate) const COMPRESS_8_CONTROLS: [u64; 256] = compact_control_table(false); + +/// Eight-byte inverse-shuffle controls indexed by an eight-bit selection mask. +#[allow( + dead_code, + reason = "Used only by SIMD backends with byte-table shuffles" +)] +pub(crate) const EXPAND_8_CONTROLS: [u64; 256] = compact_control_table(true); + +/// Population counts indexed by an eight-bit selection mask. +#[allow( + dead_code, + reason = "Used only by SIMD backends with byte-table shuffles" +)] +pub(crate) const COMPACT_8_COUNTS: [u8; 256] = compact_count_table(); + +/// Shuffle controls that splice two independently compacted eight-byte halves. +#[allow(dead_code, reason = "Used only by x86 byte compression")] +pub(crate) const COMPACT_8_SPLICE_CONTROLS: [u128; 9] = compact_splice_control_table(); + +/// Shuffle controls that append a compacted 16-byte high lane after a compacted low lane. +#[allow(dead_code, reason = "Used only by AVX2 byte compression")] +pub(crate) const COMPACT_16_SPLICE_CONTROLS: [Aligned256<[u8; 32]>; 17] = + compact_16_splice_control_table(); + +/// Shuffle controls that concatenate two independently compacted 32-byte vectors. +#[allow(dead_code, reason = "Used only by AVX2 64-byte compression")] +pub(crate) const COMPACT_32_STITCH_CONTROLS: [Compact32StitchControls; 33] = + compact_32_stitch_control_table(); + +/// Byte prefix masks indexed by the number of active lanes. +#[allow(dead_code, reason = "Used only by x86 byte compression")] +pub(crate) const COMPACT_PREFIX_MASKS: [Aligned256<[u8; 32]>; 33] = compact_prefix_mask_table(); diff --git a/fearless_simd_gen/src/generic.rs b/fearless_simd_gen/src/generic.rs index 1ceb20eb..e130f4d5 100644 --- a/fearless_simd_gen/src/generic.rs +++ b/fearless_simd_gen/src/generic.rs @@ -67,6 +67,34 @@ pub(crate) fn byte_swizzle_op(op: &Op, vec_ty: &VecType) -> TokenStream { } } +/// Forward typed compression/expansion through bytes on vector-backed mask backends. +/// +/// Each all-zero/all-one mask lane becomes a group of identical byte lanes, so entire +/// elements move together. This must not be used with compact predicate masks (AVX-512). +pub(crate) fn byte_compact_op(op: Op, vec_ty: &VecType) -> TokenStream { + assert_ne!(*vec_ty, vec_ty.bytes_ty()); + assert_ne!(vec_ty.scalar, ScalarType::Mask); + let method_sig = op.simd_trait_method_sig(vec_ty); + let byte_method = generic_op_name(op.method, &vec_ty.bytes_ty()); + let byte_mask = vec_ty.bytes_ty().mask_ty().rust(); + let merge_arg = match op.sig { + OpSig::Compress { merge: true } | OpSig::Expand { merge: true } => { + quote! { , Bytes::to_bytes(merge) } + } + OpSig::Compress { merge: false } | OpSig::Expand { merge: false } => TokenStream::new(), + _ => unreachable!("only compact operations can be forwarded through bytes"), + }; + quote! { + #method_sig { + let mask = #byte_mask { + val: crate::transmute::checked_transmute_copy(&mask.val), + simd: self, + }; + Bytes::from_bytes(self.#byte_method(Bytes::to_bytes(values), mask #merge_arg)) + } + } +} + /// Implement a greater-than comparison by reversing the corresponding less-than comparison. pub(crate) fn reversed_compare_op(op: &Op, vec_ty: &VecType) -> Option { let reversed_method = generic_op_name(op.reversed_compare_method()?, vec_ty); @@ -353,6 +381,69 @@ pub(crate) fn generic_op(op: &Op, ty: &VecType) -> TokenStream { | OpSig::ConcatSwizzleDynPrecise => { panic!("whole-vector swizzles cannot be done via split/combine"); } + OpSig::Compress { merge: false } => { + let merge_method = generic_op_name("compress_merge", ty); + let ty = ty.rust(); + quote! { + #method_sig { + self.#merge_method(values, mask, #ty::splat(self, 0)) + } + } + } + OpSig::Compress { merge: true } => { + let len = Literal::usize_unsuffixed(ty.len); + let to_bitmask = generic_op_name("to_bitmask", &ty.mask_ty()); + quote! { + #method_sig { + // Branchless: every lane is written at the cursor, which only advances on + // selected lanes. The one slot past the selected lanes may be clobbered by an + // unselected lane, so it is restored from `merge` afterwards. + let mask = self.#to_bitmask(mask); + let mut merge_padded = [0u8; #len + 1]; + merge_padded[..#len].copy_from_slice(&*merge); + let mut compacted = merge_padded; + let mut output_lane = 0; + for input_lane in 0..#len { + compacted[output_lane] = values[input_lane]; + output_lane += ((mask >> input_lane) & 1) as usize; + } + compacted[output_lane] = merge_padded[output_lane]; + let mut result = merge; + result.copy_from_slice(&compacted[..#len]); + result + } + } + } + OpSig::Expand { merge: false } => { + let merge_method = generic_op_name("expand_merge", ty); + let ty = ty.rust(); + quote! { + #method_sig { + self.#merge_method(values, mask, #ty::splat(self, 0)) + } + } + } + OpSig::Expand { merge: true } => { + let len = Literal::usize_unsuffixed(ty.len); + let to_bitmask = generic_op_name("to_bitmask", &ty.mask_ty()); + quote! { + #method_sig { + // Branchless: the input cursor never passes the output lane, so the read is + // always in bounds and selected with a lane mask instead of a branch. + let mask = self.#to_bitmask(mask); + let mut result = merge; + let mut input_lane = 0; + for output_lane in 0..#len { + let bit = ((mask >> output_lane) & 1) as u8; + let keep = 0u8.wrapping_sub(bit); + result[output_lane] = + (values[input_lane] & keep) | (result[output_lane] & !keep); + input_lane += usize::from(bit); + } + result + } + } + } OpSig::Ternary => { quote! { #method_sig { @@ -597,6 +688,326 @@ pub(crate) fn generic_op(op: &Op, ty: &VecType) -> TokenStream { } } +pub(crate) type CompactMergeSwizzle = + fn(&VecType, &TokenStream, &TokenStream, &TokenStream) -> TokenStream; + +/// Compact whole elements with one table lookup and one native 128-bit byte shuffle. +pub(crate) fn compact_128_op( + op: Op, + ty: &VecType, + merge_swizzle: Option, +) -> TokenStream { + assert_eq!(ty.n_bits(), 128, "compact tables cover one 128-bit vector"); + assert!( + matches!(ty.scalar_bits, 16 | 32 | 64), + "compact tables require wider elements" + ); + let (expand, merge) = match op.sig { + OpSig::Compress { merge } => (false, merge), + OpSig::Expand { merge } => (true, merge), + _ => unreachable!("compact tables only implement compress/expand"), + }; + let method_sig = op.simd_trait_method_sig(ty); + let to_bitmask = generic_op_name("to_bitmask", &ty.mask_ty()); + let table = Ident::new( + &format!( + "{}_{}", + if expand { "EXPAND" } else { "COMPRESS" }, + ty.scalar_bits + ), + Span::call_site(), + ); + let lane_mask = Literal::usize_unsuffixed((1 << ty.len) - 1); + let finish = if !merge { + quote! { + Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control)) + } + } else if let Some(merge_swizzle) = merge_swizzle { + let result = merge_swizzle( + &ty.bytes_ty(), + "e! { Bytes::to_bytes(values) }, + "e! { control }, + "e! { Bytes::to_bytes(merge) }, + ); + quote! { Bytes::from_bytes(#result) } + } else if expand { + let select = generic_op_name("select", ty); + quote! { + let expanded = Bytes::from_bytes( + self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control), + ); + self.#select(mask, expanded, merge) + } + } else { + quote! { + let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control); + let inactive = self.simd_lt_i8x16( + i8x16::from_bytes(control), i8x16::splat(self, 0), + ); + Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed)) + } + }; + // Merge forms may use the imprecise shuffle: every out-of-range result is + // replaced by a merge byte. Compression's 0xff controls also identify its tail. + quote! { + #method_sig { + let bits = self.#to_bitmask(mask) as usize & #lane_mask; + let control = u8x16::simd_from(self, crate::support::compact_128::#table[bits].0); + #finish + } + } +} + +/// Backend capabilities used by [`composed_compact_op`]. +pub(crate) struct CompactOptions { + /// Compact wide vectors a native 128-bit block at a time and splice them through memory. + pub(crate) splice_wide_vectors: bool, + /// Use the target's scalar population-count instruction instead of the byte-count table. + pub(crate) hardware_popcount: bool, + /// Optional backend operation that combines an out-of-range-zeroing shuffle with its merge + /// operand. The arguments are `(vector type, values, control, merge)`. + pub(crate) merge_swizzle: Option, +} + +/// Implement byte compression and expansion in terms of a backend's optimized whole-vector +/// swizzle, compact mask conversion, and selection operations. +pub(crate) fn composed_compact_op(op: Op, ty: &VecType, options: CompactOptions) -> TokenStream { + assert_eq!( + ty.scalar, + ScalarType::Unsigned, + "compact operations require unsigned vectors" + ); + assert_eq!(ty.scalar_bits, 8, "compact operations require byte lanes"); + + let method_sig = op.simd_trait_method_sig(ty); + let vec = ty.rust(); + let len = Literal::usize_unsuffixed(ty.len); + let swizzle = generic_op_name("swizzle_dyn_precise", ty); + let to_bitmask = generic_op_name("to_bitmask", &ty.mask_ty()); + let from_bitmask = generic_op_name("from_bitmask", &ty.mask_ty()); + let select = generic_op_name("select", ty); + let count_8 = |mask: TokenStream| { + if options.hardware_popcount { + quote! { #mask.count_ones() as usize } + } else { + quote! { crate::support::COMPACT_8_COUNTS[#mask] as usize } + } + }; + + // Backends whose byte shuffle is confined to 128-bit blocks can avoid an expensive emulated + // whole-vector shuffle by compacting each native block and splicing through a stack buffer. + // This is the same broad strategy used by Highway on targets without native byte + // compress/expand instructions. + if options.splice_wide_vectors && ty.len > 16 { + assert!( + options.merge_swizzle.is_none(), + "block-spliced compact operations do not support a merging shuffle" + ); + let block_vec = VecType { + scalar: ScalarType::Unsigned, + scalar_bits: 8, + len: 16, + }; + let block_swizzle = generic_op_name("swizzle_dyn_precise", &block_vec); + + return match op.sig { + OpSig::Compress { merge } => { + let low_count = count_8(quote! { low_mask }); + let high_count = count_8(quote! { high_mask }); + let finish = if merge { + quote! { + let compressed = #vec::simd_from(self, output); + let prefix_bits = if output_lane == 64 { + u64::MAX + } else { + (1u64 << output_lane) - 1 + }; + let prefix_mask = self.#from_bitmask(prefix_bits); + self.#select(prefix_mask, compressed, merge) + } + } else { + quote! { #vec::simd_from(self, output) } + }; + quote! { + #method_sig { + let mask_bits = self.#to_bitmask(mask); + let mut output = [0u8; #len]; + let mut output_lane = 0; + for block in 0..#len / 16 { + let block_bits = ((mask_bits >> (block * 16)) & 0xffff) as usize; + let low_mask = block_bits & 0xff; + let high_mask = block_bits >> 8; + let low_count = #low_count; + let low = crate::support::COMPRESS_8_CONTROLS[low_mask]; + let high = crate::support::COMPRESS_8_CONTROLS[high_mask]; + // Selected indices are 0..7; OR adds eight while + // leaving inactive 0xff controls out of range. + let high = high | 0x0808_0808_0808_0808; + let mut control = [u8::MAX; 16]; + control[..8].copy_from_slice(&low.to_le_bytes()); + control[low_count..low_count + 8] + .copy_from_slice(&high.to_le_bytes()); + let input = u8x16::from_slice( + self, + &values.as_array()[block * 16..block * 16 + 16], + ); + let control = u8x16::simd_from(self, control); + let compacted: [u8; 16] = + self.#block_swizzle(input, control).into(); + let write_len = core::cmp::min(16, #len - output_lane); + output[output_lane..output_lane + write_len] + .copy_from_slice(&compacted[..write_len]); + output_lane += low_count + #high_count; + } + #finish + } + } + } + OpSig::Expand { merge } => { + let low_count = count_8(quote! { low_mask }); + let high_count = count_8(quote! { high_mask }); + let finish = if merge { + quote! { + let expanded = #vec::simd_from(self, output); + self.#select(mask, expanded, merge) + } + } else { + quote! { #vec::simd_from(self, output) } + }; + quote! { + #method_sig { + let mask_bits = self.#to_bitmask(mask); + let values: [u8; #len] = values.into(); + let mut output = [0u8; #len]; + let mut input_lane = 0; + for block in 0..#len / 16 { + let block_bits = ((mask_bits >> (block * 16)) & 0xffff) as usize; + let low_mask = block_bits & 0xff; + let high_mask = block_bits >> 8; + let low_count = #low_count; + let low = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count as u64 * 0x0101_0101_0101_0101; + // Adding at most eight preserves the inactive 0x80 + // controls' high bit without carrying between bytes. + let high = high + high_base; + let mut control = [0u8; 16]; + control[..8].copy_from_slice(&low.to_le_bytes()); + control[8..].copy_from_slice(&high.to_le_bytes()); + let mut input = [0u8; 16]; + let read_len = core::cmp::min(16, #len - input_lane); + input[..read_len] + .copy_from_slice(&values[input_lane..input_lane + read_len]); + let input = u8x16::simd_from(self, input); + let control = u8x16::simd_from(self, control); + let expanded: [u8; 16] = + self.#block_swizzle(input, control).into(); + output[block * 16..block * 16 + 16].copy_from_slice(&expanded); + input_lane += low_count + #high_count; + } + #finish + } + } + } + _ => unreachable!("composed_compact_op only implements compact byte operations"), + }; + } + + match op.sig { + OpSig::Compress { merge } => { + let block_count = count_8(quote! { block_mask }); + let finish = if merge { + if let Some(merge_swizzle) = options.merge_swizzle { + merge_swizzle( + ty, + "e! { values }, + "e! { control }, + "e! { merge }, + ) + } else { + quote! { + let compressed = self.#swizzle(values, control); + let prefix_bits = if output_lane == 64 { + u64::MAX + } else { + (1u64 << output_lane) - 1 + }; + let prefix_mask = self.#from_bitmask(prefix_bits); + self.#select(prefix_mask, compressed, merge) + } + } + } else { + quote! { self.#swizzle(values, control) } + }; + quote! { + #method_sig { + let mask_bits = self.#to_bitmask(mask); + let mut control = [u8::MAX; #len]; + let mut output_lane = 0; + for block in 0..#len / 8 { + let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; + let packed = crate::support::COMPRESS_8_CONTROLS[block_mask]; + let base = (block * 8) as u64 * 0x0101_0101_0101_0101; + // Block offsets are multiples of eight, disjoint from + // indices 0..7. Inactive 0xff controls stay out of range. + let adjusted = packed | base; + let adjusted = adjusted.to_le_bytes(); + let write_len = core::cmp::min(8, #len - output_lane); + control[output_lane..output_lane + write_len] + .copy_from_slice(&adjusted[..write_len]); + output_lane += #block_count; + } + let control = #vec::simd_from(self, control); + #finish + } + } + } + OpSig::Expand { merge } => { + let block_count = count_8(quote! { block_mask }); + let finish = if merge { + if let Some(merge_swizzle) = options.merge_swizzle { + merge_swizzle( + ty, + "e! { values }, + "e! { control }, + "e! { merge }, + ) + } else { + quote! { + let expanded = self.#swizzle(values, control); + self.#select(mask, expanded, merge) + } + } + } else { + quote! { self.#swizzle(values, control) } + }; + quote! { + #method_sig { + let mask_bits = self.#to_bitmask(mask); + let mut control = [u8::MAX; #len]; + let mut input_lane = 0; + for block in 0..#len / 8 { + let block_mask = ((mask_bits >> (block * 8)) & 0xff) as usize; + let packed = crate::support::EXPAND_8_CONTROLS[block_mask]; + let base = input_lane as u64 * 0x0101_0101_0101_0101; + // The offset is at most 56 for a 64-byte vector. Adding + // it to inactive 0x80 controls keeps them out of range + // and cannot carry between bytes. + let adjusted = packed + base; + let output_lane = block * 8; + control[output_lane..output_lane + 8] + .copy_from_slice(&adjusted.to_le_bytes()); + input_lane += #block_count; + } + let control = #vec::simd_from(self, control); + #finish + } + } + } + _ => unreachable!("composed_compact_op only implements compact byte operations"), + } +} + pub(crate) fn unrolled_array(len: usize, item: impl FnMut(usize) -> TokenStream) -> TokenStream { let items = (0..len).map(item).collect::>(); quote! { [#(#items),*] } diff --git a/fearless_simd_gen/src/mk_fallback.rs b/fearless_simd_gen/src/mk_fallback.rs index 125a5e7c..5d3ad932 100644 --- a/fearless_simd_gen/src/mk_fallback.rs +++ b/fearless_simd_gen/src/mk_fallback.rs @@ -3,8 +3,9 @@ use crate::arch::fallback; use crate::generic::{ - generic_classify, generic_mask_from_bitmask, generic_mask_set, generic_mask_to_bitmask, - generic_op_name, integer_lane_mask_rotate, integer_lane_mask_splat_arg, + byte_compact_op, generic_classify, generic_mask_from_bitmask, generic_mask_set, + generic_mask_to_bitmask, generic_op, generic_op_name, integer_lane_mask_rotate, + integer_lane_mask_splat_arg, }; use crate::level::Level; use crate::ops::{NarrowingMode, Op, OpSig, relaxed_narrow_method}; @@ -591,6 +592,10 @@ impl Level for Fallback { } } } + OpSig::Compress { .. } | OpSig::Expand { .. } if *vec_ty != vec_ty.bytes_ty() => { + byte_compact_op(op, vec_ty) + } + OpSig::Compress { .. } | OpSig::Expand { .. } => generic_op(&op, vec_ty), OpSig::Cvt { target_ty, scalar_bits, diff --git a/fearless_simd_gen/src/mk_neon.rs b/fearless_simd_gen/src/mk_neon.rs index f40c5910..8c76c555 100644 --- a/fearless_simd_gen/src/mk_neon.rs +++ b/fearless_simd_gen/src/mk_neon.rs @@ -5,7 +5,8 @@ use proc_macro2::{Ident, Literal, Span, TokenStream}; use quote::{ToTokens as _, format_ident, quote}; use crate::generic::{ - count_zeros_method, fallback_method, generic_classify, generic_mask_set, generic_op_name, + CompactOptions, byte_compact_op, compact_128_op, composed_compact_op, count_zeros_method, + fallback_method, generic_classify, generic_mask_set, generic_op_name, generic_to_degrees_radians, integer_lane_mask_rotate, integer_lane_mask_splat_arg, reverse_method, reverse_vector_mask_method, }; @@ -21,6 +22,62 @@ use crate::{ pub(crate) struct Neon; impl Neon { + fn compact_merge_swizzle( + vec_ty: &VecType, + values: &TokenStream, + control: &TokenStream, + merge: &TokenStream, + ) -> TokenStream { + let vec = vec_ty.rust(); + let wrapper = vec_ty.aligned_wrapper(); + let arch_ty = Self.arch_ty(vec_ty); + + let body = match vec_ty.n_bits() { + 128 => quote! { + let result = vqtbx1q_u8(merge, values, control); + }, + 256 => quote! { + let result = uint8x16x2_t( + vqtbx2q_u8(merge.0, values, control.0), + vqtbx2q_u8(merge.1, values, control.1), + ); + }, + 512 => quote! { + let result = uint8x16x4_t( + vqtbx4q_u8(merge.0, values, control.0), + vqtbx4q_u8(merge.1, values, control.1), + vqtbx4q_u8(merge.2, values, control.2), + vqtbx4q_u8(merge.3, values, control.3), + ); + }, + _ => unreachable!(), + }; + + quote! { + { + crate::kernel!( + #[inline(always)] + fn merge_swizzle( + token: Neon, + values: #vec, + control: #vec, + merge: #vec, + ) -> #vec { + let values: #arch_ty = values.into(); + let control: #arch_ty = control.into(); + let merge: #arch_ty = merge.into(); + #body + #vec { + val: #wrapper(result), + simd: token, + } + } + ); + merge_swizzle(self, #values, #control, #merge) + } + } + } + fn handle_count_ones(&self, op: Op, vec_ty: &VecType) -> TokenStream { let input = match vec_ty.scalar { ScalarType::Unsigned if vec_ty.scalar_bits == 8 => quote! { a.into() }, @@ -798,6 +855,23 @@ impl Level for Neon { } }) } + OpSig::Compress { .. } | OpSig::Expand { .. } + if vec_ty.n_bits() == 128 && matches!(vec_ty.scalar_bits, 16 | 32 | 64) => + { + compact_128_op(op, vec_ty, Some(Self::compact_merge_swizzle)) + } + OpSig::Compress { .. } | OpSig::Expand { .. } if *vec_ty != vec_ty.bytes_ty() => { + byte_compact_op(op, vec_ty) + } + OpSig::Compress { .. } | OpSig::Expand { .. } => composed_compact_op( + op, + vec_ty, + CompactOptions { + splice_wide_vectors: false, + hardware_popcount: false, + merge_swizzle: Some(Self::compact_merge_swizzle), + }, + ), OpSig::Cvt { target_ty, scalar_bits, diff --git a/fearless_simd_gen/src/mk_wasm.rs b/fearless_simd_gen/src/mk_wasm.rs index ecb3c57a..c8c30778 100644 --- a/fearless_simd_gen/src/mk_wasm.rs +++ b/fearless_simd_gen/src/mk_wasm.rs @@ -6,6 +6,7 @@ use quote::{format_ident, quote}; use crate::arch::wasm::{arch_prefix, v128_intrinsic}; use crate::generic::{ + CompactOptions, byte_compact_op, compact_128_op, composed_compact_op, concat_swizzle_dyn_precise_body, count_zeros_method, fallback_method, generic_block_combine, generic_block_split, generic_classify, generic_mask_set, generic_op_name, generic_round, generic_to_degrees_radians, integer_lane_mask_rotate, integer_lane_mask_splat_arg, @@ -1188,6 +1189,23 @@ impl Level for WasmSimd128 { } _ => unreachable!(), }, + OpSig::Compress { .. } | OpSig::Expand { .. } + if vec_ty.n_bits() == 128 && matches!(vec_ty.scalar_bits, 16 | 32 | 64) => + { + compact_128_op(op, vec_ty, None) + } + OpSig::Compress { .. } | OpSig::Expand { .. } if *vec_ty != vec_ty.bytes_ty() => { + byte_compact_op(op, vec_ty) + } + OpSig::Compress { .. } | OpSig::Expand { .. } => composed_compact_op( + op, + vec_ty, + CompactOptions { + splice_wide_vectors: true, + hardware_popcount: true, + merge_swizzle: None, + }, + ), OpSig::Cvt { target_ty, scalar_bits, diff --git a/fearless_simd_gen/src/mk_x86.rs b/fearless_simd_gen/src/mk_x86.rs index adc059c2..3a9b1fd8 100644 --- a/fearless_simd_gen/src/mk_x86.rs +++ b/fearless_simd_gen/src/mk_x86.rs @@ -7,11 +7,11 @@ use crate::arch::x86::{ unpack_intrinsic, }; use crate::generic::{ - concat_swizzle_dyn_precise_body, count_zeros_method, fallback_method, generic_block_combine, - generic_block_split, generic_classify, generic_mask_from_bitmask, generic_mask_set, - generic_op_name, generic_round, generic_to_degrees_radians, integer_lane_mask_rotate, - integer_lane_mask_splat_arg, recursive_swizzle_dyn_precise_body, reverse_method, - reverse_vector_mask_method, + byte_compact_op, compact_128_op, concat_swizzle_dyn_precise_body, count_zeros_method, + fallback_method, generic_block_combine, generic_block_split, generic_classify, + generic_mask_from_bitmask, generic_mask_set, generic_op, generic_op_name, generic_round, + generic_to_degrees_radians, integer_lane_mask_rotate, integer_lane_mask_splat_arg, + recursive_swizzle_dyn_precise_body, reverse_method, reverse_vector_mask_method, }; use crate::level::Level; use crate::ops::{ @@ -333,6 +333,36 @@ impl Level for X86 { OpSig::SwizzleDynPrecise => self.handle_swizzle_dyn_precise(op, vec_ty), OpSig::ConcatSwizzleDyn => self.handle_concat_swizzle_dyn(op, vec_ty), OpSig::ConcatSwizzleDynPrecise => self.handle_concat_swizzle_dyn_precise(op, vec_ty), + OpSig::Compress { .. } | OpSig::Expand { .. } => { + if *self == Self::Avx512 { + self.handle_avx512_compact_op(op, vec_ty) + } else if *self == Self::Avx2 + && vec_ty.n_bits() == 512 + && matches!(vec_ty.scalar_bits, 32 | 64) + { + self.handle_avx2_wide_dword_compact(op, vec_ty) + } else if *self == Self::Avx2 + && vec_ty.n_bits() == 256 + && matches!(vec_ty.scalar_bits, 32 | 64) + { + self.handle_avx2_dword_compact(op, vec_ty) + } else if matches!(*self, Self::Sse4_2 | Self::Avx2) + && vec_ty.n_bits() == 128 + && matches!(vec_ty.scalar_bits, 16 | 32 | 64) + { + compact_128_op(op, vec_ty, None) + } else if *vec_ty != vec_ty.bytes_ty() { + byte_compact_op(op, vec_ty) + } else if matches!(*self, Self::Sse4_2 | Self::Avx2) && vec_ty.len == 16 { + self.handle_x86_compact_16(op, vec_ty) + } else if *self == Self::Avx2 && matches!(sig, OpSig::Compress { .. }) { + self.handle_avx2_shuffle_compact(op, vec_ty) + } else if matches!(*self, Self::Sse4_2 | Self::Avx2) { + self.handle_x86_wide_compact(op, vec_ty) + } else { + generic_op(&op, vec_ty) + } + } OpSig::Cvt { target_ty, scalar_bits, @@ -1234,7 +1264,684 @@ fn interleaved_store_indices(len: usize, block_count: usize) -> Vec { .collect() } +fn shifted_mask_bits(shift: usize) -> TokenStream { + if shift == 0 { + quote! { mask_bits } + } else { + let shift = Literal::usize_unsuffixed(shift); + quote! { mask_bits >> #shift } + } +} + +/// Emit the `PSHUFB` control construction for one 16-byte compact block. +fn compact_16_control(block_bits: TokenStream, suffix: &str) -> (TokenStream, Ident, Ident) { + let low_mask = format_ident!("low_mask{suffix}"); + let high_mask = format_ident!("high_mask{suffix}"); + let low_control = format_ident!("low_control{suffix}"); + let high_control = format_ident!("high_control{suffix}"); + let control = format_ident!("control{suffix}"); + let low_count = format_ident!("low_count{suffix}"); + let setup = quote! { + let #low_mask = (#block_bits) & 0xff; + let #high_mask = (#block_bits) >> 8; + let #low_control = crate::support::COMPRESS_8_CONTROLS[#low_mask]; + let #high_control = crate::support::COMPRESS_8_CONTROLS[#high_mask]; + // Selected indices are 0..=7; setting bit 3 offsets them by eight. + // Inactive indices retain their high bit and still zero the shuffled byte. + let #high_control = #high_control | 0x0808_0808_0808_0808; + let #control = _mm_set_epi64x(#high_control.cast_signed(), #low_control.cast_signed()); + let #low_count = #low_mask.count_ones() as usize; + }; + (setup, control, low_count) +} + +/// Emit the `PSHUFB` control construction for one 16-byte expand block. +fn expand_16_control(block_bits: TokenStream) -> TokenStream { + quote! { + let low_mask = (#block_bits) & 0xff; + let high_mask = (#block_bits) >> 8; + let low_count = low_mask.count_ones() as u64; + let low_control = crate::support::EXPAND_8_CONTROLS[low_mask]; + let high_control = crate::support::EXPAND_8_CONTROLS[high_mask]; + let high_base = low_count * 0x0101_0101_0101_0101; + // Inactive controls are 0x80; adding at most eight cannot carry between + // bytes or clear the zeroing bit. Selected controls remain in 0..=15. + let high_control = high_control + high_base; + let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed()); + } +} + impl X86 { + /// Reuse native dword/qword compaction, joining the halves with register permutations. + fn handle_avx2_wide_dword_compact(&self, op: Op, vec_ty: &VecType) -> TokenStream { + assert!(*self == Self::Avx2, "wide dword compaction requires AVX2"); + assert_eq!(vec_ty.n_bits(), 512, "requires two native AVX2 vectors"); + assert!( + matches!(vec_ty.scalar_bits, 32 | 64), + "requires dword/qword elements" + ); + let vec = vec_ty.rust(); + let half = vec_ty.split_operand().unwrap().rust(); + let low_dwords = if vec_ty.scalar_bits == 32 { + quote! { low_count } + } else { + quote! { low_count * 2 } + }; + let split_mask = generic_op_name("split", &vec_ty.mask_ty()); + self.kernel_method(op, vec_ty, |token| { + let body = match op.sig { + OpSig::Compress { merge } => { + let empty = if merge { + quote! { merge } + } else if vec_ty.scalar == ScalarType::Float { + quote! { #vec::splat(#token, 0.0) } + } else { + quote! { #vec::splat(#token, 0) } + }; + let all = Literal::u64_unsuffixed((1_u64 << vec_ty.len) - 1); + let compressed = quote! { + #vec { + val: crate::transmute::checked_transmute_copy(&[result_low, result_high]), + simd: #token, + } + }; + let finish = if merge { + let mask_ty = vec_ty.mask_ty().rust(); + let select = generic_op_name("select", vec_ty); + quote! { + let compressed = #compressed; + let count = low_count + high_mask.to_bitmask().count_ones() as usize; + let prefix = #mask_ty::from_bitmask(#token, (1u64 << count) - 1); + #token.#select(prefix, compressed, merge) + } + } else { + compressed + }; + quote! { + let bits = mask.to_bitmask(); + if bits == 0 { return #empty; } + if bits == #all { return values; } + let low = low.compress(low_mask); + let high = high.compress(high_mask); + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::SHIFTS[#low_dwords], + ); + let low_bits = crate::transmute::checked_transmute_copy(&low.val); + let high_bits = crate::transmute::checked_transmute_copy(&high.val); + let shifted = _mm256_permutevar8x32_epi32(high_bits, control); + // Negative controls identify the low half's selected prefix. + // The rotated high half fills the rest, spilling its tail + // into that same prefix of the second output register. + let result_low = _mm256_blendv_epi8(shifted, low_bits, control); + let result_high = _mm256_and_si256(shifted, _mm256_srai_epi32::<31>(control)); + #finish + } + } + OpSig::Expand { merge } => { + let finish = if merge { + quote! { + let (low_merge, high_merge) = merge.split(); + low.expand_merge(low_mask, low_merge) + .combine(high.expand_merge(high_mask, high_merge)) + } + } else { + quote! { low.expand(low_mask).combine(high.expand(high_mask)) } + }; + quote! { + // Assemble the packed window starting after the elements + // consumed by the low half. The same controls rotate both + // inputs; negative indices select from the low register. + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::SHIFTS[8 - (#low_dwords)], + ); + let low_bits = crate::transmute::checked_transmute_copy(&low.val); + let high_bits = crate::transmute::checked_transmute_copy(&high.val); + let from_low = _mm256_permutevar8x32_epi32(low_bits, control); + let from_high = _mm256_permutevar8x32_epi32(high_bits, control); + let packed = _mm256_blendv_epi8(from_high, from_low, control); + let high = #half { + val: crate::transmute::checked_transmute_copy(&packed), + simd: #token, + }; + #finish + } + } + _ => unreachable!("only compact operations reuse the native halves"), + }; + quote! { + let (low_mask, high_mask) = #token.#split_mask(mask); + let low_count = low_mask.to_bitmask().count_ones() as usize; + let (low, high) = values.split(); + #body + } + }) + } + + /// Compact whole dwords/qwords with a single full-width AVX2 permutation. + fn handle_avx2_dword_compact(&self, op: Op, vec_ty: &VecType) -> TokenStream { + assert!(*self == Self::Avx2, "dword compaction requires AVX2"); + assert_eq!(vec_ty.n_bits(), 256, "requires one native AVX2 vector"); + assert!( + matches!(vec_ty.scalar_bits, 32 | 64), + "requires dword/qword elements" + ); + let (expand, merge) = match op.sig { + OpSig::Compress { merge } => (false, merge), + OpSig::Expand { merge } => (true, merge), + _ => unreachable!("only compact operations use dword permutations"), + }; + let table = format_ident!( + "{}_{}", + if expand { "EXPAND" } else { "COMPRESS" }, + vec_ty.scalar_bits, + ); + let to_bitmask = generic_op_name("to_bitmask", &vec_ty.mask_ty()); + let lane_mask = Literal::usize_unsuffixed((1 << vec_ty.len) - 1); + let to_int = |value: TokenStream| { + if vec_ty.scalar == ScalarType::Float { + let cast = cast_ident( + ScalarType::Float, + ScalarType::Int, + vec_ty.scalar_bits, + vec_ty.scalar_bits, + 256, + ); + quote! { #cast(#value.into()) } + } else { + quote! { #value.into() } + } + }; + let values = to_int(quote! { values }); + let merge_value = to_int(quote! { merge }); + let finish = match (expand, merge) { + (false, false) => quote! { + _mm256_andnot_si256(_mm256_srai_epi32::<31>(control), reordered) + }, + // Valid controls 0..7 have no byte sign bits; -1 selects the merge. + (false, true) => quote! { _mm256_blendv_epi8(reordered, #merge_value, control) }, + // Expansion writes exactly the original mask lanes. + (true, false) => quote! { _mm256_and_si256(reordered, mask.into()) }, + (true, true) => quote! { _mm256_blendv_epi8(#merge_value, reordered, mask.into()) }, + }; + let result = if vec_ty.scalar == ScalarType::Float { + let cast = cast_ident( + ScalarType::Int, + ScalarType::Float, + vec_ty.scalar_bits, + vec_ty.scalar_bits, + 256, + ); + quote! { #cast(result) } + } else { + quote! { result } + }; + self.kernel_method(op, vec_ty, |token| { + quote! { + let bits = #token.#to_bitmask(mask) as usize & #lane_mask; + let control: __m256i = crate::transmute::checked_transmute_copy( + &crate::support::compact_256::#table[bits], + ); + let reordered = _mm256_permutevar8x32_epi32(#values, control); + let result = #finish; + #result.simd_into(#token) + } + }) + } + + fn handle_x86_compact_16(&self, op: Op, vec_ty: &VecType) -> TokenStream { + assert!( + matches!(*self, Self::Sse4_2 | Self::Avx2), + "16-byte compact shuffles require SSE4.2 or AVX2" + ); + assert_eq!(vec_ty.len, 16, "this handler is only for 16-byte vectors"); + let to_bitmask = generic_op_name("to_bitmask", &vec_ty.mask_ty()); + + self.kernel_method(op, vec_ty, |token| match op.sig { + OpSig::Compress { merge } => { + let (setup, control, low_count) = + compact_16_control(quote! { usize::from(mask_bits) }, ""); + let finish = if merge { + quote! { + let count = mask_bits.count_ones() as usize; + let [prefix, _] = crate::transmute::checked_transmute_copy::<_, [__m128i; 2]>( + &crate::support::COMPACT_PREFIX_MASKS[count], + ); + _mm_blendv_epi8(merge.into(), compressed, prefix).simd_into(#token) + } + } else { + quote! { compressed.simd_into(#token) } + }; + quote! { + let mask_bits = #token.#to_bitmask(mask) as u16; + #setup + let compacted = _mm_shuffle_epi8(values.into(), #control); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[#low_count]; + let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let compressed = _mm_shuffle_epi8(compacted, splice); + #finish + } + } + OpSig::Expand { merge } => { + let setup = expand_16_control(quote! { usize::from(mask_bits) }); + let finish = if merge { + quote! { + _mm_blendv_epi8(merge.into(), expanded, mask.into()).simd_into(#token) + } + } else { + quote! { expanded.simd_into(#token) } + }; + quote! { + let mask_bits = #token.#to_bitmask(mask) as u16; + #setup + let expanded = _mm_shuffle_epi8(values.into(), control); + #finish + } + } + _ => unreachable!("only compact operations use the x86 implementation"), + }) + } + + fn handle_x86_wide_compact(&self, op: Op, vec_ty: &VecType) -> TokenStream { + assert!( + matches!(*self, Self::Sse4_2 | Self::Avx2), + "wide x86 compact requires SSE4.2 or AVX2" + ); + assert!( + matches!(vec_ty.len, 32 | 64), + "wide x86 compact requires a 32- or 64-byte vector" + ); + let vec = vec_ty.rust(); + let len = Literal::usize_unsuffixed(vec_ty.len); + let block_count = vec_ty.len / 16; + let to_bitmask = generic_op_name("to_bitmask", &vec_ty.mask_ty()); + let block_value = |name: &str, block: usize| { + let name = format_ident!("{name}"); + match (*self, vec_ty.len) { + (Self::Sse4_2, _) => { + let block = Literal::usize_unsuffixed(block); + quote! { #name.val.0[#block] } + } + (Self::Avx2, 32) if block == 0 => { + quote! { _mm256_castsi256_si128(#name.val.0) } + } + (Self::Avx2, 32) => quote! { _mm256_extracti128_si256::<1>(#name.val.0) }, + (Self::Avx2, 64) => { + let half = Literal::usize_unsuffixed(block / 2); + if block.is_multiple_of(2) { + quote! { _mm256_castsi256_si128(#name.val.0[#half]) } + } else { + quote! { _mm256_extracti128_si256::<1>(#name.val.0[#half]) } + } + } + _ => unreachable!(), + } + }; + + self.kernel_method(op, vec_ty, |token| match op.sig { + OpSig::Compress { merge } => { + let compact_blocks = (0..block_count).map(|block| { + let shifted_mask = shifted_mask_bits(block * 16); + let values = block_value("values", block); + let (setup, control, low_count) = compact_16_control(quote! { block_bits }, ""); + let advance = if merge || block + 1 < block_count { + quote! { output_lane += block_bits.count_ones() as usize; } + } else { + TokenStream::new() + }; + quote! { + let block_bits = (#shifted_mask & 0xffff) as usize; + #setup + let compacted = _mm_shuffle_epi8(#values, #control); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[#low_count]; + let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let compressed = _mm_shuffle_epi8(compacted, splice); + crate::transmute::checked_transmute_store( + compressed, + output[output_lane..].first_chunk_mut::<16>().unwrap(), + ); + #advance + } + }); + let finish = if merge { + let blend_blocks = (0..block_count).map(|block| { + let offset = Literal::usize_unsuffixed(block * 16); + let merge = block_value("merge", block); + quote! { + let remaining = output_lane.saturating_sub(#offset).min(16); + let [prefix, _] = crate::transmute::checked_transmute_copy::<_, [__m128i; 2]>( + &crate::support::COMPACT_PREFIX_MASKS[remaining], + ); + let compressed = crate::transmute::checked_transmute_copy( + output[#offset..].first_chunk::<16>().unwrap(), + ); + let blended = _mm_blendv_epi8(#merge, compressed, prefix); + crate::transmute::checked_transmute_store( + blended, + output[#offset..].first_chunk_mut::<16>().unwrap(), + ); + } + }); + quote! { #(#blend_blocks)* } + } else { + TokenStream::new() + }; + quote! { + let mask_bits = #token.#to_bitmask(mask); + let mut output = [0u8; #len]; + let mut output_lane = 0; + #(#compact_blocks)* + #finish + #vec::simd_from(#token, output) + } + } + OpSig::Expand { merge } => { + let input = quote! { <[u8; #len]>::from(values) }; + let expand_blocks = (0..block_count).map(|block| { + // Each shuffle consumes one 128-bit mask. Extract it directly; + // a wide movemask would require extra scalar shifts to split it. + let mask = block_value("mask", block); + let offset = Literal::usize_unsuffixed(block * 16); + let setup = expand_16_control(quote! { block_bits }); + let result = if merge { + let merge = block_value("merge", block); + quote! { _mm_blendv_epi8(#merge, expanded, #mask) } + } else { + quote! { expanded } + }; + let advance = if block + 1 < block_count { + quote! { input_lane += block_bits.count_ones() as usize; } + } else { + TokenStream::new() + }; + quote! { + let block_bits = _mm_movemask_epi8(#mask) as usize; + #setup + let packed = crate::transmute::checked_transmute_copy( + input[input_lane..].first_chunk::<16>().unwrap(), + ); + let expanded = _mm_shuffle_epi8(packed, control); + let result = #result; + crate::transmute::checked_transmute_store( + result, + output[#offset..].first_chunk_mut::<16>().unwrap(), + ); + #advance + } + }); + quote! { + let input = #input; + let mut output = [0u8; #len]; + let mut input_lane = 0; + #(#expand_blocks)* + #vec::simd_from(#token, output) + } + } + _ => unreachable!("only compact operations use the x86 implementation"), + }) + } + + fn handle_avx2_shuffle_compact(&self, op: Op, vec_ty: &VecType) -> TokenStream { + assert!(*self == Self::Avx2, "wide shuffle compact requires AVX2"); + assert!( + matches!(vec_ty.len, 32 | 64), + "wide shuffle compact requires a 32- or 64-byte vector" + ); + let pair_count = vec_ty.len / 32; + let to_bitmask = generic_op_name("to_bitmask", &vec_ty.mask_ty()); + + self.kernel_method(op, vec_ty, |token| match op.sig { + OpSig::Compress { merge } => { + let vec = vec_ty.rust(); + let empty = if merge { + quote! { merge } + } else { + quote! { #vec::splat(#token, 0) } + }; + let all_bits = if vec_ty.len == 64 { + quote! { u64::MAX } + } else { + let all_bits = Literal::u64_unsuffixed((1_u64 << vec_ty.len) - 1); + quote! { #all_bits } + }; + // The 64-byte path has an additional cross-YMM stitching stage. Empty, full, and + // single-bit masks can bypass nearly all of it and are more than twice as fast in + // the corresponding mask-density benchmarks. The shorter 32-byte path does not + // save enough work to justify these data-dependent branches. + let edge_mask_fast_path = if vec_ty.len == 64 { + quote! { + if mask_bits == 0 || mask_bits == #all_bits { + return if mask_bits == 0 { #empty } else { values }; + } + } + } else { + TokenStream::new() + }; + let single_lane_fast_path = if vec_ty.len == 64 { + let result = if merge { + quote! { + let low = _mm_insert_epi8::<0>( + _mm256_castsi256_si128(merge.val.0[0]), + i32::from(selected), + ); + let first = + _mm256_inserti128_si256::<0>(merge.val.0[0], low); + u8x64 { + val: crate::support::Aligned512([first, merge.val.0[1]]), + simd: #token, + } + } + } else { + quote! { + let first = _mm256_set_epi64x(0, 0, 0, i64::from(selected)); + u8x64 { + val: crate::support::Aligned512([first, _mm256_setzero_si256()]), + simd: #token, + } + } + }; + quote! { + if mask_bits.is_power_of_two() { + let selected = values.as_array()[mask_bits.trailing_zeros() as usize]; + return { #result }; + } + } + } else { + TokenStream::new() + }; + let compact_pairs = (0..pair_count).map(|pair| { + let low_block = pair * 2; + let high_block = low_block + 1; + let low_shifted_mask = shifted_mask_bits(low_block * 16); + let high_shifted_mask = shifted_mask_bits(high_block * 16); + let input = if vec_ty.len == 32 { + quote! { values.val.0 } + } else { + let pair = Literal::usize_unsuffixed(pair); + quote! { values.val.0[#pair] } + }; + let bits0 = format_ident!("block_bits_{low_block}"); + let bits1 = format_ident!("block_bits_{high_block}"); + let splice0 = format_ident!("splice_{low_block}"); + let splice1 = format_ident!("splice_{high_block}"); + let compressed_pair = format_ident!("compressed_{pair}"); + let (setup0, control0, low_count0) = + compact_16_control(quote! { #bits0 }, &format!("_{low_block}")); + let (setup1, control1, low_count1) = + compact_16_control(quote! { #bits1 }, &format!("_{high_block}")); + let output_lane = if vec_ty.len == 32 && merge { + quote! { + let output_lane = + (#bits0.count_ones() + #bits1.count_ones()) as usize; + } + } else { + TokenStream::new() + }; + let finish_pair = quote! { + let splice = crate::transmute::checked_transmute_copy( + &crate::support::COMPACT_16_SPLICE_CONTROLS + [#bits0.count_ones() as usize], + ); + let compressed_low = + _mm256_permute2x128_si256::<0x80>(compressed, compressed); + let compressed_high = + _mm256_permute2x128_si256::<0x11>(compressed, compressed); + let #compressed_pair = _mm256_or_si256( + compressed_low, + _mm256_shuffle_epi8(compressed_high, splice), + ); + #output_lane + }; + quote! { + let #bits0 = (#low_shifted_mask & 0xffff) as usize; + #setup0 + + let #bits1 = (#high_shifted_mask & 0xffff) as usize; + #setup1 + + let control = _mm256_set_m128i(#control1, #control0); + let compacted = _mm256_shuffle_epi8(#input, control); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[#low_count0]; + let #splice0 = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[#low_count1]; + let #splice1 = _mm_set_epi64x((splice >> 64) as i64, splice as i64); + let splice = _mm256_set_m128i(#splice1, #splice0); + let compressed = _mm256_shuffle_epi8(compacted, splice); + #finish_pair + } + }); + let stitch_pairs = if vec_ty.len == 64 { + let output_lane = if merge { + quote! { let output_lane = mask_bits.count_ones() as usize; } + } else { + TokenStream::new() + }; + quote! { + let first_count = + (block_bits_0.count_ones() + block_bits_1.count_ones()) as usize; + let controls = + &crate::support::COMPACT_32_STITCH_CONTROLS[first_count]; + let left_same = crate::transmute::checked_transmute_copy(&controls.left_same); + let left_cross = crate::transmute::checked_transmute_copy(&controls.left_cross); + let right_same = crate::transmute::checked_transmute_copy(&controls.right_same); + let right_cross = crate::transmute::checked_transmute_copy(&controls.right_cross); + + let low_to_high = + _mm256_permute2x128_si256::<0x08>(compressed_1, compressed_1); + let shifted_left = _mm256_or_si256( + _mm256_shuffle_epi8(compressed_1, left_same), + _mm256_shuffle_epi8(low_to_high, left_cross), + ); + let compressed_0 = _mm256_or_si256(compressed_0, shifted_left); + + let high_to_low = + _mm256_permute2x128_si256::<0x81>(compressed_1, compressed_1); + let compressed_1 = _mm256_or_si256( + _mm256_shuffle_epi8(compressed_1, right_same), + _mm256_shuffle_epi8(high_to_low, right_cross), + ); + #output_lane + } + } else { + TokenStream::new() + }; + let result_chunks = (0..pair_count).map(|pair| { + let offset = Literal::usize_unsuffixed(pair * 32); + let compressed = format_ident!("compressed_{pair}"); + let result = format_ident!("result_{pair}"); + if merge { + let remaining = if pair == 0 { + quote! { output_lane.min(32) } + } else { + quote! { output_lane.saturating_sub(#offset).min(32) } + }; + let merge = if vec_ty.len == 32 { + quote! { merge.val.0 } + } else { + let pair = Literal::usize_unsuffixed(pair); + quote! { merge.val.0[#pair] } + }; + quote! { + let prefix = crate::transmute::checked_transmute_copy( + &crate::support::COMPACT_PREFIX_MASKS[#remaining], + ); + let #result = _mm256_blendv_epi8(#merge, #compressed, prefix); + } + } else { + quote! { + let #result = #compressed; + } + } + }); + let finish = if vec_ty.len == 32 { + quote! { result_0.simd_into(#token) } + } else { + quote! { + u8x64 { + val: crate::support::Aligned512([result_0, result_1]), + simd: #token, + } + } + }; + quote! { + let mask_bits = #token.#to_bitmask(mask); + #edge_mask_fast_path + #single_lane_fast_path + #(#compact_pairs)* + #stitch_pairs + #(#result_chunks)* + #finish + } + } + _ => unreachable!("only compact operations use the AVX2 implementation"), + }) + } + + fn handle_avx512_compact_op(&self, op: Op, vec_ty: &VecType) -> TokenStream { + assert!(*self == Self::Avx512, "compact intrinsics require AVX-512"); + let prefix = match vec_ty.n_bits() { + 128 => "_mm", + 256 => "_mm256", + 512 => "_mm512", + _ => unreachable!(), + }; + let suffix = match (vec_ty.scalar, vec_ty.scalar_bits) { + (ScalarType::Float, 32) => "ps".to_owned(), + (ScalarType::Float, 64) => "pd".to_owned(), + (ScalarType::Int | ScalarType::Unsigned, bits) => format!("epi{bits}"), + _ => unreachable!("compact intrinsics require numeric vectors"), + }; + let mask_ty = ScalarType::Unsigned.rust(avx512_mask_register_bits(vec_ty)); + let mask_bits = avx512_mask_bits_expr(quote! { mask }); + + self.kernel_method(op, vec_ty, |token| match op.sig { + OpSig::Compress { merge: false } => { + let intrinsic = format_ident!("{prefix}_maskz_compress_{suffix}"); + quote! { + #intrinsic(#mask_bits as #mask_ty, values.into()).simd_into(#token) + } + } + OpSig::Compress { merge: true } => { + let intrinsic = format_ident!("{prefix}_mask_compress_{suffix}"); + quote! { + #intrinsic(merge.into(), #mask_bits as #mask_ty, values.into()).simd_into(#token) + } + } + OpSig::Expand { merge: false } => { + let intrinsic = format_ident!("{prefix}_maskz_expand_{suffix}"); + quote! { + #intrinsic(#mask_bits as #mask_ty, values.into()).simd_into(#token) + } + } + OpSig::Expand { merge: true } => { + let intrinsic = format_ident!("{prefix}_mask_expand_{suffix}"); + quote! { + #intrinsic(merge.into(), #mask_bits as #mask_ty, values.into()).simd_into(#token) + } + } + _ => unreachable!(), + }) + } + fn handle_count_ones(&self, op: Op, vec_ty: &VecType) -> TokenStream { match *self { Self::Avx512 => { diff --git a/fearless_simd_gen/src/ops.rs b/fearless_simd_gen/src/ops.rs index 916239a0..8eff40dc 100644 --- a/fearless_simd_gen/src/ops.rs +++ b/fearless_simd_gen/src/ops.rs @@ -108,6 +108,10 @@ pub(crate) enum OpSig { /// Takes two vectors and a same-width byte-index vector, and returns the original vector type with its bytes /// dynamically selected from the concatenation of both vectors. Out-of-range indices produce zero. ConcatSwizzleDynPrecise, + /// Takes a byte vector, a same-width byte-lane mask, and optionally a merge vector. + Compress { merge: bool }, + /// Takes a byte vector, a same-width byte-lane mask, and optionally a merge vector. + Expand { merge: bool }, /// Takes a single argument of the source vector type, and returns a vector type of the target scalar type and the /// same length. Cvt { @@ -352,6 +356,17 @@ impl Op { vec, ) } + OpSig::Compress { merge: false } | OpSig::Expand { merge: false } => { + let mask = vec_ty.mask_ty().rust(); + (vec![vec.clone(), quote! { #mask<#simd_ty> }], vec) + } + OpSig::Compress { merge: true } | OpSig::Expand { merge: true } => { + let mask = vec_ty.mask_ty().rust(); + ( + vec![vec.clone(), quote! { #mask<#simd_ty> }, vec.clone()], + vec, + ) + } OpSig::Cvt { target_ty, scalar_bits, @@ -415,6 +430,12 @@ impl Op { OpSig::LoadInterleaved { .. } | OpSig::StoreInterleaved { .. } => { return None; } + OpSig::Compress { merge: false } | OpSig::Expand { merge: false } => { + quote! { (self, mask: Self::Mask) -> Self } + } + OpSig::Compress { merge: true } | OpSig::Expand { merge: true } => { + quote! { (self, mask: Self::Mask, merge: impl SimdInto) -> Self } + } OpSig::MaskFromBitmask | OpSig::MaskToBitmask | OpSig::MaskSet => return None, OpSig::Unary | OpSig::Cvt { .. } => { let arg0 = &arg_names[0]; @@ -671,6 +692,38 @@ const BASE_OPS: &[Op] = &[ "Dynamically select bytes from the concatenation of this vector and `rhs`.\n\n\ The `indices` operand is a same-width byte vector. For each output byte, index values within the concatenated vectors' byte length select the corresponding byte: the first vector comes first, followed by `rhs`. Out-of-range indices produce zero.", ), + Op::new( + "compress", + OpKind::BaseTraitMethod, + OpSig::Compress { merge: false }, + "Compact the elements selected by `{arg1}` into consecutive low lanes, preserving their order.\n\n\ + Lanes above the number of selected elements are zero (positive zero for floats).\n\n\ + Elements are moved without changing their bits, including floating-point NaN payloads and signed zeros.", + ), + Op::new( + "compress_merge", + OpKind::BaseTraitMethod, + OpSig::Compress { merge: true }, + "Compact the elements selected by `{arg1}` into consecutive low lanes, preserving their order.\n\n\ + Lanes above the number of selected elements retain the corresponding values from `{arg2}`.\n\n\ + Elements are moved without changing their bits, including floating-point NaN payloads and signed zeros.", + ), + Op::new( + "expand", + OpKind::BaseTraitMethod, + OpSig::Expand { merge: false }, + "Expand consecutive low elements from `{arg0}` into the lanes selected by `{arg1}`, preserving their order.\n\n\ + Unselected lanes are zero (positive zero for floats).\n\n\ + Elements are moved without changing their bits, including floating-point NaN payloads and signed zeros.", + ), + Op::new( + "expand_merge", + OpKind::BaseTraitMethod, + OpSig::Expand { merge: true }, + "Expand consecutive low elements from `{arg0}` into the lanes selected by `{arg1}`, preserving their order.\n\n\ + Unselected lanes retain the corresponding values from `{arg2}`.\n\n\ + Elements are moved without changing their bits, including floating-point NaN payloads and signed zeros.", + ), ]; const COMMON_BASE_OPS: &[Op] = &[ @@ -1893,6 +1946,8 @@ impl OpSig { | Self::SwizzleDynPrecise | Self::ConcatSwizzleDyn | Self::ConcatSwizzleDynPrecise + | Self::Compress { .. } + | Self::Expand { .. } | Self::Slide { granularity: SlideGranularity::AcrossBlocks, .. @@ -1943,6 +1998,10 @@ impl OpSig { &["a", "indices"] } Self::ConcatSwizzleDyn | Self::ConcatSwizzleDynPrecise => &["a", "b", "indices"], + Self::Compress { merge: false } | Self::Expand { merge: false } => &["values", "mask"], + Self::Compress { merge: true } | Self::Expand { merge: true } => { + &["values", "mask", "merge"] + } Self::Binary | Self::Compare | Self::Combine { .. } @@ -1967,6 +2026,10 @@ impl OpSig { | Self::MaskFromBitmask | Self::MaskToBitmask | Self::MaskSet => &[], + Self::Compress { merge: false } | Self::Expand { merge: false } => &["self", "mask"], + Self::Compress { merge: true } | Self::Expand { merge: true } => { + &["self", "mask", "merge"] + } Self::Unary | Self::Reduce { .. } | Self::RotateElements { .. } @@ -2046,6 +2109,12 @@ impl OpSig { | Self::SwizzleDyn | Self::SwizzleDynPrecise | Self::Slide { .. } => return None, + Self::Compress { merge: false } | Self::Expand { merge: false } => { + quote! { self, mask } + } + Self::Compress { merge: true } | Self::Expand { merge: true } => { + quote! { self, mask, merge.simd_into(self.simd) } + } }; Some(args) } diff --git a/fearless_simd_tests/tests/harness/ops/compress.rs b/fearless_simd_tests/tests/harness/ops/compress.rs new file mode 100644 index 00000000..73519f5e --- /dev/null +++ b/fearless_simd_tests/tests/harness/ops/compress.rs @@ -0,0 +1,2193 @@ +// Copyright 2026 the Fearless_SIMD Authors +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use fearless_simd::*; +use fearless_simd_dev_macros::simd_test; + +#[simd_test] +fn compact_u8x16(simd: S) { + let values: [u8; 16] = core::array::from_fn(|lane| { + (0x91e3_a5c7_d9fb_2d4f_u64.wrapping_mul(lane as u64 + 1)) as u8 + }); + let merge: [u8; 16] = core::array::from_fn(|lane| !values[lane]); + let values_vec = u8x16::simd_from(simd, values); + let merge_vec = u8x16::simd_from(simd, merge); + let all_lanes = u64::MAX >> (64 - 16); + let single_lanes = (0..16).map(|lane| 1_u64 << lane); + let prefixes = (1..=16).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..16).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + { + let mask = mask8x16::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 16]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 16]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..16 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!(*values_vec.compress(mask), expected_compress); + assert_eq!(*simd.compress_u8x16(values_vec, mask), expected_compress); + assert_eq!( + *values_vec.compress_merge(mask, merge_vec), + expected_compress_merge + ); + assert_eq!( + *simd.compress_merge_u8x16(values_vec, mask, merge_vec), + expected_compress_merge + ); + assert_eq!(*values_vec.expand(mask), expected_expand); + assert_eq!(*simd.expand_u8x16(values_vec, mask), expected_expand); + assert_eq!( + *values_vec.expand_merge(mask, merge_vec), + expected_expand_merge + ); + assert_eq!( + *simd.expand_merge_u8x16(values_vec, mask, merge_vec), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_u8x32(simd: S) { + let values: [u8; 32] = core::array::from_fn(|lane| { + (0x91e3_a5c7_d9fb_2d4f_u64.wrapping_mul(lane as u64 + 1)) as u8 + }); + let merge: [u8; 32] = core::array::from_fn(|lane| !values[lane]); + let values_vec = u8x32::simd_from(simd, values); + let merge_vec = u8x32::simd_from(simd, merge); + let all_lanes = u64::MAX >> (64 - 32); + let single_lanes = (0..32).map(|lane| 1_u64 << lane); + let prefixes = (1..=32).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..32).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + { + let mask = mask8x32::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 32]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 32]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..32 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!(*values_vec.compress(mask), expected_compress); + assert_eq!(*simd.compress_u8x32(values_vec, mask), expected_compress); + assert_eq!( + *values_vec.compress_merge(mask, merge_vec), + expected_compress_merge + ); + assert_eq!( + *simd.compress_merge_u8x32(values_vec, mask, merge_vec), + expected_compress_merge + ); + assert_eq!(*values_vec.expand(mask), expected_expand); + assert_eq!(*simd.expand_u8x32(values_vec, mask), expected_expand); + assert_eq!( + *values_vec.expand_merge(mask, merge_vec), + expected_expand_merge + ); + assert_eq!( + *simd.expand_merge_u8x32(values_vec, mask, merge_vec), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_u8x64(simd: S) { + let values: [u8; 64] = core::array::from_fn(|lane| { + (0x91e3_a5c7_d9fb_2d4f_u64.wrapping_mul(lane as u64 + 1)) as u8 + }); + let merge: [u8; 64] = core::array::from_fn(|lane| !values[lane]); + let values_vec = u8x64::simd_from(simd, values); + let merge_vec = u8x64::simd_from(simd, merge); + let all_lanes = u64::MAX; + let single_lanes = (0..64).map(|lane| 1_u64 << lane); + let prefixes = (1..=64).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..64).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + { + let mask = mask8x64::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 64]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 64]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..64 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!(*values_vec.compress(mask), expected_compress); + assert_eq!(*simd.compress_u8x64(values_vec, mask), expected_compress); + assert_eq!( + *values_vec.compress_merge(mask, merge_vec), + expected_compress_merge + ); + assert_eq!( + *simd.compress_merge_u8x64(values_vec, mask, merge_vec), + expected_compress_merge + ); + assert_eq!(*values_vec.expand(mask), expected_expand); + assert_eq!(*simd.expand_u8x64(values_vec, mask), expected_expand); + assert_eq!( + *values_vec.expand_merge(mask, merge_vec), + expected_expand_merge + ); + assert_eq!( + *simd.expand_merge_u8x64(values_vec, mask, merge_vec), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_i8x16(simd: S) { + let values: [i8; 16] = core::array::from_fn(|lane| { + (0x91e3_a5c7_d9fb_2d4f_u64.wrapping_mul(lane as u64 + 1)) as i8 + }); + let merge: [i8; 16] = core::array::from_fn(|lane| !values[lane]); + let values_vec = i8x16::simd_from(simd, values); + let merge_vec = i8x16::simd_from(simd, merge); + let all_lanes = u64::MAX >> (64 - 16); + let single_lanes = (0..16).map(|lane| 1_u64 << lane); + let prefixes = (1..=16).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..16).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + { + let mask = mask8x16::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 16]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 16]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..16 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!(*values_vec.compress(mask), expected_compress); + assert_eq!(*simd.compress_i8x16(values_vec, mask), expected_compress); + assert_eq!( + *values_vec.compress_merge(mask, merge_vec), + expected_compress_merge + ); + assert_eq!( + *simd.compress_merge_i8x16(values_vec, mask, merge_vec), + expected_compress_merge + ); + assert_eq!(*values_vec.expand(mask), expected_expand); + assert_eq!(*simd.expand_i8x16(values_vec, mask), expected_expand); + assert_eq!( + *values_vec.expand_merge(mask, merge_vec), + expected_expand_merge + ); + assert_eq!( + *simd.expand_merge_i8x16(values_vec, mask, merge_vec), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_i8x32(simd: S) { + let values: [i8; 32] = core::array::from_fn(|lane| { + (0x91e3_a5c7_d9fb_2d4f_u64.wrapping_mul(lane as u64 + 1)) as i8 + }); + let merge: [i8; 32] = core::array::from_fn(|lane| !values[lane]); + let values_vec = i8x32::simd_from(simd, values); + let merge_vec = i8x32::simd_from(simd, merge); + let all_lanes = u64::MAX >> (64 - 32); + let single_lanes = (0..32).map(|lane| 1_u64 << lane); + let prefixes = (1..=32).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..32).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + { + let mask = mask8x32::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 32]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 32]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..32 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!(*values_vec.compress(mask), expected_compress); + assert_eq!(*simd.compress_i8x32(values_vec, mask), expected_compress); + assert_eq!( + *values_vec.compress_merge(mask, merge_vec), + expected_compress_merge + ); + assert_eq!( + *simd.compress_merge_i8x32(values_vec, mask, merge_vec), + expected_compress_merge + ); + assert_eq!(*values_vec.expand(mask), expected_expand); + assert_eq!(*simd.expand_i8x32(values_vec, mask), expected_expand); + assert_eq!( + *values_vec.expand_merge(mask, merge_vec), + expected_expand_merge + ); + assert_eq!( + *simd.expand_merge_i8x32(values_vec, mask, merge_vec), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_i8x64(simd: S) { + let values: [i8; 64] = core::array::from_fn(|lane| { + (0x91e3_a5c7_d9fb_2d4f_u64.wrapping_mul(lane as u64 + 1)) as i8 + }); + let merge: [i8; 64] = core::array::from_fn(|lane| !values[lane]); + let values_vec = i8x64::simd_from(simd, values); + let merge_vec = i8x64::simd_from(simd, merge); + let all_lanes = u64::MAX; + let single_lanes = (0..64).map(|lane| 1_u64 << lane); + let prefixes = (1..=64).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..64).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + { + let mask = mask8x64::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 64]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 64]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..64 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!(*values_vec.compress(mask), expected_compress); + assert_eq!(*simd.compress_i8x64(values_vec, mask), expected_compress); + assert_eq!( + *values_vec.compress_merge(mask, merge_vec), + expected_compress_merge + ); + assert_eq!( + *simd.compress_merge_i8x64(values_vec, mask, merge_vec), + expected_compress_merge + ); + assert_eq!(*values_vec.expand(mask), expected_expand); + assert_eq!(*simd.expand_i8x64(values_vec, mask), expected_expand); + assert_eq!( + *values_vec.expand_merge(mask, merge_vec), + expected_expand_merge + ); + assert_eq!( + *simd.expand_merge_i8x64(values_vec, mask, merge_vec), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_u16x8(simd: S) { + let values: [u16; 8] = core::array::from_fn(|lane| { + (0x91e3_a5c7_d9fb_2d4f_u64.wrapping_mul(lane as u64 + 1)) as u16 + }); + let merge: [u16; 8] = core::array::from_fn(|lane| !values[lane]); + let values_vec = u16x8::simd_from(simd, values); + let merge_vec = u16x8::simd_from(simd, merge); + let all_lanes = u64::MAX >> (64 - 8); + let single_lanes = (0..8).map(|lane| 1_u64 << lane); + let prefixes = (1..=8).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..8).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + .chain(0..=all_lanes) + { + let mask = mask16x8::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 8]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 8]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..8 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!(*values_vec.compress(mask), expected_compress); + assert_eq!(*simd.compress_u16x8(values_vec, mask), expected_compress); + assert_eq!( + *values_vec.compress_merge(mask, merge_vec), + expected_compress_merge + ); + assert_eq!( + *simd.compress_merge_u16x8(values_vec, mask, merge_vec), + expected_compress_merge + ); + assert_eq!(*values_vec.expand(mask), expected_expand); + assert_eq!(*simd.expand_u16x8(values_vec, mask), expected_expand); + assert_eq!( + *values_vec.expand_merge(mask, merge_vec), + expected_expand_merge + ); + assert_eq!( + *simd.expand_merge_u16x8(values_vec, mask, merge_vec), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_u16x16(simd: S) { + let values: [u16; 16] = core::array::from_fn(|lane| { + (0x91e3_a5c7_d9fb_2d4f_u64.wrapping_mul(lane as u64 + 1)) as u16 + }); + let merge: [u16; 16] = core::array::from_fn(|lane| !values[lane]); + let values_vec = u16x16::simd_from(simd, values); + let merge_vec = u16x16::simd_from(simd, merge); + let all_lanes = u64::MAX >> (64 - 16); + let single_lanes = (0..16).map(|lane| 1_u64 << lane); + let prefixes = (1..=16).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..16).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + { + let mask = mask16x16::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 16]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 16]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..16 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!(*values_vec.compress(mask), expected_compress); + assert_eq!(*simd.compress_u16x16(values_vec, mask), expected_compress); + assert_eq!( + *values_vec.compress_merge(mask, merge_vec), + expected_compress_merge + ); + assert_eq!( + *simd.compress_merge_u16x16(values_vec, mask, merge_vec), + expected_compress_merge + ); + assert_eq!(*values_vec.expand(mask), expected_expand); + assert_eq!(*simd.expand_u16x16(values_vec, mask), expected_expand); + assert_eq!( + *values_vec.expand_merge(mask, merge_vec), + expected_expand_merge + ); + assert_eq!( + *simd.expand_merge_u16x16(values_vec, mask, merge_vec), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_u16x32(simd: S) { + let values: [u16; 32] = core::array::from_fn(|lane| { + (0x91e3_a5c7_d9fb_2d4f_u64.wrapping_mul(lane as u64 + 1)) as u16 + }); + let merge: [u16; 32] = core::array::from_fn(|lane| !values[lane]); + let values_vec = u16x32::simd_from(simd, values); + let merge_vec = u16x32::simd_from(simd, merge); + let all_lanes = u64::MAX >> (64 - 32); + let single_lanes = (0..32).map(|lane| 1_u64 << lane); + let prefixes = (1..=32).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..32).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + { + let mask = mask16x32::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 32]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 32]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..32 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!(*values_vec.compress(mask), expected_compress); + assert_eq!(*simd.compress_u16x32(values_vec, mask), expected_compress); + assert_eq!( + *values_vec.compress_merge(mask, merge_vec), + expected_compress_merge + ); + assert_eq!( + *simd.compress_merge_u16x32(values_vec, mask, merge_vec), + expected_compress_merge + ); + assert_eq!(*values_vec.expand(mask), expected_expand); + assert_eq!(*simd.expand_u16x32(values_vec, mask), expected_expand); + assert_eq!( + *values_vec.expand_merge(mask, merge_vec), + expected_expand_merge + ); + assert_eq!( + *simd.expand_merge_u16x32(values_vec, mask, merge_vec), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_i16x8(simd: S) { + let values: [i16; 8] = core::array::from_fn(|lane| { + (0x91e3_a5c7_d9fb_2d4f_u64.wrapping_mul(lane as u64 + 1)) as i16 + }); + let merge: [i16; 8] = core::array::from_fn(|lane| !values[lane]); + let values_vec = i16x8::simd_from(simd, values); + let merge_vec = i16x8::simd_from(simd, merge); + let all_lanes = u64::MAX >> (64 - 8); + let single_lanes = (0..8).map(|lane| 1_u64 << lane); + let prefixes = (1..=8).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..8).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + .chain(0..=all_lanes) + { + let mask = mask16x8::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 8]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 8]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..8 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!(*values_vec.compress(mask), expected_compress); + assert_eq!(*simd.compress_i16x8(values_vec, mask), expected_compress); + assert_eq!( + *values_vec.compress_merge(mask, merge_vec), + expected_compress_merge + ); + assert_eq!( + *simd.compress_merge_i16x8(values_vec, mask, merge_vec), + expected_compress_merge + ); + assert_eq!(*values_vec.expand(mask), expected_expand); + assert_eq!(*simd.expand_i16x8(values_vec, mask), expected_expand); + assert_eq!( + *values_vec.expand_merge(mask, merge_vec), + expected_expand_merge + ); + assert_eq!( + *simd.expand_merge_i16x8(values_vec, mask, merge_vec), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_i16x16(simd: S) { + let values: [i16; 16] = core::array::from_fn(|lane| { + (0x91e3_a5c7_d9fb_2d4f_u64.wrapping_mul(lane as u64 + 1)) as i16 + }); + let merge: [i16; 16] = core::array::from_fn(|lane| !values[lane]); + let values_vec = i16x16::simd_from(simd, values); + let merge_vec = i16x16::simd_from(simd, merge); + let all_lanes = u64::MAX >> (64 - 16); + let single_lanes = (0..16).map(|lane| 1_u64 << lane); + let prefixes = (1..=16).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..16).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + { + let mask = mask16x16::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 16]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 16]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..16 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!(*values_vec.compress(mask), expected_compress); + assert_eq!(*simd.compress_i16x16(values_vec, mask), expected_compress); + assert_eq!( + *values_vec.compress_merge(mask, merge_vec), + expected_compress_merge + ); + assert_eq!( + *simd.compress_merge_i16x16(values_vec, mask, merge_vec), + expected_compress_merge + ); + assert_eq!(*values_vec.expand(mask), expected_expand); + assert_eq!(*simd.expand_i16x16(values_vec, mask), expected_expand); + assert_eq!( + *values_vec.expand_merge(mask, merge_vec), + expected_expand_merge + ); + assert_eq!( + *simd.expand_merge_i16x16(values_vec, mask, merge_vec), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_i16x32(simd: S) { + let values: [i16; 32] = core::array::from_fn(|lane| { + (0x91e3_a5c7_d9fb_2d4f_u64.wrapping_mul(lane as u64 + 1)) as i16 + }); + let merge: [i16; 32] = core::array::from_fn(|lane| !values[lane]); + let values_vec = i16x32::simd_from(simd, values); + let merge_vec = i16x32::simd_from(simd, merge); + let all_lanes = u64::MAX >> (64 - 32); + let single_lanes = (0..32).map(|lane| 1_u64 << lane); + let prefixes = (1..=32).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..32).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + { + let mask = mask16x32::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 32]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 32]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..32 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!(*values_vec.compress(mask), expected_compress); + assert_eq!(*simd.compress_i16x32(values_vec, mask), expected_compress); + assert_eq!( + *values_vec.compress_merge(mask, merge_vec), + expected_compress_merge + ); + assert_eq!( + *simd.compress_merge_i16x32(values_vec, mask, merge_vec), + expected_compress_merge + ); + assert_eq!(*values_vec.expand(mask), expected_expand); + assert_eq!(*simd.expand_i16x32(values_vec, mask), expected_expand); + assert_eq!( + *values_vec.expand_merge(mask, merge_vec), + expected_expand_merge + ); + assert_eq!( + *simd.expand_merge_i16x32(values_vec, mask, merge_vec), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_u32x4(simd: S) { + let values: [u32; 4] = core::array::from_fn(|lane| { + (0x91e3_a5c7_d9fb_2d4f_u64.wrapping_mul(lane as u64 + 1)) as u32 + }); + let merge: [u32; 4] = core::array::from_fn(|lane| !values[lane]); + let values_vec = u32x4::simd_from(simd, values); + let merge_vec = u32x4::simd_from(simd, merge); + let all_lanes = u64::MAX >> (64 - 4); + let single_lanes = (0..4).map(|lane| 1_u64 << lane); + let prefixes = (1..=4).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..4).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + .chain(0..=all_lanes) + { + let mask = mask32x4::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 4]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 4]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..4 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!(*values_vec.compress(mask), expected_compress); + assert_eq!(*simd.compress_u32x4(values_vec, mask), expected_compress); + assert_eq!( + *values_vec.compress_merge(mask, merge_vec), + expected_compress_merge + ); + assert_eq!( + *simd.compress_merge_u32x4(values_vec, mask, merge_vec), + expected_compress_merge + ); + assert_eq!(*values_vec.expand(mask), expected_expand); + assert_eq!(*simd.expand_u32x4(values_vec, mask), expected_expand); + assert_eq!( + *values_vec.expand_merge(mask, merge_vec), + expected_expand_merge + ); + assert_eq!( + *simd.expand_merge_u32x4(values_vec, mask, merge_vec), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_u32x8(simd: S) { + let values: [u32; 8] = core::array::from_fn(|lane| { + (0x91e3_a5c7_d9fb_2d4f_u64.wrapping_mul(lane as u64 + 1)) as u32 + }); + let merge: [u32; 8] = core::array::from_fn(|lane| !values[lane]); + let values_vec = u32x8::simd_from(simd, values); + let merge_vec = u32x8::simd_from(simd, merge); + let all_lanes = u64::MAX >> (64 - 8); + let single_lanes = (0..8).map(|lane| 1_u64 << lane); + let prefixes = (1..=8).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..8).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + .chain(0..=all_lanes) + { + let mask = mask32x8::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 8]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 8]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..8 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!(*values_vec.compress(mask), expected_compress); + assert_eq!(*simd.compress_u32x8(values_vec, mask), expected_compress); + assert_eq!( + *values_vec.compress_merge(mask, merge_vec), + expected_compress_merge + ); + assert_eq!( + *simd.compress_merge_u32x8(values_vec, mask, merge_vec), + expected_compress_merge + ); + assert_eq!(*values_vec.expand(mask), expected_expand); + assert_eq!(*simd.expand_u32x8(values_vec, mask), expected_expand); + assert_eq!( + *values_vec.expand_merge(mask, merge_vec), + expected_expand_merge + ); + assert_eq!( + *simd.expand_merge_u32x8(values_vec, mask, merge_vec), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_u32x16(simd: S) { + let values: [u32; 16] = core::array::from_fn(|lane| { + (0x91e3_a5c7_d9fb_2d4f_u64.wrapping_mul(lane as u64 + 1)) as u32 + }); + let merge: [u32; 16] = core::array::from_fn(|lane| !values[lane]); + let values_vec = u32x16::simd_from(simd, values); + let merge_vec = u32x16::simd_from(simd, merge); + let all_lanes = u64::MAX >> (64 - 16); + let single_lanes = (0..16).map(|lane| 1_u64 << lane); + let prefixes = (1..=16).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..16).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + .chain(0..=all_lanes) + { + let mask = mask32x16::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 16]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 16]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..16 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!(*values_vec.compress(mask), expected_compress); + assert_eq!(*simd.compress_u32x16(values_vec, mask), expected_compress); + assert_eq!( + *values_vec.compress_merge(mask, merge_vec), + expected_compress_merge + ); + assert_eq!( + *simd.compress_merge_u32x16(values_vec, mask, merge_vec), + expected_compress_merge + ); + assert_eq!(*values_vec.expand(mask), expected_expand); + assert_eq!(*simd.expand_u32x16(values_vec, mask), expected_expand); + assert_eq!( + *values_vec.expand_merge(mask, merge_vec), + expected_expand_merge + ); + assert_eq!( + *simd.expand_merge_u32x16(values_vec, mask, merge_vec), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_i32x4(simd: S) { + let values: [i32; 4] = core::array::from_fn(|lane| { + (0x91e3_a5c7_d9fb_2d4f_u64.wrapping_mul(lane as u64 + 1)) as i32 + }); + let merge: [i32; 4] = core::array::from_fn(|lane| !values[lane]); + let values_vec = i32x4::simd_from(simd, values); + let merge_vec = i32x4::simd_from(simd, merge); + let all_lanes = u64::MAX >> (64 - 4); + let single_lanes = (0..4).map(|lane| 1_u64 << lane); + let prefixes = (1..=4).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..4).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + .chain(0..=all_lanes) + { + let mask = mask32x4::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 4]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 4]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..4 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!(*values_vec.compress(mask), expected_compress); + assert_eq!(*simd.compress_i32x4(values_vec, mask), expected_compress); + assert_eq!( + *values_vec.compress_merge(mask, merge_vec), + expected_compress_merge + ); + assert_eq!( + *simd.compress_merge_i32x4(values_vec, mask, merge_vec), + expected_compress_merge + ); + assert_eq!(*values_vec.expand(mask), expected_expand); + assert_eq!(*simd.expand_i32x4(values_vec, mask), expected_expand); + assert_eq!( + *values_vec.expand_merge(mask, merge_vec), + expected_expand_merge + ); + assert_eq!( + *simd.expand_merge_i32x4(values_vec, mask, merge_vec), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_i32x8(simd: S) { + let values: [i32; 8] = core::array::from_fn(|lane| { + (0x91e3_a5c7_d9fb_2d4f_u64.wrapping_mul(lane as u64 + 1)) as i32 + }); + let merge: [i32; 8] = core::array::from_fn(|lane| !values[lane]); + let values_vec = i32x8::simd_from(simd, values); + let merge_vec = i32x8::simd_from(simd, merge); + let all_lanes = u64::MAX >> (64 - 8); + let single_lanes = (0..8).map(|lane| 1_u64 << lane); + let prefixes = (1..=8).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..8).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + .chain(0..=all_lanes) + { + let mask = mask32x8::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 8]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 8]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..8 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!(*values_vec.compress(mask), expected_compress); + assert_eq!(*simd.compress_i32x8(values_vec, mask), expected_compress); + assert_eq!( + *values_vec.compress_merge(mask, merge_vec), + expected_compress_merge + ); + assert_eq!( + *simd.compress_merge_i32x8(values_vec, mask, merge_vec), + expected_compress_merge + ); + assert_eq!(*values_vec.expand(mask), expected_expand); + assert_eq!(*simd.expand_i32x8(values_vec, mask), expected_expand); + assert_eq!( + *values_vec.expand_merge(mask, merge_vec), + expected_expand_merge + ); + assert_eq!( + *simd.expand_merge_i32x8(values_vec, mask, merge_vec), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_i32x16(simd: S) { + let values: [i32; 16] = core::array::from_fn(|lane| { + (0x91e3_a5c7_d9fb_2d4f_u64.wrapping_mul(lane as u64 + 1)) as i32 + }); + let merge: [i32; 16] = core::array::from_fn(|lane| !values[lane]); + let values_vec = i32x16::simd_from(simd, values); + let merge_vec = i32x16::simd_from(simd, merge); + let all_lanes = u64::MAX >> (64 - 16); + let single_lanes = (0..16).map(|lane| 1_u64 << lane); + let prefixes = (1..=16).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..16).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + .chain(0..=all_lanes) + { + let mask = mask32x16::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 16]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 16]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..16 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!(*values_vec.compress(mask), expected_compress); + assert_eq!(*simd.compress_i32x16(values_vec, mask), expected_compress); + assert_eq!( + *values_vec.compress_merge(mask, merge_vec), + expected_compress_merge + ); + assert_eq!( + *simd.compress_merge_i32x16(values_vec, mask, merge_vec), + expected_compress_merge + ); + assert_eq!(*values_vec.expand(mask), expected_expand); + assert_eq!(*simd.expand_i32x16(values_vec, mask), expected_expand); + assert_eq!( + *values_vec.expand_merge(mask, merge_vec), + expected_expand_merge + ); + assert_eq!( + *simd.expand_merge_i32x16(values_vec, mask, merge_vec), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_u64x2(simd: S) { + let values: [u64; 2] = + core::array::from_fn(|lane| 0x91e3_a5c7_d9fb_2d4f_u64.wrapping_mul(lane as u64 + 1)); + let merge: [u64; 2] = core::array::from_fn(|lane| !values[lane]); + let values_vec = u64x2::simd_from(simd, values); + let merge_vec = u64x2::simd_from(simd, merge); + let all_lanes = u64::MAX >> (64 - 2); + let single_lanes = (0..2).map(|lane| 1_u64 << lane); + let prefixes = (1..=2).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..2).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + .chain(0..=all_lanes) + { + let mask = mask64x2::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 2]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 2]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..2 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!(*values_vec.compress(mask), expected_compress); + assert_eq!(*simd.compress_u64x2(values_vec, mask), expected_compress); + assert_eq!( + *values_vec.compress_merge(mask, merge_vec), + expected_compress_merge + ); + assert_eq!( + *simd.compress_merge_u64x2(values_vec, mask, merge_vec), + expected_compress_merge + ); + assert_eq!(*values_vec.expand(mask), expected_expand); + assert_eq!(*simd.expand_u64x2(values_vec, mask), expected_expand); + assert_eq!( + *values_vec.expand_merge(mask, merge_vec), + expected_expand_merge + ); + assert_eq!( + *simd.expand_merge_u64x2(values_vec, mask, merge_vec), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_u64x4(simd: S) { + let values: [u64; 4] = + core::array::from_fn(|lane| 0x91e3_a5c7_d9fb_2d4f_u64.wrapping_mul(lane as u64 + 1)); + let merge: [u64; 4] = core::array::from_fn(|lane| !values[lane]); + let values_vec = u64x4::simd_from(simd, values); + let merge_vec = u64x4::simd_from(simd, merge); + let all_lanes = u64::MAX >> (64 - 4); + let single_lanes = (0..4).map(|lane| 1_u64 << lane); + let prefixes = (1..=4).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..4).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + .chain(0..=all_lanes) + { + let mask = mask64x4::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 4]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 4]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..4 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!(*values_vec.compress(mask), expected_compress); + assert_eq!(*simd.compress_u64x4(values_vec, mask), expected_compress); + assert_eq!( + *values_vec.compress_merge(mask, merge_vec), + expected_compress_merge + ); + assert_eq!( + *simd.compress_merge_u64x4(values_vec, mask, merge_vec), + expected_compress_merge + ); + assert_eq!(*values_vec.expand(mask), expected_expand); + assert_eq!(*simd.expand_u64x4(values_vec, mask), expected_expand); + assert_eq!( + *values_vec.expand_merge(mask, merge_vec), + expected_expand_merge + ); + assert_eq!( + *simd.expand_merge_u64x4(values_vec, mask, merge_vec), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_u64x8(simd: S) { + let values: [u64; 8] = + core::array::from_fn(|lane| 0x91e3_a5c7_d9fb_2d4f_u64.wrapping_mul(lane as u64 + 1)); + let merge: [u64; 8] = core::array::from_fn(|lane| !values[lane]); + let values_vec = u64x8::simd_from(simd, values); + let merge_vec = u64x8::simd_from(simd, merge); + let all_lanes = u64::MAX >> (64 - 8); + let single_lanes = (0..8).map(|lane| 1_u64 << lane); + let prefixes = (1..=8).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..8).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + .chain(0..=all_lanes) + { + let mask = mask64x8::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 8]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 8]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..8 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!(*values_vec.compress(mask), expected_compress); + assert_eq!(*simd.compress_u64x8(values_vec, mask), expected_compress); + assert_eq!( + *values_vec.compress_merge(mask, merge_vec), + expected_compress_merge + ); + assert_eq!( + *simd.compress_merge_u64x8(values_vec, mask, merge_vec), + expected_compress_merge + ); + assert_eq!(*values_vec.expand(mask), expected_expand); + assert_eq!(*simd.expand_u64x8(values_vec, mask), expected_expand); + assert_eq!( + *values_vec.expand_merge(mask, merge_vec), + expected_expand_merge + ); + assert_eq!( + *simd.expand_merge_u64x8(values_vec, mask, merge_vec), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_i64x2(simd: S) { + let values: [i64; 2] = core::array::from_fn(|lane| { + (0x91e3_a5c7_d9fb_2d4f_u64.wrapping_mul(lane as u64 + 1)) as i64 + }); + let merge: [i64; 2] = core::array::from_fn(|lane| !values[lane]); + let values_vec = i64x2::simd_from(simd, values); + let merge_vec = i64x2::simd_from(simd, merge); + let all_lanes = u64::MAX >> (64 - 2); + let single_lanes = (0..2).map(|lane| 1_u64 << lane); + let prefixes = (1..=2).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..2).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + .chain(0..=all_lanes) + { + let mask = mask64x2::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 2]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 2]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..2 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!(*values_vec.compress(mask), expected_compress); + assert_eq!(*simd.compress_i64x2(values_vec, mask), expected_compress); + assert_eq!( + *values_vec.compress_merge(mask, merge_vec), + expected_compress_merge + ); + assert_eq!( + *simd.compress_merge_i64x2(values_vec, mask, merge_vec), + expected_compress_merge + ); + assert_eq!(*values_vec.expand(mask), expected_expand); + assert_eq!(*simd.expand_i64x2(values_vec, mask), expected_expand); + assert_eq!( + *values_vec.expand_merge(mask, merge_vec), + expected_expand_merge + ); + assert_eq!( + *simd.expand_merge_i64x2(values_vec, mask, merge_vec), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_i64x4(simd: S) { + let values: [i64; 4] = core::array::from_fn(|lane| { + (0x91e3_a5c7_d9fb_2d4f_u64.wrapping_mul(lane as u64 + 1)) as i64 + }); + let merge: [i64; 4] = core::array::from_fn(|lane| !values[lane]); + let values_vec = i64x4::simd_from(simd, values); + let merge_vec = i64x4::simd_from(simd, merge); + let all_lanes = u64::MAX >> (64 - 4); + let single_lanes = (0..4).map(|lane| 1_u64 << lane); + let prefixes = (1..=4).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..4).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + .chain(0..=all_lanes) + { + let mask = mask64x4::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 4]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 4]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..4 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!(*values_vec.compress(mask), expected_compress); + assert_eq!(*simd.compress_i64x4(values_vec, mask), expected_compress); + assert_eq!( + *values_vec.compress_merge(mask, merge_vec), + expected_compress_merge + ); + assert_eq!( + *simd.compress_merge_i64x4(values_vec, mask, merge_vec), + expected_compress_merge + ); + assert_eq!(*values_vec.expand(mask), expected_expand); + assert_eq!(*simd.expand_i64x4(values_vec, mask), expected_expand); + assert_eq!( + *values_vec.expand_merge(mask, merge_vec), + expected_expand_merge + ); + assert_eq!( + *simd.expand_merge_i64x4(values_vec, mask, merge_vec), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_i64x8(simd: S) { + let values: [i64; 8] = core::array::from_fn(|lane| { + (0x91e3_a5c7_d9fb_2d4f_u64.wrapping_mul(lane as u64 + 1)) as i64 + }); + let merge: [i64; 8] = core::array::from_fn(|lane| !values[lane]); + let values_vec = i64x8::simd_from(simd, values); + let merge_vec = i64x8::simd_from(simd, merge); + let all_lanes = u64::MAX >> (64 - 8); + let single_lanes = (0..8).map(|lane| 1_u64 << lane); + let prefixes = (1..=8).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..8).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + .chain(0..=all_lanes) + { + let mask = mask64x8::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 8]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 8]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..8 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!(*values_vec.compress(mask), expected_compress); + assert_eq!(*simd.compress_i64x8(values_vec, mask), expected_compress); + assert_eq!( + *values_vec.compress_merge(mask, merge_vec), + expected_compress_merge + ); + assert_eq!( + *simd.compress_merge_i64x8(values_vec, mask, merge_vec), + expected_compress_merge + ); + assert_eq!(*values_vec.expand(mask), expected_expand); + assert_eq!(*simd.expand_i64x8(values_vec, mask), expected_expand); + assert_eq!( + *values_vec.expand_merge(mask, merge_vec), + expected_expand_merge + ); + assert_eq!( + *simd.expand_merge_i64x8(values_vec, mask, merge_vec), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_f32x4(simd: S) { + let patterns: [u32; 8] = [ + 0x8000_0000, + 0x7fc1_2345, + 0x7f80_0000, + 0x7f80_0001, + 0, + 0xff80_0000, + 0xffc5_4321, + 0x3f80_0000, + ]; + let values: [u32; 4] = core::array::from_fn(|lane| patterns[lane % 8]); + let merge: [u32; 4] = core::array::from_fn(|lane| patterns[(lane + 3) % 8]); + let values_vec = f32x4::simd_from(simd, values.map(f32::from_bits)); + let merge_vec = f32x4::simd_from(simd, merge.map(f32::from_bits)); + let all_lanes = u64::MAX >> (64 - 4); + let single_lanes = (0..4).map(|lane| 1_u64 << lane); + let prefixes = (1..=4).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..4).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + .chain(0..=all_lanes) + { + let mask = mask32x4::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 4]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 4]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..4 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!( + values_vec.compress(mask).to_array().map(f32::to_bits), + expected_compress + ); + assert_eq!( + simd.compress_f32x4(values_vec, mask) + .to_array() + .map(f32::to_bits), + expected_compress + ); + assert_eq!( + values_vec + .compress_merge(mask, merge_vec) + .to_array() + .map(f32::to_bits), + expected_compress_merge + ); + assert_eq!( + simd.compress_merge_f32x4(values_vec, mask, merge_vec) + .to_array() + .map(f32::to_bits), + expected_compress_merge + ); + assert_eq!( + values_vec.expand(mask).to_array().map(f32::to_bits), + expected_expand + ); + assert_eq!( + simd.expand_f32x4(values_vec, mask) + .to_array() + .map(f32::to_bits), + expected_expand + ); + assert_eq!( + values_vec + .expand_merge(mask, merge_vec) + .to_array() + .map(f32::to_bits), + expected_expand_merge + ); + assert_eq!( + simd.expand_merge_f32x4(values_vec, mask, merge_vec) + .to_array() + .map(f32::to_bits), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_f32x8(simd: S) { + let patterns: [u32; 8] = [ + 0x8000_0000, + 0x7fc1_2345, + 0x7f80_0000, + 0x7f80_0001, + 0, + 0xff80_0000, + 0xffc5_4321, + 0x3f80_0000, + ]; + let values: [u32; 8] = core::array::from_fn(|lane| patterns[lane % 8]); + let merge: [u32; 8] = core::array::from_fn(|lane| patterns[(lane + 3) % 8]); + let values_vec = f32x8::simd_from(simd, values.map(f32::from_bits)); + let merge_vec = f32x8::simd_from(simd, merge.map(f32::from_bits)); + let all_lanes = u64::MAX >> (64 - 8); + let single_lanes = (0..8).map(|lane| 1_u64 << lane); + let prefixes = (1..=8).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..8).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + .chain(0..=all_lanes) + { + let mask = mask32x8::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 8]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 8]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..8 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!( + values_vec.compress(mask).to_array().map(f32::to_bits), + expected_compress + ); + assert_eq!( + simd.compress_f32x8(values_vec, mask) + .to_array() + .map(f32::to_bits), + expected_compress + ); + assert_eq!( + values_vec + .compress_merge(mask, merge_vec) + .to_array() + .map(f32::to_bits), + expected_compress_merge + ); + assert_eq!( + simd.compress_merge_f32x8(values_vec, mask, merge_vec) + .to_array() + .map(f32::to_bits), + expected_compress_merge + ); + assert_eq!( + values_vec.expand(mask).to_array().map(f32::to_bits), + expected_expand + ); + assert_eq!( + simd.expand_f32x8(values_vec, mask) + .to_array() + .map(f32::to_bits), + expected_expand + ); + assert_eq!( + values_vec + .expand_merge(mask, merge_vec) + .to_array() + .map(f32::to_bits), + expected_expand_merge + ); + assert_eq!( + simd.expand_merge_f32x8(values_vec, mask, merge_vec) + .to_array() + .map(f32::to_bits), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_f32x16(simd: S) { + let patterns: [u32; 8] = [ + 0x8000_0000, + 0x7fc1_2345, + 0x7f80_0000, + 0x7f80_0001, + 0, + 0xff80_0000, + 0xffc5_4321, + 0x3f80_0000, + ]; + let values: [u32; 16] = core::array::from_fn(|lane| patterns[lane % 8]); + let merge: [u32; 16] = core::array::from_fn(|lane| patterns[(lane + 3) % 8]); + let values_vec = f32x16::simd_from(simd, values.map(f32::from_bits)); + let merge_vec = f32x16::simd_from(simd, merge.map(f32::from_bits)); + let all_lanes = u64::MAX >> (64 - 16); + let single_lanes = (0..16).map(|lane| 1_u64 << lane); + let prefixes = (1..=16).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..16).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + .chain(0..=all_lanes) + { + let mask = mask32x16::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 16]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 16]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..16 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!( + values_vec.compress(mask).to_array().map(f32::to_bits), + expected_compress + ); + assert_eq!( + simd.compress_f32x16(values_vec, mask) + .to_array() + .map(f32::to_bits), + expected_compress + ); + assert_eq!( + values_vec + .compress_merge(mask, merge_vec) + .to_array() + .map(f32::to_bits), + expected_compress_merge + ); + assert_eq!( + simd.compress_merge_f32x16(values_vec, mask, merge_vec) + .to_array() + .map(f32::to_bits), + expected_compress_merge + ); + assert_eq!( + values_vec.expand(mask).to_array().map(f32::to_bits), + expected_expand + ); + assert_eq!( + simd.expand_f32x16(values_vec, mask) + .to_array() + .map(f32::to_bits), + expected_expand + ); + assert_eq!( + values_vec + .expand_merge(mask, merge_vec) + .to_array() + .map(f32::to_bits), + expected_expand_merge + ); + assert_eq!( + simd.expand_merge_f32x16(values_vec, mask, merge_vec) + .to_array() + .map(f32::to_bits), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_f64x2(simd: S) { + let patterns: [u64; 8] = [ + 0x8000_0000_0000_0000, + 0x7ff8_1234_5678_9abc, + 0x7ff0_0000_0000_0000, + 0x7ff0_0000_0000_0001, + 0, + 0xfff0_0000_0000_0000, + 0xfff8_abcd_1234_5678, + 0x3ff0_0000_0000_0000, + ]; + let values: [u64; 2] = core::array::from_fn(|lane| patterns[lane % 8]); + let merge: [u64; 2] = core::array::from_fn(|lane| patterns[(lane + 3) % 8]); + let values_vec = f64x2::simd_from(simd, values.map(f64::from_bits)); + let merge_vec = f64x2::simd_from(simd, merge.map(f64::from_bits)); + let all_lanes = u64::MAX >> (64 - 2); + let single_lanes = (0..2).map(|lane| 1_u64 << lane); + let prefixes = (1..=2).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..2).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + .chain(0..=all_lanes) + { + let mask = mask64x2::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 2]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 2]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..2 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!( + values_vec.compress(mask).to_array().map(f64::to_bits), + expected_compress + ); + assert_eq!( + simd.compress_f64x2(values_vec, mask) + .to_array() + .map(f64::to_bits), + expected_compress + ); + assert_eq!( + values_vec + .compress_merge(mask, merge_vec) + .to_array() + .map(f64::to_bits), + expected_compress_merge + ); + assert_eq!( + simd.compress_merge_f64x2(values_vec, mask, merge_vec) + .to_array() + .map(f64::to_bits), + expected_compress_merge + ); + assert_eq!( + values_vec.expand(mask).to_array().map(f64::to_bits), + expected_expand + ); + assert_eq!( + simd.expand_f64x2(values_vec, mask) + .to_array() + .map(f64::to_bits), + expected_expand + ); + assert_eq!( + values_vec + .expand_merge(mask, merge_vec) + .to_array() + .map(f64::to_bits), + expected_expand_merge + ); + assert_eq!( + simd.expand_merge_f64x2(values_vec, mask, merge_vec) + .to_array() + .map(f64::to_bits), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_f64x4(simd: S) { + let patterns: [u64; 8] = [ + 0x8000_0000_0000_0000, + 0x7ff8_1234_5678_9abc, + 0x7ff0_0000_0000_0000, + 0x7ff0_0000_0000_0001, + 0, + 0xfff0_0000_0000_0000, + 0xfff8_abcd_1234_5678, + 0x3ff0_0000_0000_0000, + ]; + let values: [u64; 4] = core::array::from_fn(|lane| patterns[lane % 8]); + let merge: [u64; 4] = core::array::from_fn(|lane| patterns[(lane + 3) % 8]); + let values_vec = f64x4::simd_from(simd, values.map(f64::from_bits)); + let merge_vec = f64x4::simd_from(simd, merge.map(f64::from_bits)); + let all_lanes = u64::MAX >> (64 - 4); + let single_lanes = (0..4).map(|lane| 1_u64 << lane); + let prefixes = (1..=4).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..4).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + .chain(0..=all_lanes) + { + let mask = mask64x4::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 4]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 4]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..4 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!( + values_vec.compress(mask).to_array().map(f64::to_bits), + expected_compress + ); + assert_eq!( + simd.compress_f64x4(values_vec, mask) + .to_array() + .map(f64::to_bits), + expected_compress + ); + assert_eq!( + values_vec + .compress_merge(mask, merge_vec) + .to_array() + .map(f64::to_bits), + expected_compress_merge + ); + assert_eq!( + simd.compress_merge_f64x4(values_vec, mask, merge_vec) + .to_array() + .map(f64::to_bits), + expected_compress_merge + ); + assert_eq!( + values_vec.expand(mask).to_array().map(f64::to_bits), + expected_expand + ); + assert_eq!( + simd.expand_f64x4(values_vec, mask) + .to_array() + .map(f64::to_bits), + expected_expand + ); + assert_eq!( + values_vec + .expand_merge(mask, merge_vec) + .to_array() + .map(f64::to_bits), + expected_expand_merge + ); + assert_eq!( + simd.expand_merge_f64x4(values_vec, mask, merge_vec) + .to_array() + .map(f64::to_bits), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_f64x8(simd: S) { + let patterns: [u64; 8] = [ + 0x8000_0000_0000_0000, + 0x7ff8_1234_5678_9abc, + 0x7ff0_0000_0000_0000, + 0x7ff0_0000_0000_0001, + 0, + 0xfff0_0000_0000_0000, + 0xfff8_abcd_1234_5678, + 0x3ff0_0000_0000_0000, + ]; + let values: [u64; 8] = core::array::from_fn(|lane| patterns[lane % 8]); + let merge: [u64; 8] = core::array::from_fn(|lane| patterns[(lane + 3) % 8]); + let values_vec = f64x8::simd_from(simd, values.map(f64::from_bits)); + let merge_vec = f64x8::simd_from(simd, merge.map(f64::from_bits)); + let all_lanes = u64::MAX >> (64 - 8); + let single_lanes = (0..8).map(|lane| 1_u64 << lane); + let prefixes = (1..=8).map(|count| u64::MAX >> (64 - count)); + let suffixes = (0..8).map(|lane| all_lanes << lane); + for mask_bits in [ + 0, + all_lanes, + 0xaaaa_aaaa_aaaa_aaaa, + 0x5555_5555_5555_5555, + 0x936c_a5f0_817e_42bd, + ] + .into_iter() + .chain(single_lanes) + .chain(prefixes) + .chain(suffixes) + .chain(0..=all_lanes) + { + let mask = mask64x8::from_bitmask(simd, mask_bits); + let mut expected_compress = [0; 8]; + let mut expected_compress_merge = merge; + let mut expected_expand = [0; 8]; + let mut expected_expand_merge = merge; + let mut selected = 0; + for lane in 0..8 { + if mask_bits & (1_u64 << lane) != 0 { + expected_compress[selected] = values[lane]; + expected_compress_merge[selected] = values[lane]; + expected_expand[lane] = values[selected]; + expected_expand_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!( + values_vec.compress(mask).to_array().map(f64::to_bits), + expected_compress + ); + assert_eq!( + simd.compress_f64x8(values_vec, mask) + .to_array() + .map(f64::to_bits), + expected_compress + ); + assert_eq!( + values_vec + .compress_merge(mask, merge_vec) + .to_array() + .map(f64::to_bits), + expected_compress_merge + ); + assert_eq!( + simd.compress_merge_f64x8(values_vec, mask, merge_vec) + .to_array() + .map(f64::to_bits), + expected_compress_merge + ); + assert_eq!( + values_vec.expand(mask).to_array().map(f64::to_bits), + expected_expand + ); + assert_eq!( + simd.expand_f64x8(values_vec, mask) + .to_array() + .map(f64::to_bits), + expected_expand + ); + assert_eq!( + values_vec + .expand_merge(mask, merge_vec) + .to_array() + .map(f64::to_bits), + expected_expand_merge + ); + assert_eq!( + simd.expand_merge_f64x8(values_vec, mask, merge_vec) + .to_array() + .map(f64::to_bits), + expected_expand_merge + ); + } +} + +#[simd_test] +fn compact_generic_base(simd: S) { + fn compact>(value: V, mask: V::Mask, merge: V::Element) -> [V; 4] { + [ + value.compress(mask), + value.compress_merge(mask, merge), + value.expand(mask), + value.expand_merge(mask, merge), + ] + } + let values = u32x4::simd_from(simd, [10, 20, 30, 40]); + let mask = mask32x4::from_bitmask(simd, 0b1010); + let results = compact(values, mask, 99); + assert_eq!(*results[0], [20, 40, 0, 0]); + assert_eq!(*results[1], [20, 40, 99, 99]); + assert_eq!(*results[2], [0, 10, 0, 20]); + assert_eq!(*results[3], [99, 10, 99, 20]); +} + +#[simd_test] +fn compress_all_widths(simd: S) { + macro_rules! check_width { + ($bytes:ident, $mask:ident, $lanes:literal, $compress:ident, $compress_merge:ident) => {{ + let values: [u8; $lanes] = core::array::from_fn(|lane| (lane * 3 + 1) as u8); + let merge: [u8; $lanes] = core::array::from_fn(|lane| 0xe0_u8.wrapping_add(lane as u8)); + let patterned_mask: u64 = (0..$lanes) + .filter(|lane| lane % 3 == 0 || lane % 7 == 2) + .fold(0, |mask, lane| mask | (1_u64 << lane)); + let values_vec = $bytes::simd_from(simd, values); + let merge_vec = $bytes::simd_from(simd, merge); + let all_lanes = u64::MAX >> (64 - $lanes); + + let repeated_byte_masks = (0_u64..=255).map(|byte_mask| { + (0..($lanes / 8)).fold(0, |mask, block| mask | (byte_mask << (block * 8))) + }); + let stitch_masks = ($lanes == 64).then(|| { + (0..=32).map(|low_count| { + let low = if low_count == 32 { + u64::from(u32::MAX) + } else { + (1_u64 << low_count) - 1 + }; + low | (0xa5a5_a5a5_u64 << 32) + }) + }); + for mask_bits in [0, 1, patterned_mask, all_lanes] + .into_iter() + .chain(repeated_byte_masks) + .chain(stitch_masks.into_iter().flatten()) + { + let mask = $mask::from_bitmask(simd, mask_bits); + let mut expected = [0; $lanes]; + let mut expected_merge = merge; + let mut output_lane = 0; + for (input_lane, value) in values.into_iter().enumerate() { + if mask_bits & (1_u64 << input_lane) != 0 { + expected[output_lane] = value; + expected_merge[output_lane] = value; + output_lane += 1; + } + } + + assert_eq!(*simd.$compress(values_vec, mask), expected); + assert_eq!( + *simd.$compress_merge(values_vec, mask, merge_vec), + expected_merge + ); + } + }}; + } + + check_width!(u8x16, mask8x16, 16, compress_u8x16, compress_merge_u8x16); + check_width!(u8x32, mask8x32, 32, compress_u8x32, compress_merge_u8x32); + check_width!(u8x64, mask8x64, 64, compress_u8x64, compress_merge_u8x64); +} + +#[simd_test] +fn compress_u8x16_exhaustive_masks(simd: S) { + let values = [ + 0, 255, 128, 1, 127, 254, 2, 253, 3, 252, 4, 251, 5, 250, 6, 249, + ]; + let merge = [ + 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, + ]; + let values_vec = u8x16::simd_from(simd, values); + let merge_vec = u8x16::simd_from(simd, merge); + for bits in 0..=u16::MAX { + let mask = mask8x16::from_bitmask(simd, u64::from(bits)); + let mut expected = [0; 16]; + let mut expected_merge = merge; + let mut selected = 0; + for (lane, value) in values.into_iter().enumerate() { + if bits & (1 << lane) != 0 { + expected[selected] = value; + expected_merge[selected] = value; + selected += 1; + } + } + assert_eq!(*values_vec.compress(mask), expected, "mask {bits:#06x}"); + assert_eq!( + *values_vec.compress_merge(mask, merge_vec), + expected_merge, + "mask {bits:#06x}" + ); + } +} diff --git a/fearless_simd_tests/tests/harness/ops/expand.rs b/fearless_simd_tests/tests/harness/ops/expand.rs new file mode 100644 index 00000000..40ece8e5 --- /dev/null +++ b/fearless_simd_tests/tests/harness/ops/expand.rs @@ -0,0 +1,82 @@ +// Copyright 2026 the Fearless_SIMD Authors +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use fearless_simd::*; +use fearless_simd_dev_macros::simd_test; + +#[simd_test] +fn expand_all_widths(simd: S) { + macro_rules! check_width { + ($bytes:ident, $mask:ident, $lanes:literal, $expand:ident, $expand_merge:ident) => {{ + let values: [u8; $lanes] = core::array::from_fn(|lane| (lane * 3 + 1) as u8); + let merge: [u8; $lanes] = core::array::from_fn(|lane| 0xe0_u8.wrapping_add(lane as u8)); + let patterned_mask: u64 = (0..$lanes) + .filter(|lane| lane % 3 == 0 || lane % 7 == 2) + .fold(0, |mask, lane| mask | (1_u64 << lane)); + let values_vec = $bytes::simd_from(simd, values); + let merge_vec = $bytes::simd_from(simd, merge); + let all_lanes = u64::MAX >> (64 - $lanes); + + let repeated_byte_masks = (0_u64..=255).map(|byte_mask| { + (0..($lanes / 8)).fold(0, |mask, block| mask | (byte_mask << (block * 8))) + }); + for mask_bits in [0, 1, patterned_mask, all_lanes] + .into_iter() + .chain(repeated_byte_masks) + { + let mask = $mask::from_bitmask(simd, mask_bits); + let mut expected = [0; $lanes]; + let mut expected_merge = merge; + let mut input_lane = 0; + for lane in 0..$lanes { + if mask_bits & (1_u64 << lane) != 0 { + expected[lane] = values[input_lane]; + expected_merge[lane] = values[input_lane]; + input_lane += 1; + } + } + + assert_eq!(*simd.$expand(values_vec, mask), expected); + assert_eq!( + *simd.$expand_merge(values_vec, mask, merge_vec), + expected_merge + ); + } + }}; + } + + check_width!(u8x16, mask8x16, 16, expand_u8x16, expand_merge_u8x16); + check_width!(u8x32, mask8x32, 32, expand_u8x32, expand_merge_u8x32); + check_width!(u8x64, mask8x64, 64, expand_u8x64, expand_merge_u8x64); +} + +#[simd_test] +fn expand_u8x16_exhaustive_masks(simd: S) { + let values = [ + 0, 255, 128, 1, 127, 254, 2, 253, 3, 252, 4, 251, 5, 250, 6, 249, + ]; + let merge = [ + 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, + ]; + let values_vec = u8x16::simd_from(simd, values); + let merge_vec = u8x16::simd_from(simd, merge); + for bits in 0..=u16::MAX { + let mask = mask8x16::from_bitmask(simd, u64::from(bits)); + let mut expected = [0; 16]; + let mut expected_merge = merge; + let mut selected = 0; + for lane in 0..16 { + if bits & (1 << lane) != 0 { + expected[lane] = values[selected]; + expected_merge[lane] = values[selected]; + selected += 1; + } + } + assert_eq!(*values_vec.expand(mask), expected, "mask {bits:#06x}"); + assert_eq!( + *values_vec.expand_merge(mask, merge_vec), + expected_merge, + "mask {bits:#06x}" + ); + } +} diff --git a/fearless_simd_tests/tests/harness/ops/mod.rs b/fearless_simd_tests/tests/harness/ops/mod.rs index f44766f2..99f2a88b 100644 --- a/fearless_simd_tests/tests/harness/ops/mod.rs +++ b/fearless_simd_tests/tests/harness/ops/mod.rs @@ -17,6 +17,7 @@ mod bitcast; mod block_splat; mod ceil; mod combine; +mod compress; mod concat_swizzle_dyn; mod concat_swizzle_dyn_precise; mod copysign; @@ -34,6 +35,7 @@ mod cvt_u64; mod cvt_u64_precise; mod deinterleave; mod div; +mod expand; mod floor; mod fp_classify; mod fract;