Skip to content

Make X86 intrinsics const #1925

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 20 commits into
base: main
Choose a base branch
from
+7,441 −3,355
Draft

Make X86 intrinsics const #1925

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
a0e2c7e
Make some helpers `const`
sayantn Sep 14, 2025
445a5b1
Use ordered reduction intrinsics for integer reductions
sayantn Nov 10, 2025
667759f
Make `abm`, `bmi1`, `bmi2`, `bswap` and `tbm` functions const
sayantn Sep 23, 2025
974eaa2
Make `sse` functions const
sayantn Sep 23, 2025
f430c53
Make `sse2` functions const
sayantn Sep 23, 2025
2025219
Make `sse3` functions const
sayantn Sep 23, 2025
0a858fe
Make `ssse3` functions const
sayantn Sep 23, 2025
4fe36c1
Make `sse4.1` functions const
sayantn Sep 23, 2025
d278183
Make `sse4.2` functions const
sayantn Sep 23, 2025
102a873
Make `avx` functions const
sayantn Sep 23, 2025
19065aa
Make `avx2` functions const
sayantn Sep 23, 2025
a79a18d
Make `fma` functions const
sayantn Oct 1, 2025
654648a
Make `avx512f` functions const
sayantn Sep 23, 2025
314294a
Make `avx512bw` functions const
sayantn Sep 23, 2025
18f5c76
Make `avx512cd` functions const
sayantn Sep 23, 2025
310f5c7
Make `avx512dq` functions const
sayantn Sep 23, 2025
84164c5
Make `avx512vbmi2` functions const
sayantn Sep 23, 2025
c916023
Make `avx512bitalg` functions const
sayantn Sep 23, 2025
f6113b6
Make `avx512vpopcntdq` functions const
sayantn Sep 23, 2025
4d1ed15
Make `avx512fp16` functions const
sayantn Sep 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion crates/core_arch/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,10 @@
aarch64_unstable_target_feature,
bigint_helper_methods,
funnel_shifts,
avx10_target_feature
avx10_target_feature,
const_trait_impl,
const_cmp,
const_convert
)]
#![cfg_attr(test, feature(test, abi_vectorcall, stdarch_internal))]
#![deny(clippy::missing_inline_in_public_items)]
Expand Down
12 changes: 7 additions & 5 deletions crates/core_arch/src/simd.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,8 @@ macro_rules! simd_ty {
}
// FIXME: Workaround rust@60637
#[inline(always)]
pub(crate) fn splat(value: $elem_type) -> Self {
#[rustc_const_unstable(feature = "stdarch_const_intrinsics", issue = "none")]
pub(crate) const fn splat(value: $elem_type) -> Self {
#[derive(Copy, Clone)]
#[repr(simd)]
struct JustOne([$elem_type; 1]);
Expand All @@ -38,12 +39,12 @@ macro_rules! simd_ty {
/// Use for testing only.
// FIXME: Workaround rust@60637
#[inline(always)]
pub(crate) fn extract(&self, index: usize) -> $elem_type {
pub(crate) const fn extract(&self, index: usize) -> $elem_type {
self.as_array()[index]
}

#[inline]
pub(crate) fn as_array(&self) -> &[$elem_type; $len] {
pub(crate) const fn as_array(&self) -> &[$elem_type; $len] {
let simd_ptr: *const Self = self;
let array_ptr: *const [$elem_type; $len] = simd_ptr.cast();
// SAFETY: We can always read the prefix of a simd type as an array.
Expand Down Expand Up @@ -89,7 +90,8 @@ macro_rules! simd_m_ty {

// FIXME: Workaround rust@60637
#[inline(always)]
pub(crate) fn splat(value: bool) -> Self {
#[rustc_const_unstable(feature = "stdarch_const_intrinsics", issue = "none")]
pub(crate) const fn splat(value: bool) -> Self {
#[derive(Copy, Clone)]
#[repr(simd)]
struct JustOne([$elem_type; 1]);
Expand All @@ -100,7 +102,7 @@ macro_rules! simd_m_ty {
}

#[inline]
pub(crate) fn as_array(&self) -> &[$elem_type; $len] {
pub(crate) const fn as_array(&self) -> &[$elem_type; $len] {
let simd_ptr: *const Self = self;
let array_ptr: *const [$elem_type; $len] = simd_ptr.cast();
// SAFETY: We can always read the prefix of a simd type as an array.
Expand Down
6 changes: 4 additions & 2 deletions crates/core_arch/src/x86/abm.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,8 @@ use stdarch_test::assert_instr;
#[target_feature(enable = "lzcnt")]
#[cfg_attr(test, assert_instr(lzcnt))]
#[stable(feature = "simd_x86", since = "1.27.0")]
pub fn _lzcnt_u32(x: u32) -> u32 {
#[rustc_const_unstable(feature = "stdarch_const_intrinsics", issue = "none")]
pub const fn _lzcnt_u32(x: u32) -> u32 {
x.leading_zeros()
}

Expand All @@ -40,7 +41,8 @@ pub fn _lzcnt_u32(x: u32) -> u32 {
#[target_feature(enable = "popcnt")]
#[cfg_attr(test, assert_instr(popcnt))]
#[stable(feature = "simd_x86", since = "1.27.0")]
pub fn _popcnt32(x: i32) -> i32 {
#[rustc_const_unstable(feature = "stdarch_const_intrinsics", issue = "none")]
pub const fn _popcnt32(x: i32) -> i32 {
x.count_ones() as i32
}

Expand Down
459 changes: 312 additions & 147 deletions crates/core_arch/src/x86/avx.rs
View file
Open in desktop

Large diffs are not rendered by default.

408 changes: 272 additions & 136 deletions crates/core_arch/src/x86/avx2.rs
View file
Open in desktop

Large diffs are not rendered by default.

54 changes: 36 additions & 18 deletions crates/core_arch/src/x86/avx512bitalg.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,8 @@ unsafe extern "C" {
#[target_feature(enable = "avx512bitalg")]
#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
#[cfg_attr(test, assert_instr(vpopcntw))]
pub fn _mm512_popcnt_epi16(a: __m512i) -> __m512i {
#[rustc_const_unstable(feature = "stdarch_const_intrinsics", issue = "none")]
pub const fn _mm512_popcnt_epi16(a: __m512i) -> __m512i {
unsafe { transmute(simd_ctpop(a.as_i16x32())) }
}

Expand All @@ -57,7 +58,8 @@ pub fn _mm512_popcnt_epi16(a: __m512i) -> __m512i {
#[target_feature(enable = "avx512bitalg")]
#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
#[cfg_attr(test, assert_instr(vpopcntw))]
pub fn _mm512_maskz_popcnt_epi16(k: __mmask32, a: __m512i) -> __m512i {
#[rustc_const_unstable(feature = "stdarch_const_intrinsics", issue = "none")]
pub const fn _mm512_maskz_popcnt_epi16(k: __mmask32, a: __m512i) -> __m512i {
unsafe {
transmute(simd_select_bitmask(
k,
Expand All @@ -77,7 +79,8 @@ pub fn _mm512_maskz_popcnt_epi16(k: __mmask32, a: __m512i) -> __m512i {
#[target_feature(enable = "avx512bitalg")]
#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
#[cfg_attr(test, assert_instr(vpopcntw))]
pub fn _mm512_mask_popcnt_epi16(src: __m512i, k: __mmask32, a: __m512i) -> __m512i {
#[rustc_const_unstable(feature = "stdarch_const_intrinsics", issue = "none")]
pub const fn _mm512_mask_popcnt_epi16(src: __m512i, k: __mmask32, a: __m512i) -> __m512i {
unsafe {
transmute(simd_select_bitmask(
k,
Expand All @@ -94,7 +97,8 @@ pub fn _mm512_mask_popcnt_epi16(src: __m512i, k: __mmask32, a: __m512i) -> __m51
#[target_feature(enable = "avx512bitalg,avx512vl")]
#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
#[cfg_attr(test, assert_instr(vpopcntw))]
pub fn _mm256_popcnt_epi16(a: __m256i) -> __m256i {
#[rustc_const_unstable(feature = "stdarch_const_intrinsics", issue = "none")]
pub const fn _mm256_popcnt_epi16(a: __m256i) -> __m256i {
unsafe { transmute(simd_ctpop(a.as_i16x16())) }
}

Expand All @@ -108,7 +112,8 @@ pub fn _mm256_popcnt_epi16(a: __m256i) -> __m256i {
#[target_feature(enable = "avx512bitalg,avx512vl")]
#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
#[cfg_attr(test, assert_instr(vpopcntw))]
pub fn _mm256_maskz_popcnt_epi16(k: __mmask16, a: __m256i) -> __m256i {
#[rustc_const_unstable(feature = "stdarch_const_intrinsics", issue = "none")]
pub const fn _mm256_maskz_popcnt_epi16(k: __mmask16, a: __m256i) -> __m256i {
unsafe {
transmute(simd_select_bitmask(
k,
Expand All @@ -128,7 +133,8 @@ pub fn _mm256_maskz_popcnt_epi16(k: __mmask16, a: __m256i) -> __m256i {
#[target_feature(enable = "avx512bitalg,avx512vl")]
#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
#[cfg_attr(test, assert_instr(vpopcntw))]
pub fn _mm256_mask_popcnt_epi16(src: __m256i, k: __mmask16, a: __m256i) -> __m256i {
#[rustc_const_unstable(feature = "stdarch_const_intrinsics", issue = "none")]
pub const fn _mm256_mask_popcnt_epi16(src: __m256i, k: __mmask16, a: __m256i) -> __m256i {
unsafe {
transmute(simd_select_bitmask(
k,
Expand All @@ -145,7 +151,8 @@ pub fn _mm256_mask_popcnt_epi16(src: __m256i, k: __mmask16, a: __m256i) -> __m25
#[target_feature(enable = "avx512bitalg,avx512vl")]
#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
#[cfg_attr(test, assert_instr(vpopcntw))]
pub fn _mm_popcnt_epi16(a: __m128i) -> __m128i {
#[rustc_const_unstable(feature = "stdarch_const_intrinsics", issue = "none")]
pub const fn _mm_popcnt_epi16(a: __m128i) -> __m128i {
unsafe { transmute(simd_ctpop(a.as_i16x8())) }
}

Expand All @@ -159,7 +166,8 @@ pub fn _mm_popcnt_epi16(a: __m128i) -> __m128i {
#[target_feature(enable = "avx512bitalg,avx512vl")]
#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
#[cfg_attr(test, assert_instr(vpopcntw))]
pub fn _mm_maskz_popcnt_epi16(k: __mmask8, a: __m128i) -> __m128i {
#[rustc_const_unstable(feature = "stdarch_const_intrinsics", issue = "none")]
pub const fn _mm_maskz_popcnt_epi16(k: __mmask8, a: __m128i) -> __m128i {
unsafe {
transmute(simd_select_bitmask(
k,
Expand All @@ -179,7 +187,8 @@ pub fn _mm_maskz_popcnt_epi16(k: __mmask8, a: __m128i) -> __m128i {
#[target_feature(enable = "avx512bitalg,avx512vl")]
#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
#[cfg_attr(test, assert_instr(vpopcntw))]
pub fn _mm_mask_popcnt_epi16(src: __m128i, k: __mmask8, a: __m128i) -> __m128i {
#[rustc_const_unstable(feature = "stdarch_const_intrinsics", issue = "none")]
pub const fn _mm_mask_popcnt_epi16(src: __m128i, k: __mmask8, a: __m128i) -> __m128i {
unsafe {
transmute(simd_select_bitmask(
k,
Expand All @@ -196,7 +205,8 @@ pub fn _mm_mask_popcnt_epi16(src: __m128i, k: __mmask8, a: __m128i) -> __m128i {
#[target_feature(enable = "avx512bitalg")]
#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
#[cfg_attr(test, assert_instr(vpopcntb))]
pub fn _mm512_popcnt_epi8(a: __m512i) -> __m512i {
#[rustc_const_unstable(feature = "stdarch_const_intrinsics", issue = "none")]
pub const fn _mm512_popcnt_epi8(a: __m512i) -> __m512i {
unsafe { transmute(simd_ctpop(a.as_i8x64())) }
}

Expand All @@ -210,7 +220,8 @@ pub fn _mm512_popcnt_epi8(a: __m512i) -> __m512i {
#[target_feature(enable = "avx512bitalg")]
#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
#[cfg_attr(test, assert_instr(vpopcntb))]
pub fn _mm512_maskz_popcnt_epi8(k: __mmask64, a: __m512i) -> __m512i {
#[rustc_const_unstable(feature = "stdarch_const_intrinsics", issue = "none")]
pub const fn _mm512_maskz_popcnt_epi8(k: __mmask64, a: __m512i) -> __m512i {
unsafe {
transmute(simd_select_bitmask(
k,
Expand All @@ -230,7 +241,8 @@ pub fn _mm512_maskz_popcnt_epi8(k: __mmask64, a: __m512i) -> __m512i {
#[target_feature(enable = "avx512bitalg")]
#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
#[cfg_attr(test, assert_instr(vpopcntb))]
pub fn _mm512_mask_popcnt_epi8(src: __m512i, k: __mmask64, a: __m512i) -> __m512i {
#[rustc_const_unstable(feature = "stdarch_const_intrinsics", issue = "none")]
pub const fn _mm512_mask_popcnt_epi8(src: __m512i, k: __mmask64, a: __m512i) -> __m512i {
unsafe {
transmute(simd_select_bitmask(
k,
Expand All @@ -247,7 +259,8 @@ pub fn _mm512_mask_popcnt_epi8(src: __m512i, k: __mmask64, a: __m512i) -> __m512
#[target_feature(enable = "avx512bitalg,avx512vl")]
#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
#[cfg_attr(test, assert_instr(vpopcntb))]
pub fn _mm256_popcnt_epi8(a: __m256i) -> __m256i {
#[rustc_const_unstable(feature = "stdarch_const_intrinsics", issue = "none")]
pub const fn _mm256_popcnt_epi8(a: __m256i) -> __m256i {
unsafe { transmute(simd_ctpop(a.as_i8x32())) }
}

Expand All @@ -261,7 +274,8 @@ pub fn _mm256_popcnt_epi8(a: __m256i) -> __m256i {
#[target_feature(enable = "avx512bitalg,avx512vl")]
#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
#[cfg_attr(test, assert_instr(vpopcntb))]
pub fn _mm256_maskz_popcnt_epi8(k: __mmask32, a: __m256i) -> __m256i {
#[rustc_const_unstable(feature = "stdarch_const_intrinsics", issue = "none")]
pub const fn _mm256_maskz_popcnt_epi8(k: __mmask32, a: __m256i) -> __m256i {
unsafe {
transmute(simd_select_bitmask(
k,
Expand All @@ -281,7 +295,8 @@ pub fn _mm256_maskz_popcnt_epi8(k: __mmask32, a: __m256i) -> __m256i {
#[target_feature(enable = "avx512bitalg,avx512vl")]
#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
#[cfg_attr(test, assert_instr(vpopcntb))]
pub fn _mm256_mask_popcnt_epi8(src: __m256i, k: __mmask32, a: __m256i) -> __m256i {
#[rustc_const_unstable(feature = "stdarch_const_intrinsics", issue = "none")]
pub const fn _mm256_mask_popcnt_epi8(src: __m256i, k: __mmask32, a: __m256i) -> __m256i {
unsafe {
transmute(simd_select_bitmask(
k,
Expand All @@ -298,7 +313,8 @@ pub fn _mm256_mask_popcnt_epi8(src: __m256i, k: __mmask32, a: __m256i) -> __m256
#[target_feature(enable = "avx512bitalg,avx512vl")]
#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
#[cfg_attr(test, assert_instr(vpopcntb))]
pub fn _mm_popcnt_epi8(a: __m128i) -> __m128i {
#[rustc_const_unstable(feature = "stdarch_const_intrinsics", issue = "none")]
pub const fn _mm_popcnt_epi8(a: __m128i) -> __m128i {
unsafe { transmute(simd_ctpop(a.as_i8x16())) }
}

Expand All @@ -312,7 +328,8 @@ pub fn _mm_popcnt_epi8(a: __m128i) -> __m128i {
#[target_feature(enable = "avx512bitalg,avx512vl")]
#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
#[cfg_attr(test, assert_instr(vpopcntb))]
pub fn _mm_maskz_popcnt_epi8(k: __mmask16, a: __m128i) -> __m128i {
#[rustc_const_unstable(feature = "stdarch_const_intrinsics", issue = "none")]
pub const fn _mm_maskz_popcnt_epi8(k: __mmask16, a: __m128i) -> __m128i {
unsafe {
transmute(simd_select_bitmask(
k,
Expand All @@ -332,7 +349,8 @@ pub fn _mm_maskz_popcnt_epi8(k: __mmask16, a: __m128i) -> __m128i {
#[target_feature(enable = "avx512bitalg,avx512vl")]
#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
#[cfg_attr(test, assert_instr(vpopcntb))]
pub fn _mm_mask_popcnt_epi8(src: __m128i, k: __mmask16, a: __m128i) -> __m128i {
#[rustc_const_unstable(feature = "stdarch_const_intrinsics", issue = "none")]
pub const fn _mm_mask_popcnt_epi8(src: __m128i, k: __mmask16, a: __m128i) -> __m128i {
unsafe {
transmute(simd_select_bitmask(
k,
Expand Down
Loading
Loading