From d0efc9e8743327347721afebcd1dfb4b068bcb23 Mon Sep 17 00:00:00 2001 From: TIHan Date: Tue, 2 Jul 2024 12:54:41 -0700 Subject: [PATCH 01/83] Initial work --- src/coreclr/jit/gentree.cpp | 4 +- src/coreclr/jit/hwintrinsic.cpp | 1 + src/coreclr/jit/hwintrinsiccodegenarm64.cpp | 1 + src/coreclr/jit/hwintrinsiclistarm64sve.h | 4 + .../Arm/Sve.PlatformNotSupported.cs | 226 +++++++++ .../src/System/Runtime/Intrinsics/Arm/Sve.cs | 227 +++++++++ .../ref/System.Runtime.Intrinsics.cs | 43 ++ .../GenerateHWIntrinsicTests_Arm.cs | 25 + .../_SveLoadVectorFirstFaultingTest.template | 436 ++++++++++++++++++ 9 files changed, 966 insertions(+), 1 deletion(-) create mode 100644 src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveLoadVectorFirstFaultingTest.template diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 6d3d670b2dc7d..97bad35e7425f 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -27421,6 +27421,7 @@ bool GenTreeHWIntrinsic::OperIsMemoryLoad(GenTree** pAddr) const case NI_Sve_LoadVectorByteZeroExtendToUInt16: case NI_Sve_LoadVectorByteZeroExtendToUInt32: case NI_Sve_LoadVectorByteZeroExtendToUInt64: + case NI_Sve_LoadVectorFirstFaulting: case NI_Sve_LoadVectorInt16SignExtendToInt32: case NI_Sve_LoadVectorInt16SignExtendToInt64: case NI_Sve_LoadVectorInt16SignExtendToUInt32: @@ -27451,6 +27452,7 @@ bool GenTreeHWIntrinsic::OperIsMemoryLoad(GenTree** pAddr) const case NI_Sve_GatherVector: case NI_Sve_GatherVectorByteZeroExtend: + case NI_Sve_GatherVectorFirstFaulting: case NI_Sve_GatherVectorInt16SignExtend: case NI_Sve_GatherVectorInt16WithByteOffsetsSignExtend: case NI_Sve_GatherVectorInt32SignExtend: @@ -27542,7 +27544,7 @@ bool GenTreeHWIntrinsic::OperIsMemoryLoad(GenTree** pAddr) const { #ifdef TARGET_ARM64 static_assert_no_msg( - AreContiguous(NI_Sve_GatherVector, NI_Sve_GatherVectorByteZeroExtend, NI_Sve_GatherVectorInt16SignExtend, + AreContiguous(NI_Sve_GatherVector, NI_Sve_GatherVectorByteZeroExtend, NI_Sve_GatherVectorFirstFaulting, NI_Sve_GatherVectorInt16SignExtend, NI_Sve_GatherVectorInt16WithByteOffsetsSignExtend, NI_Sve_GatherVectorInt32SignExtend, NI_Sve_GatherVectorInt32WithByteOffsetsSignExtend, NI_Sve_GatherVectorSByteSignExtend, NI_Sve_GatherVectorUInt16WithByteOffsetsZeroExtend, NI_Sve_GatherVectorUInt16ZeroExtend, diff --git a/src/coreclr/jit/hwintrinsic.cpp b/src/coreclr/jit/hwintrinsic.cpp index 9db2767f50796..784f48035cf2c 100644 --- a/src/coreclr/jit/hwintrinsic.cpp +++ b/src/coreclr/jit/hwintrinsic.cpp @@ -1858,6 +1858,7 @@ GenTree* Compiler::impHWIntrinsic(NamedIntrinsic intrinsic, #elif defined(TARGET_ARM64) case NI_Sve_GatherVector: case NI_Sve_GatherVectorByteZeroExtend: + case NI_Sve_GatherVectorFirstFaulting: case NI_Sve_GatherVectorInt16SignExtend: case NI_Sve_GatherVectorInt16WithByteOffsetsSignExtend: case NI_Sve_GatherVectorInt32SignExtend: diff --git a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp index c568a3bad4977..ba7def282493b 100644 --- a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp @@ -1994,6 +1994,7 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) case NI_Sve_GatherVector: case NI_Sve_GatherVectorByteZeroExtend: + case NI_Sve_GatherVectorFirstFaulting: case NI_Sve_GatherVectorInt16SignExtend: case NI_Sve_GatherVectorInt16WithByteOffsetsSignExtend: case NI_Sve_GatherVectorInt32SignExtend: diff --git a/src/coreclr/jit/hwintrinsiclistarm64sve.h b/src/coreclr/jit/hwintrinsiclistarm64sve.h index 688691124eab6..244888e1dd39c 100644 --- a/src/coreclr/jit/hwintrinsiclistarm64sve.h +++ b/src/coreclr/jit/hwintrinsiclistarm64sve.h @@ -82,6 +82,7 @@ HARDWARE_INTRINSIC(Sve, FusedMultiplySubtractBySelectedScalar, HARDWARE_INTRINSIC(Sve, FusedMultiplySubtractNegated, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fnmls, INS_sve_fnmls}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_FmaIntrinsic|HW_Flag_SpecialCodeGen) HARDWARE_INTRINSIC(Sve, GatherVector, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_sve_ld1w, INS_sve_ld1d, INS_sve_ld1d, INS_sve_ld1w, INS_sve_ld1d}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, GatherVectorByteZeroExtend, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1b, INS_sve_ld1b, INS_sve_ld1b, INS_sve_ld1b, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, GatherVectorFirstFaulting, -1, 3, true, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1w, INS_sve_ldff1w, INS_sve_ldff1d, INS_sve_ldff1d, INS_sve_ldff1w, INS_sve_ldff1d}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, GatherVectorInt16SignExtend, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sh, INS_sve_ld1sh, INS_sve_ld1sh, INS_sve_ld1sh, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, GatherVectorInt16WithByteOffsetsSignExtend, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sh, INS_sve_ld1sh, INS_sve_ld1sh, INS_sve_ld1sh, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, GatherVectorInt32SignExtend, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sw, INS_sve_ld1sw, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) @@ -93,6 +94,7 @@ HARDWARE_INTRINSIC(Sve, GatherVectorUInt32WithByteOffsetsZeroExtend, HARDWARE_INTRINSIC(Sve, GatherVectorUInt32ZeroExtend, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_sve_ld1w, INS_sve_ld1w, INS_sve_ld1w, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, GatherVectorWithByteOffsets, -1, 3, true, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_sve_ld1w, INS_sve_ld1d, INS_sve_ld1d, INS_sve_ld1w, INS_sve_ld1d}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, GetActiveElementCount, -1, 2, true, {INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ExplicitMaskedOperation) +HARDWARE_INTRINSIC(Sve, GetFfr, -1, 0, true, {INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) HARDWARE_INTRINSIC(Sve, InsertIntoShiftedVector, -1, 2, true, {INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics) HARDWARE_INTRINSIC(Sve, LeadingSignCount, -1, -1, false, {INS_sve_cls, INS_invalid, INS_sve_cls, INS_invalid, INS_sve_cls, INS_invalid, INS_sve_cls, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LeadingZeroCount, -1, -1, false, {INS_sve_clz, INS_sve_clz, INS_sve_clz, INS_sve_clz, INS_sve_clz, INS_sve_clz, INS_sve_clz, INS_sve_clz, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) @@ -113,6 +115,7 @@ HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendToInt64, HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendToUInt16, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendToUInt32, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendToUInt64, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1b, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorFirstFaulting, -1, 2, true, {INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1h, INS_sve_ldff1h, INS_sve_ldff1w, INS_sve_ldff1w, INS_sve_ldff1d, INS_sve_ldff1d, INS_sve_ldff1w, INS_sve_ldff1d}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorInt16NonFaultingSignExtendToInt32, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorInt16NonFaultingSignExtendToInt64, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sh, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorInt16NonFaultingSignExtendToUInt32, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sh, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) @@ -199,6 +202,7 @@ HARDWARE_INTRINSIC(Sve, SaturatingIncrementBy64BitElementCount, HARDWARE_INTRINSIC(Sve, SaturatingIncrementBy8BitElementCount, 0, 3, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqincb, INS_sve_uqincb, INS_sve_sqincb, INS_sve_uqincb, INS_invalid, INS_invalid}, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasEnumOperand|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) HARDWARE_INTRINSIC(Sve, SaturatingIncrementByActiveElementCount, -1, 2, true, {INS_invalid, INS_sve_sqincp, INS_sve_sqincp, INS_sve_sqincp, INS_sve_sqincp, INS_sve_sqincp, INS_sve_sqincp, INS_sve_sqincp, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) HARDWARE_INTRINSIC(Sve, Scale, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fscale, INS_sve_fscale}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve, SetFfr, -1, 1, true, {INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation) HARDWARE_INTRINSIC(Sve, ShiftLeftLogical, -1, -1, false, {INS_sve_lsl, INS_sve_lsl, INS_sve_lsl, INS_sve_lsl, INS_sve_lsl, INS_sve_lsl, INS_sve_lsl, INS_sve_lsl, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) HARDWARE_INTRINSIC(Sve, ShiftRightArithmetic, -1, -1, false, {INS_sve_asr, INS_invalid, INS_sve_asr, INS_invalid, INS_sve_asr, INS_invalid, INS_sve_asr, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) HARDWARE_INTRINSIC(Sve, ShiftRightLogical, -1, -1, false, {INS_invalid, INS_sve_lsr, INS_invalid, INS_sve_lsr, INS_invalid, INS_sve_lsr, INS_invalid, INS_sve_lsr, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs index 371da71f844ed..54e8ad9846c94 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs @@ -1939,6 +1939,115 @@ internal Arm64() { } public static unsafe Vector GatherVectorByteZeroExtend(Vector mask, byte* address, Vector indices) { throw new PlatformNotSupportedException(); } + /// + /// svfloat64_t svldff1_gather_[s64]index[_f64](svbool_t pg, const float64_t *base, svint64_t indices) + /// LDFF1D Zresult.D, Pg/Z, [Xbase, Zindices.D, LSL #3] + /// + public static unsafe Vector GatherVectorFirstFaulting(Vector mask, double* address, Vector indices) { throw new PlatformNotSupportedException(); } + + /// + /// svfloat64_t svldff1_gather[_u64base]_f64(svbool_t pg, svuint64_t bases) + /// LDFF1D Zresult.D, Pg/Z, [Zbases.D, #0] + /// + public static unsafe Vector GatherVectorFirstFaulting(Vector mask, Vector addresses) { throw new PlatformNotSupportedException(); } + + /// + /// svfloat64_t svldff1_gather_[u64]index[_f64](svbool_t pg, const float64_t *base, svuint64_t indices) + /// LDFF1D Zresult.D, Pg/Z, [Xbase, Zindices.D, LSL #3] + /// + public static unsafe Vector GatherVectorFirstFaulting(Vector mask, double* address, Vector indices) { throw new PlatformNotSupportedException(); } + + /// + /// svint32_t svldff1_gather[_u32base]_s32(svbool_t pg, svuint32_t bases) + /// LDFF1W Zresult.S, Pg/Z, [Zbases.S, #0] + /// + public static unsafe Vector GatherVectorFirstFaulting(Vector mask, Vector addresses) { throw new PlatformNotSupportedException(); } + + /// + /// svint32_t svldff1_gather_[s32]index[_s32](svbool_t pg, const int32_t *base, svint32_t indices) + /// LDFF1W Zresult.S, Pg/Z, [Xbase, Zindices.S, SXTW #2] + /// + public static unsafe Vector GatherVectorFirstFaulting(Vector mask, int* address, Vector indices) { throw new PlatformNotSupportedException(); } + + /// + /// svint32_t svldff1_gather_[u32]index[_s32](svbool_t pg, const int32_t *base, svuint32_t indices) + /// LDFF1W Zresult.S, Pg/Z, [Xbase, Zindices.S, UXTW #2] + /// + public static unsafe Vector GatherVectorFirstFaulting(Vector mask, int* address, Vector indices) { throw new PlatformNotSupportedException(); } + + /// + /// svint64_t svldff1_gather[_u64base]_s64(svbool_t pg, svuint64_t bases) + /// LDFF1D Zresult.D, Pg/Z, [Zbases.D, #0] + /// + public static unsafe Vector GatherVectorFirstFaulting(Vector mask, Vector addresses) { throw new PlatformNotSupportedException(); } + + /// + /// svint64_t svldff1_gather_[s64]index[_s64](svbool_t pg, const int64_t *base, svint64_t indices) + /// LDFF1D Zresult.D, Pg/Z, [Xbase, Zindices.D, LSL #3] + /// + public static unsafe Vector GatherVectorFirstFaulting(Vector mask, long* address, Vector indices) { throw new PlatformNotSupportedException(); } + + /// + /// svint64_t svldff1_gather_[u64]index[_s64](svbool_t pg, const int64_t *base, svuint64_t indices) + /// LDFF1D Zresult.D, Pg/Z, [Xbase, Zindices.D, LSL #3] + /// + public static unsafe Vector GatherVectorFirstFaulting(Vector mask, long* address, Vector indices) { throw new PlatformNotSupportedException(); } + + /// + /// svfloat32_t svldff1_gather_[s32]index[_f32](svbool_t pg, const float32_t *base, svint32_t indices) + /// LDFF1W Zresult.S, Pg/Z, [Xbase, Zindices.S, SXTW #2] + /// + public static unsafe Vector GatherVectorFirstFaulting(Vector mask, float* address, Vector indices) { throw new PlatformNotSupportedException(); } + + /// + /// svfloat32_t svldff1_gather[_u32base]_f32(svbool_t pg, svuint32_t bases) + /// LDFF1W Zresult.S, Pg/Z, [Zbases.S, #0] + /// + public static unsafe Vector GatherVectorFirstFaulting(Vector mask, Vector addresses) { throw new PlatformNotSupportedException(); } + + /// + /// svfloat32_t svldff1_gather_[u32]index[_f32](svbool_t pg, const float32_t *base, svuint32_t indices) + /// LDFF1W Zresult.S, Pg/Z, [Xbase, Zindices.S, UXTW #2] + /// + public static unsafe Vector GatherVectorFirstFaulting(Vector mask, float* address, Vector indices) { throw new PlatformNotSupportedException(); } + + /// + /// svuint32_t svldff1_gather[_u32base]_u32(svbool_t pg, svuint32_t bases) + /// LDFF1W Zresult.S, Pg/Z, [Zbases.S, #0] + /// + public static unsafe Vector GatherVectorFirstFaulting(Vector mask, Vector addresses) { throw new PlatformNotSupportedException(); } + + /// + /// svuint32_t svldff1_gather_[s32]index[_u32](svbool_t pg, const uint32_t *base, svint32_t indices) + /// LDFF1W Zresult.S, Pg/Z, [Xbase, Zindices.S, SXTW #2] + /// + public static unsafe Vector GatherVectorFirstFaulting(Vector mask, uint* address, Vector indices) { throw new PlatformNotSupportedException(); } + + /// + /// svuint32_t svldff1_gather_[u32]index[_u32](svbool_t pg, const uint32_t *base, svuint32_t indices) + /// LDFF1W Zresult.S, Pg/Z, [Xbase, Zindices.S, UXTW #2] + /// + public static unsafe Vector GatherVectorFirstFaulting(Vector mask, uint* address, Vector indices) { throw new PlatformNotSupportedException(); } + + /// + /// svuint64_t svldff1_gather[_u64base]_u64(svbool_t pg, svuint64_t bases) + /// LDFF1D Zresult.D, Pg/Z, [Zbases.D, #0] + /// + public static unsafe Vector GatherVectorFirstFaulting(Vector mask, Vector addresses) { throw new PlatformNotSupportedException(); } + + /// + /// svuint64_t svldff1_gather_[s64]index[_u64](svbool_t pg, const uint64_t *base, svint64_t indices) + /// LDFF1D Zresult.D, Pg/Z, [Xbase, Zindices.D, LSL #3] + /// + public static unsafe Vector GatherVectorFirstFaulting(Vector mask, ulong* address, Vector indices) { throw new PlatformNotSupportedException(); } + + /// + /// svuint64_t svldff1_gather_[u64]index[_u64](svbool_t pg, const uint64_t *base, svuint64_t indices) + /// LDFF1D Zresult.D, Pg/Z, [Xbase, Zindices.D, LSL #3] + /// + public static unsafe Vector GatherVectorFirstFaulting(Vector mask, ulong* address, Vector indices) { throw new PlatformNotSupportedException(); } + + /// Load 16-bit data and sign-extend /// @@ -2604,6 +2713,13 @@ internal Arm64() { } public static unsafe ulong GetActiveElementCount(Vector mask, Vector from) { throw new PlatformNotSupportedException(); } + /// + /// svbool_t svrdffr() + /// RDFFR Presult.B + /// + public static unsafe Vector GetFfr() { throw new PlatformNotSupportedException(); } + + /// Insert scalar into shifted vector /// @@ -3089,6 +3205,67 @@ internal Arm64() { } public static unsafe Vector LoadVectorByteZeroExtendToUInt64(Vector mask, byte* address) { throw new PlatformNotSupportedException(); } + /// + /// svuint8_t svldff1[_u8](svbool_t pg, const uint8_t *base) + /// LDFF1B Zresult.B, Pg/Z, [Xbase, XZR] + /// + public static unsafe Vector LoadVectorFirstFaulting(Vector mask, byte* address) { throw new PlatformNotSupportedException(); } + + /// + /// svfloat64_t svldff1[_f64](svbool_t pg, const float64_t *base) + /// LDFF1D Zresult.D, Pg/Z, [Xbase, XZR, LSL #3] + /// + public static unsafe Vector LoadVectorFirstFaulting(Vector mask, double* address) { throw new PlatformNotSupportedException(); } + + /// + /// svint16_t svldff1[_s16](svbool_t pg, const int16_t *base) + /// LDFF1H Zresult.H, Pg/Z, [Xbase, XZR, LSL #1] + /// + public static unsafe Vector LoadVectorFirstFaulting(Vector mask, short* address) { throw new PlatformNotSupportedException(); } + + /// + /// svint32_t svldff1[_s32](svbool_t pg, const int32_t *base) + /// LDFF1W Zresult.S, Pg/Z, [Xbase, XZR, LSL #2] + /// + public static unsafe Vector LoadVectorFirstFaulting(Vector mask, int* address) { throw new PlatformNotSupportedException(); } + + /// + /// svint64_t svldff1[_s64](svbool_t pg, const int64_t *base) + /// LDFF1D Zresult.D, Pg/Z, [Xbase, XZR, LSL #3] + /// + public static unsafe Vector LoadVectorFirstFaulting(Vector mask, long* address) { throw new PlatformNotSupportedException(); } + + /// + /// svint8_t svldff1[_s8](svbool_t pg, const int8_t *base) + /// LDFF1B Zresult.B, Pg/Z, [Xbase, XZR] + /// + public static unsafe Vector LoadVectorFirstFaulting(Vector mask, sbyte* address) { throw new PlatformNotSupportedException(); } + + /// + /// svfloat32_t svldff1[_f32](svbool_t pg, const float32_t *base) + /// LDFF1W Zresult.S, Pg/Z, [Xbase, XZR, LSL #2] + /// + public static unsafe Vector LoadVectorFirstFaulting(Vector mask, float* address) { throw new PlatformNotSupportedException(); } + + /// + /// svuint16_t svldff1[_u16](svbool_t pg, const uint16_t *base) + /// LDFF1H Zresult.H, Pg/Z, [Xbase, XZR, LSL #1] + /// + public static unsafe Vector LoadVectorFirstFaulting(Vector mask, ushort* address) { throw new PlatformNotSupportedException(); } + + /// + /// svuint32_t svldff1[_u32](svbool_t pg, const uint32_t *base) + /// LDFF1W Zresult.S, Pg/Z, [Xbase, XZR, LSL #2] + /// + public static unsafe Vector LoadVectorFirstFaulting(Vector mask, uint* address) { throw new PlatformNotSupportedException(); } + + /// + /// svuint64_t svldff1[_u64](svbool_t pg, const uint64_t *base) + /// LDFF1D Zresult.D, Pg/Z, [Xbase, XZR, LSL #3] + /// + public static unsafe Vector LoadVectorFirstFaulting(Vector mask, ulong* address) { throw new PlatformNotSupportedException(); } + + /// LoadVectorInt16SignExtendToInt32 : Load 16-bit data and sign-extend /// @@ -5654,6 +5831,55 @@ internal Arm64() { } public static unsafe Vector Scale(Vector left, Vector right) { throw new PlatformNotSupportedException(); } + /// + /// void svwrffr(svbool_t op) + /// WRFFR Pop.B + /// + public static unsafe void SetFfr(Vector value) { throw new PlatformNotSupportedException(); } + + /// + /// void svwrffr(svbool_t op) + /// WRFFR Pop.B + /// + public static unsafe void SetFfr(Vector value) { throw new PlatformNotSupportedException(); } + + /// + /// void svwrffr(svbool_t op) + /// WRFFR Pop.B + /// + public static unsafe void SetFfr(Vector value) { throw new PlatformNotSupportedException(); } + + /// + /// void svwrffr(svbool_t op) + /// WRFFR Pop.B + /// + public static unsafe void SetFfr(Vector value) { throw new PlatformNotSupportedException(); } + + /// + /// void svwrffr(svbool_t op) + /// WRFFR Pop.B + /// + public static unsafe void SetFfr(Vector value) { throw new PlatformNotSupportedException(); } + + /// + /// void svwrffr(svbool_t op) + /// WRFFR Pop.B + /// + public static unsafe void SetFfr(Vector value) { throw new PlatformNotSupportedException(); } + + /// + /// void svwrffr(svbool_t op) + /// WRFFR Pop.B + /// + public static unsafe void SetFfr(Vector value) { throw new PlatformNotSupportedException(); } + + /// + /// void svwrffr(svbool_t op) + /// WRFFR Pop.B + /// + public static unsafe void SetFfr(Vector value) { throw new PlatformNotSupportedException(); } + + /// Logical shift left /// diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs index 39d8dde224aa9..be3b4cc81af9c 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs @@ -1996,6 +1996,116 @@ internal Arm64() { } /// public static unsafe Vector GatherVectorByteZeroExtend(Vector mask, byte* address, Vector indices) => GatherVectorByteZeroExtend(mask, address, indices); + + /// + /// svfloat64_t svldff1_gather_[s64]index[_f64](svbool_t pg, const float64_t *base, svint64_t indices) + /// LDFF1D Zresult.D, Pg/Z, [Xbase, Zindices.D, LSL #3] + /// + public static unsafe Vector GatherVectorFirstFaulting(Vector mask, double* address, Vector indices) => GatherVectorFirstFaulting(mask, address, indices); + + /// + /// svfloat64_t svldff1_gather[_u64base]_f64(svbool_t pg, svuint64_t bases) + /// LDFF1D Zresult.D, Pg/Z, [Zbases.D, #0] + /// + public static unsafe Vector GatherVectorFirstFaulting(Vector mask, Vector addresses) => GatherVectorFirstFaulting(mask, addresses); + + /// + /// svfloat64_t svldff1_gather_[u64]index[_f64](svbool_t pg, const float64_t *base, svuint64_t indices) + /// LDFF1D Zresult.D, Pg/Z, [Xbase, Zindices.D, LSL #3] + /// + public static unsafe Vector GatherVectorFirstFaulting(Vector mask, double* address, Vector indices) => GatherVectorFirstFaulting(mask, address, indices); + + /// + /// svint32_t svldff1_gather[_u32base]_s32(svbool_t pg, svuint32_t bases) + /// LDFF1W Zresult.S, Pg/Z, [Zbases.S, #0] + /// + public static unsafe Vector GatherVectorFirstFaulting(Vector mask, Vector addresses) => GatherVectorFirstFaulting(mask, addresses); + + /// + /// svint32_t svldff1_gather_[s32]index[_s32](svbool_t pg, const int32_t *base, svint32_t indices) + /// LDFF1W Zresult.S, Pg/Z, [Xbase, Zindices.S, SXTW #2] + /// + public static unsafe Vector GatherVectorFirstFaulting(Vector mask, int* address, Vector indices) => GatherVectorFirstFaulting(mask, address, indices); + + /// + /// svint32_t svldff1_gather_[u32]index[_s32](svbool_t pg, const int32_t *base, svuint32_t indices) + /// LDFF1W Zresult.S, Pg/Z, [Xbase, Zindices.S, UXTW #2] + /// + public static unsafe Vector GatherVectorFirstFaulting(Vector mask, int* address, Vector indices) => GatherVectorFirstFaulting(mask, address, indices); + + /// + /// svint64_t svldff1_gather[_u64base]_s64(svbool_t pg, svuint64_t bases) + /// LDFF1D Zresult.D, Pg/Z, [Zbases.D, #0] + /// + public static unsafe Vector GatherVectorFirstFaulting(Vector mask, Vector addresses) => GatherVectorFirstFaulting(mask, addresses); + + /// + /// svint64_t svldff1_gather_[s64]index[_s64](svbool_t pg, const int64_t *base, svint64_t indices) + /// LDFF1D Zresult.D, Pg/Z, [Xbase, Zindices.D, LSL #3] + /// + public static unsafe Vector GatherVectorFirstFaulting(Vector mask, long* address, Vector indices) => GatherVectorFirstFaulting(mask, address, indices); + + /// + /// svint64_t svldff1_gather_[u64]index[_s64](svbool_t pg, const int64_t *base, svuint64_t indices) + /// LDFF1D Zresult.D, Pg/Z, [Xbase, Zindices.D, LSL #3] + /// + public static unsafe Vector GatherVectorFirstFaulting(Vector mask, long* address, Vector indices) => GatherVectorFirstFaulting(mask, address, indices); + + /// + /// svfloat32_t svldff1_gather_[s32]index[_f32](svbool_t pg, const float32_t *base, svint32_t indices) + /// LDFF1W Zresult.S, Pg/Z, [Xbase, Zindices.S, SXTW #2] + /// + public static unsafe Vector GatherVectorFirstFaulting(Vector mask, float* address, Vector indices) => GatherVectorFirstFaulting(mask, address, indices); + + /// + /// svfloat32_t svldff1_gather[_u32base]_f32(svbool_t pg, svuint32_t bases) + /// LDFF1W Zresult.S, Pg/Z, [Zbases.S, #0] + /// + public static unsafe Vector GatherVectorFirstFaulting(Vector mask, Vector addresses) => GatherVectorFirstFaulting(mask, addresses); + + /// + /// svfloat32_t svldff1_gather_[u32]index[_f32](svbool_t pg, const float32_t *base, svuint32_t indices) + /// LDFF1W Zresult.S, Pg/Z, [Xbase, Zindices.S, UXTW #2] + /// + public static unsafe Vector GatherVectorFirstFaulting(Vector mask, float* address, Vector indices) => GatherVectorFirstFaulting(mask, address, indices); + + /// + /// svuint32_t svldff1_gather[_u32base]_u32(svbool_t pg, svuint32_t bases) + /// LDFF1W Zresult.S, Pg/Z, [Zbases.S, #0] + /// + public static unsafe Vector GatherVectorFirstFaulting(Vector mask, Vector addresses) => GatherVectorFirstFaulting(mask, addresses); + + /// + /// svuint32_t svldff1_gather_[s32]index[_u32](svbool_t pg, const uint32_t *base, svint32_t indices) + /// LDFF1W Zresult.S, Pg/Z, [Xbase, Zindices.S, SXTW #2] + /// + public static unsafe Vector GatherVectorFirstFaulting(Vector mask, uint* address, Vector indices) => GatherVectorFirstFaulting(mask, address, indices); + + /// + /// svuint32_t svldff1_gather_[u32]index[_u32](svbool_t pg, const uint32_t *base, svuint32_t indices) + /// LDFF1W Zresult.S, Pg/Z, [Xbase, Zindices.S, UXTW #2] + /// + public static unsafe Vector GatherVectorFirstFaulting(Vector mask, uint* address, Vector indices) => GatherVectorFirstFaulting(mask, address, indices); + + /// + /// svuint64_t svldff1_gather[_u64base]_u64(svbool_t pg, svuint64_t bases) + /// LDFF1D Zresult.D, Pg/Z, [Zbases.D, #0] + /// + public static unsafe Vector GatherVectorFirstFaulting(Vector mask, Vector addresses) => GatherVectorFirstFaulting(mask, addresses); + + /// + /// svuint64_t svldff1_gather_[s64]index[_u64](svbool_t pg, const uint64_t *base, svint64_t indices) + /// LDFF1D Zresult.D, Pg/Z, [Xbase, Zindices.D, LSL #3] + /// + public static unsafe Vector GatherVectorFirstFaulting(Vector mask, ulong* address, Vector indices) => GatherVectorFirstFaulting(mask, address, indices); + + /// + /// svuint64_t svldff1_gather_[u64]index[_u64](svbool_t pg, const uint64_t *base, svuint64_t indices) + /// LDFF1D Zresult.D, Pg/Z, [Xbase, Zindices.D, LSL #3] + /// + public static unsafe Vector GatherVectorFirstFaulting(Vector mask, ulong* address, Vector indices) => GatherVectorFirstFaulting(mask, address, indices); + + /// Load 16-bit data and sign-extend /// @@ -2661,6 +2771,13 @@ internal Arm64() { } public static unsafe ulong GetActiveElementCount(Vector mask, Vector from) => GetActiveElementCount(mask, from); + /// + /// svbool_t svrdffr() + /// RDFFR Presult.B + /// + public static unsafe Vector GetFfr() => GetFfr(); + + /// Insert scalar into shifted vector /// @@ -3145,6 +3262,67 @@ internal Arm64() { } public static unsafe Vector LoadVectorByteZeroExtendToUInt64(Vector mask, byte* address) => LoadVectorByteZeroExtendToUInt64(mask, address); + /// + /// svuint8_t svldff1[_u8](svbool_t pg, const uint8_t *base) + /// LDFF1B Zresult.B, Pg/Z, [Xbase, XZR] + /// + public static unsafe Vector LoadVectorFirstFaulting(Vector mask, byte* address) => LoadVectorFirstFaulting(mask, address); + + /// + /// svfloat64_t svldff1[_f64](svbool_t pg, const float64_t *base) + /// LDFF1D Zresult.D, Pg/Z, [Xbase, XZR, LSL #3] + /// + public static unsafe Vector LoadVectorFirstFaulting(Vector mask, double* address) => LoadVectorFirstFaulting(mask, address); + + /// + /// svint16_t svldff1[_s16](svbool_t pg, const int16_t *base) + /// LDFF1H Zresult.H, Pg/Z, [Xbase, XZR, LSL #1] + /// + public static unsafe Vector LoadVectorFirstFaulting(Vector mask, short* address) => LoadVectorFirstFaulting(mask, address); + + /// + /// svint32_t svldff1[_s32](svbool_t pg, const int32_t *base) + /// LDFF1W Zresult.S, Pg/Z, [Xbase, XZR, LSL #2] + /// + public static unsafe Vector LoadVectorFirstFaulting(Vector mask, int* address) => LoadVectorFirstFaulting(mask, address); + + /// + /// svint64_t svldff1[_s64](svbool_t pg, const int64_t *base) + /// LDFF1D Zresult.D, Pg/Z, [Xbase, XZR, LSL #3] + /// + public static unsafe Vector LoadVectorFirstFaulting(Vector mask, long* address) => LoadVectorFirstFaulting(mask, address); + + /// + /// svint8_t svldff1[_s8](svbool_t pg, const int8_t *base) + /// LDFF1B Zresult.B, Pg/Z, [Xbase, XZR] + /// + public static unsafe Vector LoadVectorFirstFaulting(Vector mask, sbyte* address) => LoadVectorFirstFaulting(mask, address); + + /// + /// svfloat32_t svldff1[_f32](svbool_t pg, const float32_t *base) + /// LDFF1W Zresult.S, Pg/Z, [Xbase, XZR, LSL #2] + /// + public static unsafe Vector LoadVectorFirstFaulting(Vector mask, float* address) => LoadVectorFirstFaulting(mask, address); + + /// + /// svuint16_t svldff1[_u16](svbool_t pg, const uint16_t *base) + /// LDFF1H Zresult.H, Pg/Z, [Xbase, XZR, LSL #1] + /// + public static unsafe Vector LoadVectorFirstFaulting(Vector mask, ushort* address) => LoadVectorFirstFaulting(mask, address); + + /// + /// svuint32_t svldff1[_u32](svbool_t pg, const uint32_t *base) + /// LDFF1W Zresult.S, Pg/Z, [Xbase, XZR, LSL #2] + /// + public static unsafe Vector LoadVectorFirstFaulting(Vector mask, uint* address) => LoadVectorFirstFaulting(mask, address); + + /// + /// svuint64_t svldff1[_u64](svbool_t pg, const uint64_t *base) + /// LDFF1D Zresult.D, Pg/Z, [Xbase, XZR, LSL #3] + /// + public static unsafe Vector LoadVectorFirstFaulting(Vector mask, ulong* address) => LoadVectorFirstFaulting(mask, address); + + /// LoadVectorInt16SignExtendToInt32 : Load 16-bit data and sign-extend /// @@ -5698,6 +5876,55 @@ internal Arm64() { } public static unsafe Vector Scale(Vector left, Vector right) => Scale(left, right); + /// + /// void svwrffr(svbool_t op) + /// WRFFR Pop.B + /// + public static unsafe void SetFfr(Vector value) => SetFfr(value); + + /// + /// void svwrffr(svbool_t op) + /// WRFFR Pop.B + /// + public static unsafe void SetFfr(Vector value) => SetFfr(value); + + /// + /// void svwrffr(svbool_t op) + /// WRFFR Pop.B + /// + public static unsafe void SetFfr(Vector value) => SetFfr(value); + + /// + /// void svwrffr(svbool_t op) + /// WRFFR Pop.B + /// + public static unsafe void SetFfr(Vector value) => SetFfr(value); + + /// + /// void svwrffr(svbool_t op) + /// WRFFR Pop.B + /// + public static unsafe void SetFfr(Vector value) => SetFfr(value); + + /// + /// void svwrffr(svbool_t op) + /// WRFFR Pop.B + /// + public static unsafe void SetFfr(Vector value) => SetFfr(value); + + /// + /// void svwrffr(svbool_t op) + /// WRFFR Pop.B + /// + public static unsafe void SetFfr(Vector value) => SetFfr(value); + + /// + /// void svwrffr(svbool_t op) + /// WRFFR Pop.B + /// + public static unsafe void SetFfr(Vector value) => SetFfr(value); + + /// Logical shift left /// diff --git a/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs b/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs index 1904b956195d0..9d369c6761bc3 100644 --- a/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs +++ b/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs @@ -4475,6 +4475,26 @@ internal Arm64() { } public static unsafe System.Numerics.Vector GatherVectorByteZeroExtend(System.Numerics.Vector mask, byte* address, System.Numerics.Vector indices) { throw null; } public static System.Numerics.Vector GatherVectorByteZeroExtend(System.Numerics.Vector mask, System.Numerics.Vector addresses) { throw null; } public static unsafe System.Numerics.Vector GatherVectorByteZeroExtend(System.Numerics.Vector mask, byte* address, System.Numerics.Vector indices) { throw null; } + + public static unsafe System.Numerics.Vector GatherVectorFirstFaulting(System.Numerics.Vector mask, double* address, System.Numerics.Vector indices) { throw null; } + public static unsafe System.Numerics.Vector GatherVectorFirstFaulting(System.Numerics.Vector mask, System.Numerics.Vector addresses) { throw null; } + public static unsafe System.Numerics.Vector GatherVectorFirstFaulting(System.Numerics.Vector mask, double* address, System.Numerics.Vector indices) { throw null; } + public static unsafe System.Numerics.Vector GatherVectorFirstFaulting(System.Numerics.Vector mask, System.Numerics.Vector addresses) { throw null; } + public static unsafe System.Numerics.Vector GatherVectorFirstFaulting(System.Numerics.Vector mask, int* address, System.Numerics.Vector indices) { throw null; } + public static unsafe System.Numerics.Vector GatherVectorFirstFaulting(System.Numerics.Vector mask, int* address, System.Numerics.Vector indices) { throw null; } + public static unsafe System.Numerics.Vector GatherVectorFirstFaulting(System.Numerics.Vector mask, System.Numerics.Vector addresses) { throw null; } + public static unsafe System.Numerics.Vector GatherVectorFirstFaulting(System.Numerics.Vector mask, long* address, System.Numerics.Vector indices) { throw null; } + public static unsafe System.Numerics.Vector GatherVectorFirstFaulting(System.Numerics.Vector mask, long* address, System.Numerics.Vector indices) { throw null; } + public static unsafe System.Numerics.Vector GatherVectorFirstFaulting(System.Numerics.Vector mask, float* address, System.Numerics.Vector indices) { throw null; } + public static unsafe System.Numerics.Vector GatherVectorFirstFaulting(System.Numerics.Vector mask, System.Numerics.Vector addresses) { throw null; } + public static unsafe System.Numerics.Vector GatherVectorFirstFaulting(System.Numerics.Vector mask, float* address, System.Numerics.Vector indices) { throw null; } + public static unsafe System.Numerics.Vector GatherVectorFirstFaulting(System.Numerics.Vector mask, System.Numerics.Vector addresses) { throw null; } + public static unsafe System.Numerics.Vector GatherVectorFirstFaulting(System.Numerics.Vector mask, uint* address, System.Numerics.Vector indices) { throw null; } + public static unsafe System.Numerics.Vector GatherVectorFirstFaulting(System.Numerics.Vector mask, uint* address, System.Numerics.Vector indices) { throw null; } + public static unsafe System.Numerics.Vector GatherVectorFirstFaulting(System.Numerics.Vector mask, System.Numerics.Vector addresses) { throw null; } + public static unsafe System.Numerics.Vector GatherVectorFirstFaulting(System.Numerics.Vector mask, ulong* address, System.Numerics.Vector indices) { throw null; } + public static unsafe System.Numerics.Vector GatherVectorFirstFaulting(System.Numerics.Vector mask, ulong* address, System.Numerics.Vector indices) { throw null; } + public static unsafe System.Numerics.Vector GatherVectorInt16SignExtend(System.Numerics.Vector mask, short* address, System.Numerics.Vector indices) { throw null; } // public static System.Numerics.Vector GatherVectorInt16SignExtend(System.Numerics.Vector mask, System.Numerics.Vector addresses) { throw null; } public static unsafe System.Numerics.Vector GatherVectorInt16SignExtend(System.Numerics.Vector mask, short* address, System.Numerics.Vector indices) { throw null; } @@ -4582,6 +4602,8 @@ internal Arm64() { } public static ulong GetActiveElementCount(System.Numerics.Vector mask, System.Numerics.Vector from) { throw null; } public static ulong GetActiveElementCount(System.Numerics.Vector mask, System.Numerics.Vector from) { throw null; } + public static unsafe System.Numerics.Vector GetFfr() { throw null; } + public static System.Numerics.Vector LeadingSignCount(System.Numerics.Vector value) { throw null; } public static System.Numerics.Vector LeadingSignCount(System.Numerics.Vector value) { throw null; } public static System.Numerics.Vector LeadingSignCount(System.Numerics.Vector value) { throw null; } @@ -4656,6 +4678,18 @@ internal Arm64() { } public static unsafe System.Numerics.Vector LoadVectorByteZeroExtendToUInt16(System.Numerics.Vector mask, byte* address) { throw null; } public static unsafe System.Numerics.Vector LoadVectorByteZeroExtendToUInt32(System.Numerics.Vector mask, byte* address) { throw null; } public static unsafe System.Numerics.Vector LoadVectorByteZeroExtendToUInt64(System.Numerics.Vector mask, byte* address) { throw null; } + + public static unsafe System.Numerics.Vector LoadVectorFirstFaulting(System.Numerics.Vector mask, byte* address) { throw null; } + public static unsafe System.Numerics.Vector LoadVectorFirstFaulting(System.Numerics.Vector mask, double* address) { throw null; } + public static unsafe System.Numerics.Vector LoadVectorFirstFaulting(System.Numerics.Vector mask, short* address) { throw null; } + public static unsafe System.Numerics.Vector LoadVectorFirstFaulting(System.Numerics.Vector mask, int* address) { throw null; } + public static unsafe System.Numerics.Vector LoadVectorFirstFaulting(System.Numerics.Vector mask, long* address) { throw null; } + public static unsafe System.Numerics.Vector LoadVectorFirstFaulting(System.Numerics.Vector mask, sbyte* address) { throw null; } + public static unsafe System.Numerics.Vector LoadVectorFirstFaulting(System.Numerics.Vector mask, float* address) { throw null; } + public static unsafe System.Numerics.Vector LoadVectorFirstFaulting(System.Numerics.Vector mask, ushort* address) { throw null; } + public static unsafe System.Numerics.Vector LoadVectorFirstFaulting(System.Numerics.Vector mask, uint* address) { throw null; } + public static unsafe System.Numerics.Vector LoadVectorFirstFaulting(System.Numerics.Vector mask, ulong* address) { throw null; } + public static unsafe System.Numerics.Vector LoadVectorInt16SignExtendToInt32(System.Numerics.Vector mask, short* address) { throw null; } public static unsafe System.Numerics.Vector LoadVectorInt16SignExtendToInt64(System.Numerics.Vector mask, short* address) { throw null; } public static unsafe System.Numerics.Vector LoadVectorInt16SignExtendToUInt32(System.Numerics.Vector mask, short* address) { throw null; } @@ -5015,6 +5049,15 @@ internal Arm64() { } public static System.Numerics.Vector Scale(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } public static System.Numerics.Vector Scale(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static unsafe void SetFfr(System.Numerics.Vector value) { throw null; } + public static unsafe void SetFfr(System.Numerics.Vector value) { throw null; } + public static unsafe void SetFfr(System.Numerics.Vector value) { throw null; } + public static unsafe void SetFfr(System.Numerics.Vector value) { throw null; } + public static unsafe void SetFfr(System.Numerics.Vector value) { throw null; } + public static unsafe void SetFfr(System.Numerics.Vector value) { throw null; } + public static unsafe void SetFfr(System.Numerics.Vector value) { throw null; } + public static unsafe void SetFfr(System.Numerics.Vector value) { throw null; } + public static System.Numerics.Vector ShiftLeftLogical(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } public static System.Numerics.Vector ShiftLeftLogical(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } public static System.Numerics.Vector ShiftLeftLogical(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } diff --git a/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs b/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs index 0719ba73f68c5..60adf06b637d7 100644 --- a/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs +++ b/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs @@ -181,6 +181,7 @@ ("_SveBinaryOpTestTemplate.template", "SveVecBinOpTest.template", new Dictionary { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = SimpleVecOpTest_ValidationLogic, ["TemplateValidationLogicForCndSel"] = SimpleVecOpTest_ValidationLogicForCndSel }), ("_SveBinaryOpTestTemplate.template", "SveVecBinOpVecTest.template", new Dictionary { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = SimpleVecOpTest_VectorValidationLogic, ["TemplateValidationLogicForCndSel"] = SimpleVecOpTest_VectorValidationLogicForCndSel }), ("_SveBinaryOpTestTemplate.template", "SveVecBinOpConvertTest.template", new Dictionary { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = SimpleVecOpTest_ValidationLogic, ["TemplateValidationLogicForCndSel"] = SimpleTernVecOpTest_ValidationLogicForCndSel }), + ("_SveLoadVectorFirstFaultingTest.template", "SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = SimpleVecOpTest_ValidationLogic, ["TemplateValidationLogicForCndSel"] = SimpleVecOpTest_ValidationLogicForCndSel }), ("_SveBinaryOpDifferentTypesTestTemplate.template", "SveVecBinOpDifferentTypesTest.template", new Dictionary { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = SimpleVecOpTest_ValidationLogic, ["TemplateValidationLogicForCndSel"] = SimpleVecOpTest_ValidationLogicForCndSel }), ("_SveBinaryMaskOpTestTemplate.template", "SveMaskVecBinOpConvertTest.template", new Dictionary { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = SimpleVecOpTest_ValidationLogic, ["TemplateValidationLogicForCndSel"] = SimpleVecOpTest_ValidationLogicForCndSel }), ("_SveImmBinaryOpTestTemplate.template", "SveVecImmBinOpTest.template", new Dictionary { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = SimpleVecOpTest_ValidationLogic, ["TemplateValidationLogicForCndSel"] = SimpleVecOpTest_ValidationLogicForCndSel }), @@ -3344,6 +3345,19 @@ ("SveGatherVectorIndices.template", new Dictionary {["TestName"] = "Sve_GatherVectorUInt32ZeroExtend_Indices_ulong_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorUInt32ZeroExtend", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2BaseType"] = "UInt32", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt64", ["ExtendedElementType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt64()"}), ("SveGatherVectorIndices.template", new Dictionary {["TestName"] = "Sve_GatherVectorUInt32ZeroExtend_Indices_uint_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorUInt32ZeroExtend", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2BaseType"] = "UInt32", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt32", ["ExtendedElementType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt32()"}), + ("SveGatherVectorIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_float_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2BaseType"] = "Single", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int32", ["ExtendedElementType"] = "Single", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskSingle()", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt32()"}), + ("SveGatherVectorIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_int_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2BaseType"] = "Int32", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int32", ["ExtendedElementType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt32()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt32()"}), + ("SveGatherVectorIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_uint_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2BaseType"] = "UInt32", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int32", ["ExtendedElementType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt32()"}), + ("SveGatherVectorIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_float_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2BaseType"] = "Single", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt32", ["ExtendedElementType"] = "Single", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskSingle()", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt32()"}), + ("SveGatherVectorIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_int_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2BaseType"] = "Int32", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt32", ["ExtendedElementType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt32()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt32()"}), + ("SveGatherVectorIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_uint_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2BaseType"] = "UInt32", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt32", ["ExtendedElementType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt32()"}), + ("SveGatherVectorIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_double_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2BaseType"] = "Double", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int64", ["ExtendedElementType"] = "Double", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskDouble()", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt64()"}), + ("SveGatherVectorIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_long_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2BaseType"] = "Int64", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int64", ["ExtendedElementType"] = "Int64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt64()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt64()"}), + ("SveGatherVectorIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_ulong_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2BaseType"] = "UInt64", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int64", ["ExtendedElementType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt64()"}), + ("SveGatherVectorIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_double_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2BaseType"] = "Double", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt64", ["ExtendedElementType"] = "Double", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskDouble()", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt64()"}), + ("SveGatherVectorIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_long_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2BaseType"] = "Int64", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt64", ["ExtendedElementType"] = "Int64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt64()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt64()"}), + ("SveGatherVectorIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_ulong_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2BaseType"] = "UInt64", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt64", ["ExtendedElementType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt64()"}), + ("SveGatherVectorByteOffsets.template",new Dictionary {["TestName"] = "Sve_GatherVectorWithByteOffsets_float_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorWithByteOffsets", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2BaseType"] = "Single", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int32", ["ExtendedElementType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskSingle()", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt32()"}), ("SveGatherVectorByteOffsets.template",new Dictionary {["TestName"] = "Sve_GatherVectorWithByteOffsets_int_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorWithByteOffsets", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2BaseType"] = "Int32", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int32", ["ExtendedElementType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt32()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt32()"}), ("SveGatherVectorByteOffsets.template",new Dictionary {["TestName"] = "Sve_GatherVectorWithByteOffsets_uint_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorWithByteOffsets", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2BaseType"] = "UInt32", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int32", ["ExtendedElementType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt32()"}), @@ -3403,6 +3417,17 @@ ("SveLoadNonFaultingUnOpTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorNonFaulting_uint", ["Isa"] = "Sve", ["Method"] = "LoadVectorNonFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2BaseType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt32()", ["Cast"] = "(uint*)", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), ("SveLoadNonFaultingUnOpTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorNonFaulting_ulong", ["Isa"] = "Sve", ["Method"] = "LoadVectorNonFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2BaseType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt64()", ["Cast"] = "(ulong*)", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_float", ["Isa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Single", ["LargestVectorSize"] = "64", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_double", ["Isa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Double", ["LargestVectorSize"] = "64", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_sbyte", ["Isa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "SByte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "SByte", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "SByte", ["LargestVectorSize"] = "64", ["NextValueOp2"] = "TestLibrary.Generator.GetSByte()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_short", ["Isa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int16", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int16", ["LargestVectorSize"] = "64", ["NextValueOp2"] = "TestLibrary.Generator.GetInt16()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_int", ["Isa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp2"] = "TestLibrary.Generator.GetInt32()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_long", ["Isa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int64", ["LargestVectorSize"] = "64", ["NextValueOp2"] = "TestLibrary.Generator.GetInt64()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_byte", ["Isa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Byte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Byte", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp2"] = "TestLibrary.Generator.GetByte()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_ushort", ["Isa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt16", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt16", ["LargestVectorSize"] = "64", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt16()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_uint", ["Isa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_ulong", ["Isa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), + ("SveLoadVectorMaskedTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorNonTemporal_float", ["Isa"] = "Sve", ["Method"] = "LoadVectorNonTemporal", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2BaseType"] = "Single", ["LargestVectorSize"] = "64", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), ("SveLoadVectorMaskedTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorNonTemporal_double", ["Isa"] = "Sve", ["Method"] = "LoadVectorNonTemporal", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2BaseType"] = "Double", ["LargestVectorSize"] = "64", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), ("SveLoadVectorMaskedTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorNonTemporal_sbyte", ["Isa"] = "Sve", ["Method"] = "LoadVectorNonTemporal", ["RetVectorType"] = "Vector", ["RetBaseType"] = "SByte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "SByte", ["Op2BaseType"] = "SByte", ["LargestVectorSize"] = "64", ["NextValueOp2"] = "TestLibrary.Generator.GetSByte()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveLoadVectorFirstFaultingTest.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveLoadVectorFirstFaultingTest.template new file mode 100644 index 0000000000000..0a10283036e83 --- /dev/null +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveLoadVectorFirstFaultingTest.template @@ -0,0 +1,436 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +/****************************************************************************** + * This file is auto-generated from a template file by the GenerateTests.csx * + * script in tests\src\JIT\HardwareIntrinsics.Arm\Shared. In order to make * + * changes, please update the corresponding template and run according to the * + * directions listed in the file. * + ******************************************************************************/ + +using System; +using System.Linq; +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.Arm; +using Xunit; + +namespace JIT.HardwareIntrinsics.Arm +{ + public static partial class Program + { + [Fact] + public static void {TestName}() + { + var test = new {TemplateName}BinaryOpTest__{TestName}(); + + if (test.IsSupported) + { + // Validates basic functionality works, using Unsafe.Read + test.RunBasicScenario_UnsafeRead(); + + if ({LoadIsa}.IsSupported) + { + // Validates basic functionality works, using Load + test.RunBasicScenario_Load(); + } + + // Validates calling via reflection works, using Unsafe.Read + test.RunReflectionScenario_UnsafeRead(); + + // Validates passing a local works, using Unsafe.Read + test.RunLclVarScenario_UnsafeRead(); + + // Validates passing an instance member of a class works + test.RunClassFldScenario(); + + // Validates passing the field of a local struct works + test.RunStructLclFldScenario(); + + // Validates passing an instance member of a struct works + test.RunStructFldScenario(); + + // Validates executing the test inside conditional, with op1 as falseValue + test.ConditionalSelect_Op1(); + + // Validates executing the test inside conditional, with op2 as falseValue + test.ConditionalSelect_Op2(); + + // Validates executing the test inside conditional, with op3 as falseValue + test.ConditionalSelect_FalseOp(); + + // Validates executing the test inside conditional, with op3 as zero + test.ConditionalSelect_ZeroOp(); + } + else + { + // Validates we throw on unsupported hardware + test.RunUnsupportedScenario(); + } + + if (!test.Succeeded) + { + throw new Exception("One or more scenarios did not complete as expected."); + } + } + } + + public sealed unsafe class {TemplateName}BinaryOpTest__{TestName} + { + private struct DataTable + { + private byte[] inArray1; + private byte[] inArray2; + private byte[] outArray; + + private GCHandle inHandle1; + private GCHandle inHandle2; + private GCHandle outHandle; + + private ulong alignment; + + public DataTable({Op1BaseType}[] inArray1, {Op2BaseType}[] inArray2, {RetBaseType}[] outArray, int alignment) + { + int sizeOfinArray1 = inArray1.Length * Unsafe.SizeOf<{Op1BaseType}>(); + int sizeOfinArray2 = inArray2.Length * Unsafe.SizeOf<{Op2BaseType}>(); + int sizeOfoutArray = outArray.Length * Unsafe.SizeOf<{RetBaseType}>(); + if ((alignment != 64 && alignment != 16 && alignment != 8) || (alignment * 2) < sizeOfinArray1 || (alignment * 2) < sizeOfinArray2 || (alignment * 2) < sizeOfoutArray) + { + throw new ArgumentException($"Invalid value of alignment: {alignment}, sizeOfinArray1: {sizeOfinArray1}, sizeOfinArray2: {sizeOfinArray2}, sizeOfoutArray: {sizeOfoutArray}"); + } + + this.inArray1 = new byte[alignment * 2]; + this.inArray2 = new byte[alignment * 2]; + this.outArray = new byte[alignment * 2]; + + this.inHandle1 = GCHandle.Alloc(this.inArray1, GCHandleType.Pinned); + this.inHandle2 = GCHandle.Alloc(this.inArray2, GCHandleType.Pinned); + this.outHandle = GCHandle.Alloc(this.outArray, GCHandleType.Pinned); + + this.alignment = (ulong)alignment; + + Unsafe.CopyBlockUnaligned(ref Unsafe.AsRef(inArray1Ptr), ref Unsafe.As<{Op1BaseType}, byte>(ref inArray1[0]), (uint)sizeOfinArray1); + Unsafe.CopyBlockUnaligned(ref Unsafe.AsRef(inArray2Ptr), ref Unsafe.As<{Op2BaseType}, byte>(ref inArray2[0]), (uint)sizeOfinArray2); + } + + public void* inArray1Ptr => Align((byte*)(inHandle1.AddrOfPinnedObject().ToPointer()), alignment); + public void* inArray2Ptr => Align((byte*)(inHandle2.AddrOfPinnedObject().ToPointer()), alignment); + public void* outArrayPtr => Align((byte*)(outHandle.AddrOfPinnedObject().ToPointer()), alignment); + + public void Dispose() + { + inHandle1.Free(); + inHandle2.Free(); + outHandle.Free(); + } + + private static unsafe void* Align(byte* buffer, ulong expectedAlignment) + { + return (void*)(((ulong)buffer + expectedAlignment - 1) & ~(expectedAlignment - 1)); + } + } + + private struct TestStruct + { + public {Op1VectorType}<{Op1BaseType}> _fld1; + public {Op2VectorType}<{Op2BaseType}> _fld2; + + public static TestStruct Create() + { + var testStruct = new TestStruct(); + + for (var i = 0; i < Op1ElementCount; i++) { _data1[i] = {NextValueOp1}; } + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1VectorType}<{Op1BaseType}>, byte>(ref testStruct._fld1), ref Unsafe.As<{Op1BaseType}, byte>(ref _data1[0]), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + for (var i = 0; i < Op2ElementCount; i++) { _data2[i] = {NextValueOp2}; } + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op2VectorType}<{Op2BaseType}>, byte>(ref testStruct._fld2), ref Unsafe.As<{Op2BaseType}, byte>(ref _data2[0]), (uint)Unsafe.SizeOf<{Op2VectorType}<{Op2BaseType}>>()); + + return testStruct; + } + + public void RunStructFldScenario({TemplateName}BinaryOpTest__{TestName} testClass) + { + var result = {Isa}.{Method}(_fld1, _fld2); + + Unsafe.Write(testClass._dataTable.outArrayPtr, result); + testClass.ValidateResult(_fld1, _fld2, testClass._dataTable.outArrayPtr); + } + } + + private static readonly int LargestVectorSize = {LargestVectorSize}; + + private static readonly int Op1ElementCount = Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>() / sizeof({Op1BaseType}); + private static readonly int Op2ElementCount = Unsafe.SizeOf<{Op2VectorType}<{Op2BaseType}>>() / sizeof({Op2BaseType}); + private static readonly int RetElementCount = Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>() / sizeof({RetBaseType}); + + private static {Op1BaseType}[] _maskData = new {Op1BaseType}[Op1ElementCount]; + private static {Op1BaseType}[] _data1 = new {Op1BaseType}[Op1ElementCount]; + private static {Op2BaseType}[] _data2 = new {Op2BaseType}[Op2ElementCount]; + + private {Op1VectorType}<{Op1BaseType}> _mask; + private {Op1VectorType}<{Op1BaseType}> _fld1; + private {Op2VectorType}<{Op2BaseType}> _fld2; + private {Op2VectorType}<{Op2BaseType}> _falseFld; + + private DataTable _dataTable; + + public {TemplateName}BinaryOpTest__{TestName}() + { + Succeeded = true; + + for (var i = 0; i < Op1ElementCount; i++) { _maskData[i] = ({Op1BaseType})({NextValueOp1} % 2); } + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1VectorType}<{Op1BaseType}>, byte>(ref _mask), ref Unsafe.As<{Op1BaseType}, byte>(ref _maskData[0]), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + for (var i = 0; i < Op1ElementCount; i++) { _data1[i] = {NextValueOp1}; } + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1VectorType}<{Op1BaseType}>, byte>(ref _fld1), ref Unsafe.As<{Op1BaseType}, byte>(ref _data1[0]), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + for (var i = 0; i < Op2ElementCount; i++) { _data2[i] = {NextValueOp2}; } + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op2VectorType}<{Op2BaseType}>, byte>(ref _fld2), ref Unsafe.As<{Op2BaseType}, byte>(ref _data2[0]), (uint)Unsafe.SizeOf<{Op2VectorType}<{Op2BaseType}>>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op2VectorType}<{Op2BaseType}>, byte>(ref _falseFld), ref Unsafe.As<{Op2BaseType}, byte>(ref _data2[0]), (uint)Unsafe.SizeOf<{Op2VectorType}<{Op2BaseType}>>()); + + for (var i = 0; i < Op1ElementCount; i++) { _data1[i] = {NextValueOp1}; } + for (var i = 0; i < Op2ElementCount; i++) { _data2[i] = {NextValueOp2}; } + _dataTable = new DataTable(_data1, _data2, new {RetBaseType}[RetElementCount], LargestVectorSize); + } + + public bool IsSupported => {Isa}.IsSupported; + + public bool Succeeded { get; set; } + + public void RunBasicScenario_UnsafeRead() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunBasicScenario_UnsafeRead)); + + var result = {Isa}.{Method}( + Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inArray1Ptr), + Unsafe.Read<{Op2VectorType}<{Op2BaseType}>>(_dataTable.inArray2Ptr) + ); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateResult(_dataTable.inArray1Ptr, _dataTable.inArray2Ptr, _dataTable.outArrayPtr); + } + + public void RunBasicScenario_Load() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunBasicScenario_Load)); + + {Op1VectorType}<{Op1BaseType}> loadMask = Sve.CreateTrueMask{RetBaseType}(SveMaskPattern.All); + + var result = {Isa}.{Method}( + {LoadIsa}.Load{Op1VectorType}(loadMask, ({Op1BaseType}*)(_dataTable.inArray1Ptr)), + {LoadIsa}.Load{Op2VectorType}(loadMask, ({Op2BaseType}*)(_dataTable.inArray2Ptr)) + ); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateResult(_dataTable.inArray1Ptr, _dataTable.inArray2Ptr, _dataTable.outArrayPtr); + } + + public void RunReflectionScenario_UnsafeRead() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunReflectionScenario_UnsafeRead)); + + var result = typeof({Isa}).GetMethod(nameof({Isa}.{Method}), new Type[] { typeof({Op1VectorType}<{Op1BaseType}>), typeof({Op2VectorType}<{Op2BaseType}>) }) + .Invoke(null, new object[] { + Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inArray1Ptr), + Unsafe.Read<{Op2VectorType}<{Op2BaseType}>>(_dataTable.inArray2Ptr) + }); + + Unsafe.Write(_dataTable.outArrayPtr, ({RetVectorType}<{RetBaseType}>)(result)); + ValidateResult(_dataTable.inArray1Ptr, _dataTable.inArray2Ptr, _dataTable.outArrayPtr); + } + + public void RunLclVarScenario_UnsafeRead() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunLclVarScenario_UnsafeRead)); + + var op1 = Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inArray1Ptr); + var op2 = Unsafe.Read<{Op2VectorType}<{Op2BaseType}>>(_dataTable.inArray2Ptr); + var result = {Isa}.{Method}(op1, op2); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateResult(op1, op2, _dataTable.outArrayPtr); + } + + public void RunClassFldScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunClassFldScenario)); + + var result = {Isa}.{Method}(_fld1, _fld2); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateResult(_fld1, _fld2, _dataTable.outArrayPtr); + } + + public void RunStructLclFldScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunStructLclFldScenario)); + + var test = TestStruct.Create(); + var result = {Isa}.{Method}(test._fld1, test._fld2); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateResult(test._fld1, test._fld2, _dataTable.outArrayPtr); + } + + public void RunStructFldScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunStructFldScenario)); + + var test = TestStruct.Create(); + test.RunStructFldScenario(this); + } + + public void ConditionalSelect_Op1() + { + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op1_mask"); + ConditionalSelectScenario(_mask, _fld1, _fld2, _fld1); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op1_zero"); + ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}>.Zero, _fld1, _fld2, _fld1); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op1_all"); + ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}>.AllBitsSet, _fld1, _fld2, _fld1); + } + + public void ConditionalSelect_Op2() + { + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op2_mask"); + ConditionalSelectScenario(_mask, _fld1, _fld2, _fld2); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op2_zero"); + ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}>.Zero, _fld1, _fld2, _fld2); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op2_all"); + ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}>.AllBitsSet, _fld1, _fld2, _fld2); + } + + public void ConditionalSelect_FalseOp() + { + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_mask"); + ConditionalSelectScenario(_mask, _fld1, _fld2, _falseFld); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_zero"); + ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}>.Zero, _fld1, _fld2, _falseFld); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_all"); + ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}>.AllBitsSet, _fld1, _fld2, _falseFld); + } + + public void ConditionalSelect_ZeroOp() + { + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_mask"); + ConditionalSelectScenario(_mask, _fld1, _fld2, {Op1VectorType}<{Op1BaseType}>.Zero); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_zero"); + ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}>.Zero, _fld1, _fld2, {Op1VectorType}<{Op1BaseType}>.Zero); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_all"); + ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}>.AllBitsSet, _fld1, _fld2, {Op1VectorType}<{Op1BaseType}>.Zero); + } + + [method: MethodImpl(MethodImplOptions.AggressiveInlining)] + private void ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}> mask, {Op1VectorType}<{Op1BaseType}> op1, {Op1VectorType}<{Op1BaseType}> op2, {Op1VectorType}<{Op1BaseType}> falseOp) + { + var result = Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2), falseOp); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateConditionalSelectResult(mask, op1, op2, falseOp, _dataTable.outArrayPtr); + } + + public void RunUnsupportedScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunUnsupportedScenario)); + + bool succeeded = false; + + try + { + RunBasicScenario_UnsafeRead(); + } + catch (PlatformNotSupportedException) + { + succeeded = true; + } + + if (!succeeded) + { + Succeeded = false; + } + } + + private void ValidateConditionalSelectResult({Op1VectorType}<{Op1BaseType}> maskOp, {Op1VectorType}<{Op1BaseType}> leftOp, {Op1VectorType}<{Op1BaseType}> rightOp, {Op1VectorType}<{Op1BaseType}> falseOp, void* output, [CallerMemberName] string method = "") + { + {Op1BaseType}[] mask = new {Op1BaseType}[Op1ElementCount]; + {Op1BaseType}[] left = new {Op1BaseType}[Op1ElementCount]; + {Op1BaseType}[] right = new {Op1BaseType}[Op1ElementCount]; + {Op1BaseType}[] falseVal = new {Op1BaseType}[Op1ElementCount]; + {RetBaseType}[] result = new {RetBaseType}[RetElementCount]; + + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref mask[0]), maskOp); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref left[0]), leftOp); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref right[0]), rightOp); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref falseVal[0]), falseOp); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref result[0]), ref Unsafe.AsRef(output), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); + + bool succeeded = true; + + {TemplateValidationLogicForCndSel} + + if (!succeeded) + { + TestLibrary.TestFramework.LogInformation($"{nameof({Isa})}.{nameof({Isa}.{Method})}<{RetBaseType}>({Op1VectorType}<{Op1BaseType}>, {Op2VectorType}<{Op2BaseType}>): {method} failed:"); + TestLibrary.TestFramework.LogInformation($" mask: ({string.Join(", ", mask)})"); + TestLibrary.TestFramework.LogInformation($" left: ({string.Join(", ", left)})"); + TestLibrary.TestFramework.LogInformation($" right: ({string.Join(", ", right)})"); + TestLibrary.TestFramework.LogInformation($" falseOp: ({string.Join(", ", falseVal)})"); + TestLibrary.TestFramework.LogInformation($" result: ({string.Join(", ", result)})"); + TestLibrary.TestFramework.LogInformation(string.Empty); + + Succeeded = false; + } + } + + private void ValidateResult({Op1VectorType}<{Op1BaseType}> op1, {Op2VectorType}<{Op2BaseType}> op2, void* result, [CallerMemberName] string method = "") + { + {Op1BaseType}[] inArray1 = new {Op1BaseType}[Op1ElementCount]; + {Op2BaseType}[] inArray2 = new {Op2BaseType}[Op2ElementCount]; + {RetBaseType}[] outArray = new {RetBaseType}[RetElementCount]; + + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref inArray1[0]), op1); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op2BaseType}, byte>(ref inArray2[0]), op2); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref outArray[0]), ref Unsafe.AsRef(result), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); + + ValidateResult(inArray1, inArray2, outArray, method); + } + + private void ValidateResult(void* op1, void* op2, void* result, [CallerMemberName] string method = "") + { + {Op1BaseType}[] inArray1 = new {Op1BaseType}[Op1ElementCount]; + {Op2BaseType}[] inArray2 = new {Op2BaseType}[Op2ElementCount]; + {RetBaseType}[] outArray = new {RetBaseType}[RetElementCount]; + + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref inArray1[0]), ref Unsafe.AsRef(op1), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op2BaseType}, byte>(ref inArray2[0]), ref Unsafe.AsRef(op2), (uint)Unsafe.SizeOf<{Op2VectorType}<{Op2BaseType}>>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref outArray[0]), ref Unsafe.AsRef(result), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); + + ValidateResult(inArray1, inArray2, outArray, method); + } + + private void ValidateResult({Op1BaseType}[] left, {Op2BaseType}[] right, {RetBaseType}[] result, [CallerMemberName] string method = "") + { + bool succeeded = true; + + {TemplateValidationLogic} + + if (!succeeded) + { + TestLibrary.TestFramework.LogInformation($"{nameof({Isa})}.{nameof({Isa}.{Method})}<{RetBaseType}>({Op1VectorType}<{Op1BaseType}>, {Op2VectorType}<{Op2BaseType}>): {method} failed:"); + TestLibrary.TestFramework.LogInformation($" left: ({string.Join(", ", left)})"); + TestLibrary.TestFramework.LogInformation($" right: ({string.Join(", ", right)})"); + TestLibrary.TestFramework.LogInformation($" result: ({string.Join(", ", result)})"); + TestLibrary.TestFramework.LogInformation(string.Empty); + + Succeeded = false; + } + } + } +} From a7773acce24352fa4623a9c9bfa825c60e31271a Mon Sep 17 00:00:00 2001 From: TIHan Date: Tue, 2 Jul 2024 14:42:08 -0700 Subject: [PATCH 02/83] FirstFaulting partially works --- src/coreclr/jit/hwintrinsiccodegenarm64.cpp | 11 + src/coreclr/jit/hwintrinsiclistarm64sve.h | 4 +- .../GenerateHWIntrinsicTests_Arm.cs | 21 +- .../_SveLoadVectorFirstFaultingTest.template | 436 ------------------ 4 files changed, 23 insertions(+), 449 deletions(-) delete mode 100644 src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveLoadVectorFirstFaultingTest.template diff --git a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp index d44f9253014f0..3be591c1811ee 100644 --- a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp @@ -2313,6 +2313,17 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) break; } + case NI_Sve_LoadVectorFirstFaulting: + { + insScalableOpts sopt = INS_SCALABLE_OPTS_LSL_N; + if (opt == INS_OPTS_SCALABLE_B) + { + sopt = INS_SCALABLE_OPTS_NONE; + } + GetEmitter()->emitIns_R_R_R_R(ins, emitSize, targetReg, op1Reg, op2Reg, REG_ZR, opt, sopt); + break; + } + default: unreached(); } diff --git a/src/coreclr/jit/hwintrinsiclistarm64sve.h b/src/coreclr/jit/hwintrinsiclistarm64sve.h index 7ac9467358663..c003ff22ee99b 100644 --- a/src/coreclr/jit/hwintrinsiclistarm64sve.h +++ b/src/coreclr/jit/hwintrinsiclistarm64sve.h @@ -102,7 +102,7 @@ HARDWARE_INTRINSIC(Sve, GatherVectorUInt32WithByteOffsetsZeroExtend, HARDWARE_INTRINSIC(Sve, GatherVectorUInt32ZeroExtend, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_sve_ld1w, INS_sve_ld1w, INS_sve_ld1w, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, GatherVectorWithByteOffsets, -1, 3, true, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_sve_ld1w, INS_sve_ld1d, INS_sve_ld1d, INS_sve_ld1w, INS_sve_ld1d}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, GetActiveElementCount, -1, 2, true, {INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ExplicitMaskedOperation) -HARDWARE_INTRINSIC(Sve, GetFfr, -1, 0, true, {INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, GetFfr, -1, 0, false, {INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) HARDWARE_INTRINSIC(Sve, InsertIntoShiftedVector, -1, 2, true, {INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics) HARDWARE_INTRINSIC(Sve, LeadingSignCount, -1, -1, false, {INS_sve_cls, INS_invalid, INS_sve_cls, INS_invalid, INS_sve_cls, INS_invalid, INS_sve_cls, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LeadingZeroCount, -1, -1, false, {INS_sve_clz, INS_sve_clz, INS_sve_clz, INS_sve_clz, INS_sve_clz, INS_sve_clz, INS_sve_clz, INS_sve_clz, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) @@ -123,7 +123,7 @@ HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendToInt64, HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendToUInt16, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendToUInt32, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendToUInt64, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1b, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorFirstFaulting, -1, 2, true, {INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1h, INS_sve_ldff1h, INS_sve_ldff1w, INS_sve_ldff1w, INS_sve_ldff1d, INS_sve_ldff1d, INS_sve_ldff1w, INS_sve_ldff1d}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorFirstFaulting, -1, 2, true, {INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1h, INS_sve_ldff1h, INS_sve_ldff1w, INS_sve_ldff1w, INS_sve_ldff1d, INS_sve_ldff1d, INS_sve_ldff1w, INS_sve_ldff1d}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialCodeGen) HARDWARE_INTRINSIC(Sve, LoadVectorInt16NonFaultingSignExtendToInt32, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorInt16NonFaultingSignExtendToInt64, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sh, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorInt16NonFaultingSignExtendToUInt32, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sh, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) diff --git a/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs b/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs index a01ea0acee169..d6c0179d2afab 100644 --- a/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs +++ b/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs @@ -181,7 +181,6 @@ ("_SveBinaryOpTestTemplate.template", "SveVecBinOpTest.template", new Dictionary { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = SimpleVecOpTest_ValidationLogic, ["TemplateValidationLogicForCndSel"] = SimpleVecOpTest_ValidationLogicForCndSel }), ("_SveBinaryOpTestTemplate.template", "SveVecBinOpVecTest.template", new Dictionary { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = SimpleVecOpTest_VectorValidationLogic, ["TemplateValidationLogicForCndSel"] = SimpleVecOpTest_VectorValidationLogicForCndSel }), ("_SveBinaryOpTestTemplate.template", "SveVecBinOpConvertTest.template", new Dictionary { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = SimpleVecOpTest_ValidationLogic, ["TemplateValidationLogicForCndSel"] = SimpleTernVecOpTest_ValidationLogicForCndSel }), - ("_SveLoadVectorFirstFaultingTest.template", "SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = SimpleVecOpTest_ValidationLogic, ["TemplateValidationLogicForCndSel"] = SimpleVecOpTest_ValidationLogicForCndSel }), ("_SveBinaryOpDifferentTypesTestTemplate.template", "SveVecBinOpDifferentTypesTest.template", new Dictionary { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = SimpleVecOpTest_ValidationLogic, ["TemplateValidationLogicForCndSel"] = SimpleVecOpTest_ValidationLogicForCndSel }), ("_SveBinaryMaskOpTestTemplate.template", "SveMaskVecBinOpConvertTest.template", new Dictionary { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = SimpleVecOpTest_ValidationLogic, ["TemplateValidationLogicForCndSel"] = SimpleVecOpTest_ValidationLogicForCndSel }), ("_SveImmBinaryOpTestTemplate.template", "SveVecImmBinOpTest.template", new Dictionary { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = SimpleVecOpTest_ValidationLogic, ["TemplateValidationLogicForCndSel"] = SimpleVecOpTest_ValidationLogicForCndSel }), @@ -3503,16 +3502,16 @@ ("SveLoadNonFaultingUnOpTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorNonFaulting_uint", ["Isa"] = "Sve", ["Method"] = "LoadVectorNonFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2BaseType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt32()", ["Cast"] = "(uint*)", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), ("SveLoadNonFaultingUnOpTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorNonFaulting_ulong", ["Isa"] = "Sve", ["Method"] = "LoadVectorNonFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2BaseType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt64()", ["Cast"] = "(ulong*)", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_float", ["Isa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Single", ["LargestVectorSize"] = "64", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_double", ["Isa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Double", ["LargestVectorSize"] = "64", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_sbyte", ["Isa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "SByte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "SByte", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "SByte", ["LargestVectorSize"] = "64", ["NextValueOp2"] = "TestLibrary.Generator.GetSByte()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_short", ["Isa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int16", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int16", ["LargestVectorSize"] = "64", ["NextValueOp2"] = "TestLibrary.Generator.GetInt16()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_int", ["Isa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp2"] = "TestLibrary.Generator.GetInt32()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_long", ["Isa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int64", ["LargestVectorSize"] = "64", ["NextValueOp2"] = "TestLibrary.Generator.GetInt64()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_byte", ["Isa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Byte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Byte", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp2"] = "TestLibrary.Generator.GetByte()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_ushort", ["Isa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt16", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt16", ["LargestVectorSize"] = "64", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt16()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_uint", ["Isa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_ulong", ["Isa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_float", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Single", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSingle()", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_double", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Double", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetDouble()", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_sbyte", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "SByte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "SByte", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "SByte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSByte()", ["NextValueOp2"] = "TestLibrary.Generator.GetSByte()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_short", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int16", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt16()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt16()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt32()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt64()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_byte", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Byte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Byte", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetByte()", ["NextValueOp2"] = "TestLibrary.Generator.GetByte()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_ushort", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt16", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt16()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt16()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), ("SveLoadVectorMaskedTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorNonTemporal_float", ["Isa"] = "Sve", ["Method"] = "LoadVectorNonTemporal", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2BaseType"] = "Single", ["LargestVectorSize"] = "64", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), ("SveLoadVectorMaskedTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorNonTemporal_double", ["Isa"] = "Sve", ["Method"] = "LoadVectorNonTemporal", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2BaseType"] = "Double", ["LargestVectorSize"] = "64", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveLoadVectorFirstFaultingTest.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveLoadVectorFirstFaultingTest.template deleted file mode 100644 index 0a10283036e83..0000000000000 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveLoadVectorFirstFaultingTest.template +++ /dev/null @@ -1,436 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -/****************************************************************************** - * This file is auto-generated from a template file by the GenerateTests.csx * - * script in tests\src\JIT\HardwareIntrinsics.Arm\Shared. In order to make * - * changes, please update the corresponding template and run according to the * - * directions listed in the file. * - ******************************************************************************/ - -using System; -using System.Linq; -using System.Numerics; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; -using System.Runtime.Intrinsics; -using System.Runtime.Intrinsics.Arm; -using Xunit; - -namespace JIT.HardwareIntrinsics.Arm -{ - public static partial class Program - { - [Fact] - public static void {TestName}() - { - var test = new {TemplateName}BinaryOpTest__{TestName}(); - - if (test.IsSupported) - { - // Validates basic functionality works, using Unsafe.Read - test.RunBasicScenario_UnsafeRead(); - - if ({LoadIsa}.IsSupported) - { - // Validates basic functionality works, using Load - test.RunBasicScenario_Load(); - } - - // Validates calling via reflection works, using Unsafe.Read - test.RunReflectionScenario_UnsafeRead(); - - // Validates passing a local works, using Unsafe.Read - test.RunLclVarScenario_UnsafeRead(); - - // Validates passing an instance member of a class works - test.RunClassFldScenario(); - - // Validates passing the field of a local struct works - test.RunStructLclFldScenario(); - - // Validates passing an instance member of a struct works - test.RunStructFldScenario(); - - // Validates executing the test inside conditional, with op1 as falseValue - test.ConditionalSelect_Op1(); - - // Validates executing the test inside conditional, with op2 as falseValue - test.ConditionalSelect_Op2(); - - // Validates executing the test inside conditional, with op3 as falseValue - test.ConditionalSelect_FalseOp(); - - // Validates executing the test inside conditional, with op3 as zero - test.ConditionalSelect_ZeroOp(); - } - else - { - // Validates we throw on unsupported hardware - test.RunUnsupportedScenario(); - } - - if (!test.Succeeded) - { - throw new Exception("One or more scenarios did not complete as expected."); - } - } - } - - public sealed unsafe class {TemplateName}BinaryOpTest__{TestName} - { - private struct DataTable - { - private byte[] inArray1; - private byte[] inArray2; - private byte[] outArray; - - private GCHandle inHandle1; - private GCHandle inHandle2; - private GCHandle outHandle; - - private ulong alignment; - - public DataTable({Op1BaseType}[] inArray1, {Op2BaseType}[] inArray2, {RetBaseType}[] outArray, int alignment) - { - int sizeOfinArray1 = inArray1.Length * Unsafe.SizeOf<{Op1BaseType}>(); - int sizeOfinArray2 = inArray2.Length * Unsafe.SizeOf<{Op2BaseType}>(); - int sizeOfoutArray = outArray.Length * Unsafe.SizeOf<{RetBaseType}>(); - if ((alignment != 64 && alignment != 16 && alignment != 8) || (alignment * 2) < sizeOfinArray1 || (alignment * 2) < sizeOfinArray2 || (alignment * 2) < sizeOfoutArray) - { - throw new ArgumentException($"Invalid value of alignment: {alignment}, sizeOfinArray1: {sizeOfinArray1}, sizeOfinArray2: {sizeOfinArray2}, sizeOfoutArray: {sizeOfoutArray}"); - } - - this.inArray1 = new byte[alignment * 2]; - this.inArray2 = new byte[alignment * 2]; - this.outArray = new byte[alignment * 2]; - - this.inHandle1 = GCHandle.Alloc(this.inArray1, GCHandleType.Pinned); - this.inHandle2 = GCHandle.Alloc(this.inArray2, GCHandleType.Pinned); - this.outHandle = GCHandle.Alloc(this.outArray, GCHandleType.Pinned); - - this.alignment = (ulong)alignment; - - Unsafe.CopyBlockUnaligned(ref Unsafe.AsRef(inArray1Ptr), ref Unsafe.As<{Op1BaseType}, byte>(ref inArray1[0]), (uint)sizeOfinArray1); - Unsafe.CopyBlockUnaligned(ref Unsafe.AsRef(inArray2Ptr), ref Unsafe.As<{Op2BaseType}, byte>(ref inArray2[0]), (uint)sizeOfinArray2); - } - - public void* inArray1Ptr => Align((byte*)(inHandle1.AddrOfPinnedObject().ToPointer()), alignment); - public void* inArray2Ptr => Align((byte*)(inHandle2.AddrOfPinnedObject().ToPointer()), alignment); - public void* outArrayPtr => Align((byte*)(outHandle.AddrOfPinnedObject().ToPointer()), alignment); - - public void Dispose() - { - inHandle1.Free(); - inHandle2.Free(); - outHandle.Free(); - } - - private static unsafe void* Align(byte* buffer, ulong expectedAlignment) - { - return (void*)(((ulong)buffer + expectedAlignment - 1) & ~(expectedAlignment - 1)); - } - } - - private struct TestStruct - { - public {Op1VectorType}<{Op1BaseType}> _fld1; - public {Op2VectorType}<{Op2BaseType}> _fld2; - - public static TestStruct Create() - { - var testStruct = new TestStruct(); - - for (var i = 0; i < Op1ElementCount; i++) { _data1[i] = {NextValueOp1}; } - Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1VectorType}<{Op1BaseType}>, byte>(ref testStruct._fld1), ref Unsafe.As<{Op1BaseType}, byte>(ref _data1[0]), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); - for (var i = 0; i < Op2ElementCount; i++) { _data2[i] = {NextValueOp2}; } - Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op2VectorType}<{Op2BaseType}>, byte>(ref testStruct._fld2), ref Unsafe.As<{Op2BaseType}, byte>(ref _data2[0]), (uint)Unsafe.SizeOf<{Op2VectorType}<{Op2BaseType}>>()); - - return testStruct; - } - - public void RunStructFldScenario({TemplateName}BinaryOpTest__{TestName} testClass) - { - var result = {Isa}.{Method}(_fld1, _fld2); - - Unsafe.Write(testClass._dataTable.outArrayPtr, result); - testClass.ValidateResult(_fld1, _fld2, testClass._dataTable.outArrayPtr); - } - } - - private static readonly int LargestVectorSize = {LargestVectorSize}; - - private static readonly int Op1ElementCount = Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>() / sizeof({Op1BaseType}); - private static readonly int Op2ElementCount = Unsafe.SizeOf<{Op2VectorType}<{Op2BaseType}>>() / sizeof({Op2BaseType}); - private static readonly int RetElementCount = Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>() / sizeof({RetBaseType}); - - private static {Op1BaseType}[] _maskData = new {Op1BaseType}[Op1ElementCount]; - private static {Op1BaseType}[] _data1 = new {Op1BaseType}[Op1ElementCount]; - private static {Op2BaseType}[] _data2 = new {Op2BaseType}[Op2ElementCount]; - - private {Op1VectorType}<{Op1BaseType}> _mask; - private {Op1VectorType}<{Op1BaseType}> _fld1; - private {Op2VectorType}<{Op2BaseType}> _fld2; - private {Op2VectorType}<{Op2BaseType}> _falseFld; - - private DataTable _dataTable; - - public {TemplateName}BinaryOpTest__{TestName}() - { - Succeeded = true; - - for (var i = 0; i < Op1ElementCount; i++) { _maskData[i] = ({Op1BaseType})({NextValueOp1} % 2); } - Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1VectorType}<{Op1BaseType}>, byte>(ref _mask), ref Unsafe.As<{Op1BaseType}, byte>(ref _maskData[0]), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); - for (var i = 0; i < Op1ElementCount; i++) { _data1[i] = {NextValueOp1}; } - Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1VectorType}<{Op1BaseType}>, byte>(ref _fld1), ref Unsafe.As<{Op1BaseType}, byte>(ref _data1[0]), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); - for (var i = 0; i < Op2ElementCount; i++) { _data2[i] = {NextValueOp2}; } - Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op2VectorType}<{Op2BaseType}>, byte>(ref _fld2), ref Unsafe.As<{Op2BaseType}, byte>(ref _data2[0]), (uint)Unsafe.SizeOf<{Op2VectorType}<{Op2BaseType}>>()); - Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op2VectorType}<{Op2BaseType}>, byte>(ref _falseFld), ref Unsafe.As<{Op2BaseType}, byte>(ref _data2[0]), (uint)Unsafe.SizeOf<{Op2VectorType}<{Op2BaseType}>>()); - - for (var i = 0; i < Op1ElementCount; i++) { _data1[i] = {NextValueOp1}; } - for (var i = 0; i < Op2ElementCount; i++) { _data2[i] = {NextValueOp2}; } - _dataTable = new DataTable(_data1, _data2, new {RetBaseType}[RetElementCount], LargestVectorSize); - } - - public bool IsSupported => {Isa}.IsSupported; - - public bool Succeeded { get; set; } - - public void RunBasicScenario_UnsafeRead() - { - TestLibrary.TestFramework.BeginScenario(nameof(RunBasicScenario_UnsafeRead)); - - var result = {Isa}.{Method}( - Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inArray1Ptr), - Unsafe.Read<{Op2VectorType}<{Op2BaseType}>>(_dataTable.inArray2Ptr) - ); - - Unsafe.Write(_dataTable.outArrayPtr, result); - ValidateResult(_dataTable.inArray1Ptr, _dataTable.inArray2Ptr, _dataTable.outArrayPtr); - } - - public void RunBasicScenario_Load() - { - TestLibrary.TestFramework.BeginScenario(nameof(RunBasicScenario_Load)); - - {Op1VectorType}<{Op1BaseType}> loadMask = Sve.CreateTrueMask{RetBaseType}(SveMaskPattern.All); - - var result = {Isa}.{Method}( - {LoadIsa}.Load{Op1VectorType}(loadMask, ({Op1BaseType}*)(_dataTable.inArray1Ptr)), - {LoadIsa}.Load{Op2VectorType}(loadMask, ({Op2BaseType}*)(_dataTable.inArray2Ptr)) - ); - - Unsafe.Write(_dataTable.outArrayPtr, result); - ValidateResult(_dataTable.inArray1Ptr, _dataTable.inArray2Ptr, _dataTable.outArrayPtr); - } - - public void RunReflectionScenario_UnsafeRead() - { - TestLibrary.TestFramework.BeginScenario(nameof(RunReflectionScenario_UnsafeRead)); - - var result = typeof({Isa}).GetMethod(nameof({Isa}.{Method}), new Type[] { typeof({Op1VectorType}<{Op1BaseType}>), typeof({Op2VectorType}<{Op2BaseType}>) }) - .Invoke(null, new object[] { - Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inArray1Ptr), - Unsafe.Read<{Op2VectorType}<{Op2BaseType}>>(_dataTable.inArray2Ptr) - }); - - Unsafe.Write(_dataTable.outArrayPtr, ({RetVectorType}<{RetBaseType}>)(result)); - ValidateResult(_dataTable.inArray1Ptr, _dataTable.inArray2Ptr, _dataTable.outArrayPtr); - } - - public void RunLclVarScenario_UnsafeRead() - { - TestLibrary.TestFramework.BeginScenario(nameof(RunLclVarScenario_UnsafeRead)); - - var op1 = Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inArray1Ptr); - var op2 = Unsafe.Read<{Op2VectorType}<{Op2BaseType}>>(_dataTable.inArray2Ptr); - var result = {Isa}.{Method}(op1, op2); - - Unsafe.Write(_dataTable.outArrayPtr, result); - ValidateResult(op1, op2, _dataTable.outArrayPtr); - } - - public void RunClassFldScenario() - { - TestLibrary.TestFramework.BeginScenario(nameof(RunClassFldScenario)); - - var result = {Isa}.{Method}(_fld1, _fld2); - - Unsafe.Write(_dataTable.outArrayPtr, result); - ValidateResult(_fld1, _fld2, _dataTable.outArrayPtr); - } - - public void RunStructLclFldScenario() - { - TestLibrary.TestFramework.BeginScenario(nameof(RunStructLclFldScenario)); - - var test = TestStruct.Create(); - var result = {Isa}.{Method}(test._fld1, test._fld2); - - Unsafe.Write(_dataTable.outArrayPtr, result); - ValidateResult(test._fld1, test._fld2, _dataTable.outArrayPtr); - } - - public void RunStructFldScenario() - { - TestLibrary.TestFramework.BeginScenario(nameof(RunStructFldScenario)); - - var test = TestStruct.Create(); - test.RunStructFldScenario(this); - } - - public void ConditionalSelect_Op1() - { - TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op1_mask"); - ConditionalSelectScenario(_mask, _fld1, _fld2, _fld1); - - TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op1_zero"); - ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}>.Zero, _fld1, _fld2, _fld1); - - TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op1_all"); - ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}>.AllBitsSet, _fld1, _fld2, _fld1); - } - - public void ConditionalSelect_Op2() - { - TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op2_mask"); - ConditionalSelectScenario(_mask, _fld1, _fld2, _fld2); - - TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op2_zero"); - ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}>.Zero, _fld1, _fld2, _fld2); - - TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op2_all"); - ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}>.AllBitsSet, _fld1, _fld2, _fld2); - } - - public void ConditionalSelect_FalseOp() - { - TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_mask"); - ConditionalSelectScenario(_mask, _fld1, _fld2, _falseFld); - - TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_zero"); - ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}>.Zero, _fld1, _fld2, _falseFld); - - TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_all"); - ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}>.AllBitsSet, _fld1, _fld2, _falseFld); - } - - public void ConditionalSelect_ZeroOp() - { - TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_mask"); - ConditionalSelectScenario(_mask, _fld1, _fld2, {Op1VectorType}<{Op1BaseType}>.Zero); - - TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_zero"); - ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}>.Zero, _fld1, _fld2, {Op1VectorType}<{Op1BaseType}>.Zero); - - TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_all"); - ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}>.AllBitsSet, _fld1, _fld2, {Op1VectorType}<{Op1BaseType}>.Zero); - } - - [method: MethodImpl(MethodImplOptions.AggressiveInlining)] - private void ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}> mask, {Op1VectorType}<{Op1BaseType}> op1, {Op1VectorType}<{Op1BaseType}> op2, {Op1VectorType}<{Op1BaseType}> falseOp) - { - var result = Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2), falseOp); - - Unsafe.Write(_dataTable.outArrayPtr, result); - ValidateConditionalSelectResult(mask, op1, op2, falseOp, _dataTable.outArrayPtr); - } - - public void RunUnsupportedScenario() - { - TestLibrary.TestFramework.BeginScenario(nameof(RunUnsupportedScenario)); - - bool succeeded = false; - - try - { - RunBasicScenario_UnsafeRead(); - } - catch (PlatformNotSupportedException) - { - succeeded = true; - } - - if (!succeeded) - { - Succeeded = false; - } - } - - private void ValidateConditionalSelectResult({Op1VectorType}<{Op1BaseType}> maskOp, {Op1VectorType}<{Op1BaseType}> leftOp, {Op1VectorType}<{Op1BaseType}> rightOp, {Op1VectorType}<{Op1BaseType}> falseOp, void* output, [CallerMemberName] string method = "") - { - {Op1BaseType}[] mask = new {Op1BaseType}[Op1ElementCount]; - {Op1BaseType}[] left = new {Op1BaseType}[Op1ElementCount]; - {Op1BaseType}[] right = new {Op1BaseType}[Op1ElementCount]; - {Op1BaseType}[] falseVal = new {Op1BaseType}[Op1ElementCount]; - {RetBaseType}[] result = new {RetBaseType}[RetElementCount]; - - Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref mask[0]), maskOp); - Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref left[0]), leftOp); - Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref right[0]), rightOp); - Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref falseVal[0]), falseOp); - Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref result[0]), ref Unsafe.AsRef(output), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); - - bool succeeded = true; - - {TemplateValidationLogicForCndSel} - - if (!succeeded) - { - TestLibrary.TestFramework.LogInformation($"{nameof({Isa})}.{nameof({Isa}.{Method})}<{RetBaseType}>({Op1VectorType}<{Op1BaseType}>, {Op2VectorType}<{Op2BaseType}>): {method} failed:"); - TestLibrary.TestFramework.LogInformation($" mask: ({string.Join(", ", mask)})"); - TestLibrary.TestFramework.LogInformation($" left: ({string.Join(", ", left)})"); - TestLibrary.TestFramework.LogInformation($" right: ({string.Join(", ", right)})"); - TestLibrary.TestFramework.LogInformation($" falseOp: ({string.Join(", ", falseVal)})"); - TestLibrary.TestFramework.LogInformation($" result: ({string.Join(", ", result)})"); - TestLibrary.TestFramework.LogInformation(string.Empty); - - Succeeded = false; - } - } - - private void ValidateResult({Op1VectorType}<{Op1BaseType}> op1, {Op2VectorType}<{Op2BaseType}> op2, void* result, [CallerMemberName] string method = "") - { - {Op1BaseType}[] inArray1 = new {Op1BaseType}[Op1ElementCount]; - {Op2BaseType}[] inArray2 = new {Op2BaseType}[Op2ElementCount]; - {RetBaseType}[] outArray = new {RetBaseType}[RetElementCount]; - - Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref inArray1[0]), op1); - Unsafe.WriteUnaligned(ref Unsafe.As<{Op2BaseType}, byte>(ref inArray2[0]), op2); - Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref outArray[0]), ref Unsafe.AsRef(result), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); - - ValidateResult(inArray1, inArray2, outArray, method); - } - - private void ValidateResult(void* op1, void* op2, void* result, [CallerMemberName] string method = "") - { - {Op1BaseType}[] inArray1 = new {Op1BaseType}[Op1ElementCount]; - {Op2BaseType}[] inArray2 = new {Op2BaseType}[Op2ElementCount]; - {RetBaseType}[] outArray = new {RetBaseType}[RetElementCount]; - - Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref inArray1[0]), ref Unsafe.AsRef(op1), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); - Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op2BaseType}, byte>(ref inArray2[0]), ref Unsafe.AsRef(op2), (uint)Unsafe.SizeOf<{Op2VectorType}<{Op2BaseType}>>()); - Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref outArray[0]), ref Unsafe.AsRef(result), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); - - ValidateResult(inArray1, inArray2, outArray, method); - } - - private void ValidateResult({Op1BaseType}[] left, {Op2BaseType}[] right, {RetBaseType}[] result, [CallerMemberName] string method = "") - { - bool succeeded = true; - - {TemplateValidationLogic} - - if (!succeeded) - { - TestLibrary.TestFramework.LogInformation($"{nameof({Isa})}.{nameof({Isa}.{Method})}<{RetBaseType}>({Op1VectorType}<{Op1BaseType}>, {Op2VectorType}<{Op2BaseType}>): {method} failed:"); - TestLibrary.TestFramework.LogInformation($" left: ({string.Join(", ", left)})"); - TestLibrary.TestFramework.LogInformation($" right: ({string.Join(", ", right)})"); - TestLibrary.TestFramework.LogInformation($" result: ({string.Join(", ", result)})"); - TestLibrary.TestFramework.LogInformation(string.Empty); - - Succeeded = false; - } - } - } -} From 76b42bd4bc1b53f9d547a1695014d5cebb76b56d Mon Sep 17 00:00:00 2001 From: TIHan Date: Tue, 2 Jul 2024 14:42:24 -0700 Subject: [PATCH 03/83] Added template --- .../SveLoadVectorFirstFaultingTest.template | 270 ++++++++++++++++++ 1 file changed, 270 insertions(+) create mode 100644 src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveLoadVectorFirstFaultingTest.template diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveLoadVectorFirstFaultingTest.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveLoadVectorFirstFaultingTest.template new file mode 100644 index 0000000000000..6db96f274322c --- /dev/null +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveLoadVectorFirstFaultingTest.template @@ -0,0 +1,270 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +/****************************************************************************** + * This file is auto-generated from a template file by the GenerateTests.csx * + * script in tests\src\JIT\HardwareIntrinsics\Arm\Shared. In order to make * + * changes, please update the corresponding template and run according to the * + * directions listed in the file. * + ******************************************************************************/ + +using System; +using System.Numerics; +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.Arm; +using Xunit; + +namespace JIT.HardwareIntrinsics.Arm +{ + public static partial class Program + { + [Fact] + public static void {TestName}() + { + var test = new LoadUnaryOpTest__{TestName}(); + + if (test.IsSupported) + { + // Validates basic functionality works + test.RunBasicScenario_Load(); + + // Validates calling via reflection works + test.RunReflectionScenario_Load(); + + // Validates passing an instance member of a class works + test.RunClassFldScenario(); + + // Validates passing the field of a local struct works + test.RunStructLclFldScenario(); + + // Validates passing an instance member of a struct works + test.RunStructFldScenario(); + } + else + { + // Validates we throw on unsupported hardware + test.RunUnsupportedScenario(); + } + + if (!test.Succeeded) + { + throw new Exception("One or more scenarios did not complete as expected."); + } + } + } + + public sealed unsafe class LoadUnaryOpTest__{TestName} + { + private struct DataTable + { + private byte[] inArray1; + private byte[] outArray; + + private GCHandle inHandle1; + private GCHandle outHandle; + + private ulong alignment; + + public DataTable({Op2BaseType}[] inArray1, {RetBaseType}[] outArray, int alignment) + { + int sizeOfinArray1 = inArray1.Length * Unsafe.SizeOf<{Op2BaseType}>(); + int sizeOfoutArray = outArray.Length * Unsafe.SizeOf<{RetBaseType}>(); + if ((alignment != 64 && alignment != 16 && alignment != 8) || (alignment * 2) < sizeOfinArray1 || (alignment * 2) < sizeOfoutArray) + { + throw new ArgumentException($"Invalid value of alignment: {alignment}, sizeOfinArray1: {sizeOfinArray1}, sizeOfoutArray: {sizeOfoutArray}"); + } + + this.inArray1 = new byte[alignment * 2]; + this.outArray = new byte[alignment * 2]; + + this.inHandle1 = GCHandle.Alloc(this.inArray1, GCHandleType.Pinned); + this.outHandle = GCHandle.Alloc(this.outArray, GCHandleType.Pinned); + + this.alignment = (ulong)alignment; + + Unsafe.CopyBlockUnaligned(ref Unsafe.AsRef(inArray1Ptr), ref Unsafe.As<{Op2BaseType}, byte>(ref inArray1[0]), (uint)sizeOfinArray1); + + } + + public void* inArray1Ptr => Align((byte*)(inHandle1.AddrOfPinnedObject().ToPointer()), alignment); + public void* outArrayPtr => Align((byte*)(outHandle.AddrOfPinnedObject().ToPointer()), alignment); + + public void Dispose() + { + inHandle1.Free(); + outHandle.Free(); + } + + private static unsafe void* Align(byte* buffer, ulong expectedAlignment) + { + return (void*)(((ulong)buffer + expectedAlignment - 1) & ~(expectedAlignment - 1)); + } + } + + private struct TestStruct + { + public {Op1VectorType}<{RetBaseType}> _fld1; + + public static TestStruct Create() + { + var testStruct = new TestStruct(); + + for (var i = 0; i < Op2ElementCount; i++) { _data[i] = {NextValueOp2}; } + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1VectorType}<{RetBaseType}>, byte>(ref testStruct._fld1), ref Unsafe.As<{Op1BaseType}, byte>(ref _data[0]), (uint)Unsafe.SizeOf<{Op1VectorType}<{RetBaseType}>>()); + + return testStruct; + } + + public void RunStructFldScenario(LoadUnaryOpTest__{TestName} testClass) + { + {Op1VectorType}<{Op1BaseType}> loadMask = Sve.CreateTrueMask{RetBaseType}(SveMaskPattern.All); + + var result = {Isa}.{Method}(loadMask, ({Op2BaseType}*)testClass._dataTable.inArray1Ptr); + + Unsafe.Write(testClass._dataTable.outArrayPtr, result); + testClass.ValidateResult(testClass._dataTable.inArray1Ptr, testClass._dataTable.outArrayPtr); + } + } + + private static readonly int LargestVectorSize = {LargestVectorSize}; + + private static readonly int Op2ElementCount = Unsafe.SizeOf<{RetVectorType}<{Op2BaseType}>>() / sizeof({Op2BaseType}); + private static readonly int RetElementCount = Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>() / sizeof({RetBaseType}); + + private static {Op1BaseType}[] _data = new {Op1BaseType}[Op2ElementCount]; + + private {Op1VectorType}<{RetBaseType}> _fld1; + + private DataTable _dataTable; + + public LoadUnaryOpTest__{TestName}() + { + Succeeded = true; + + for (var i = 0; i < Op2ElementCount; i++) { _data[i] = {NextValueOp2}; } + _dataTable = new DataTable(_data, new {RetBaseType}[RetElementCount], LargestVectorSize); + } + + public bool IsSupported => {Isa}.IsSupported; + + public bool Succeeded { get; set; } + + public void RunBasicScenario_Load() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunBasicScenario_Load)); + + {Op1VectorType}<{Op1BaseType}> loadMask = Sve.CreateTrueMask{RetBaseType}(SveMaskPattern.All); + + var result = {Isa}.{Method}( + loadMask, + ({Op2BaseType}*)(_dataTable.inArray1Ptr) + ); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateResult(_dataTable.inArray1Ptr, _dataTable.outArrayPtr); + } + + public void RunReflectionScenario_Load() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunReflectionScenario_Load)); + + {Op1VectorType}<{Op1BaseType}> loadMask = Sve.CreateTrueMask{RetBaseType}(SveMaskPattern.All); + + var result = typeof({Isa}).GetMethod(nameof({Isa}.{Method}), new Type[] { typeof(Vector<{Op1BaseType}>), typeof({Op2BaseType}*) }) + .Invoke(null, new object[] { + loadMask, + Pointer.Box(_dataTable.inArray1Ptr, typeof({Op2BaseType}*)) + }); + + Unsafe.Write(_dataTable.outArrayPtr, ({RetVectorType}<{RetBaseType}>)(result)); + ValidateResult(_dataTable.inArray1Ptr, _dataTable.outArrayPtr); + } + + public void RunClassFldScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunClassFldScenario)); + + {Op1VectorType}<{Op1BaseType}> loadMask = Sve.CreateTrueMask{RetBaseType}(SveMaskPattern.All); + + _fld1 = {Isa}.{Method}(loadMask, ({Op2BaseType}*)_dataTable.inArray1Ptr); + + Unsafe.Write(_dataTable.outArrayPtr, _fld1); + ValidateResult(_dataTable.inArray1Ptr, _dataTable.outArrayPtr); + } + + public void RunStructLclFldScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunStructLclFldScenario)); + + {Op1VectorType}<{Op1BaseType}> loadMask = Sve.CreateTrueMask{RetBaseType}(SveMaskPattern.All); + + var test = TestStruct.Create(); + test._fld1 = {Isa}.{Method}(loadMask, ({Op2BaseType}*)_dataTable.inArray1Ptr); + + Unsafe.Write(_dataTable.outArrayPtr, test._fld1); + ValidateResult(_dataTable.inArray1Ptr, _dataTable.outArrayPtr); + } + + public void RunStructFldScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunStructFldScenario)); + + var test = TestStruct.Create(); + test.RunStructFldScenario(this); + } + + public void RunUnsupportedScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunUnsupportedScenario)); + + Succeeded = false; + + try + { + RunBasicScenario_Load(); + } + catch (PlatformNotSupportedException) + { + Succeeded = true; + } + } + + private void ValidateResult(void* firstOp, void* result, [CallerMemberName] string method = "") + { + {Op2BaseType}[] inArray = new {Op2BaseType}[Op2ElementCount]; + {RetBaseType}[] outArray = new {RetBaseType}[RetElementCount]; + + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op2BaseType}, byte>(ref inArray[0]), ref Unsafe.AsRef(firstOp), (uint)Unsafe.SizeOf<{RetVectorType}<{Op2BaseType}>>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref outArray[0]), ref Unsafe.AsRef(result), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); + + ValidateResult(inArray, outArray, method); + } + + private void ValidateResult({Op2BaseType}[] firstOp, {RetBaseType}[] result, [CallerMemberName] string method = "") + { + bool succeeded = true; + + for (var i = 0; i < RetElementCount; i++) + { + if ({ValidateIterResult}) + { + succeeded = false; + break; + } + } + + if (!succeeded) + { + TestLibrary.TestFramework.LogInformation($"{nameof({Isa})}.{nameof({Isa}.{Method})}<{RetBaseType}>({Op1VectorType}<{Op1BaseType}>, {Op2BaseType}): {method} failed:"); + TestLibrary.TestFramework.LogInformation($" firstOp: ({string.Join(", ", firstOp)})"); + TestLibrary.TestFramework.LogInformation($" result: ({string.Join(", ", result)})"); + TestLibrary.TestFramework.LogInformation(string.Empty); + + Succeeded = false; + } + } + } +} From bb01e370b73f9b4a2f55af723c774f6d58462055 Mon Sep 17 00:00:00 2001 From: TIHan Date: Wed, 3 Jul 2024 19:43:22 -0700 Subject: [PATCH 04/83] Trying to test first-faulting behavior --- src/coreclr/jit/fgdiagnostic.cpp | 3 + src/coreclr/jit/gentree.cpp | 6 ++ src/coreclr/jit/hwintrinsiccodegenarm64.cpp | 7 ++ src/coreclr/jit/hwintrinsiclistarm64sve.h | 6 +- .../SveLoadVectorFirstFaultingTest.template | 100 +++++++++++++++++- 5 files changed, 114 insertions(+), 8 deletions(-) diff --git a/src/coreclr/jit/fgdiagnostic.cpp b/src/coreclr/jit/fgdiagnostic.cpp index ffc7a933b5cb5..c70e177b6c317 100644 --- a/src/coreclr/jit/fgdiagnostic.cpp +++ b/src/coreclr/jit/fgdiagnostic.cpp @@ -3459,6 +3459,9 @@ void Compiler::fgDebugCheckFlags(GenTree* tree, BasicBlock* block) #if defined(TARGET_ARM64) case NI_ArmBase_Yield: + case NI_Sve_GatherVectorFirstFaulting: + case NI_Sve_LoadVectorFirstFaulting: + case NI_Sve_SetFfr: { assert(tree->OperRequiresCallFlag(this)); expectedFlags |= GTF_GLOB_REF; diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 485be7a5cd28d..1fcc4215a208d 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -27983,6 +27983,9 @@ bool GenTreeHWIntrinsic::OperRequiresCallFlag() const #if defined(TARGET_ARM64) case NI_ArmBase_Yield: + case NI_Sve_GatherVectorFirstFaulting: + case NI_Sve_LoadVectorFirstFaulting: + case NI_Sve_SetFfr: { return true; } @@ -28165,6 +28168,9 @@ void GenTreeHWIntrinsic::Initialize(NamedIntrinsic intrinsicId) #if defined(TARGET_ARM64) case NI_ArmBase_Yield: + case NI_Sve_GatherVectorFirstFaulting: + case NI_Sve_LoadVectorFirstFaulting: + case NI_Sve_SetFfr: { // Mark as a call and global reference, much as is done for GT_KEEPALIVE gtFlags |= (GTF_CALL | GTF_GLOB_REF); diff --git a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp index 3be591c1811ee..d1eaab44c7103 100644 --- a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp @@ -2324,6 +2324,13 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) break; } + case NI_Sve_SetFfr: + { + assert(targetReg == REG_NA); + GetEmitter()->emitIns_R(ins, emitSize, op1Reg, opt); + break; + } + default: unreached(); } diff --git a/src/coreclr/jit/hwintrinsiclistarm64sve.h b/src/coreclr/jit/hwintrinsiclistarm64sve.h index c003ff22ee99b..6d6587711c97a 100644 --- a/src/coreclr/jit/hwintrinsiclistarm64sve.h +++ b/src/coreclr/jit/hwintrinsiclistarm64sve.h @@ -90,7 +90,7 @@ HARDWARE_INTRINSIC(Sve, GatherPrefetch64Bit, HARDWARE_INTRINSIC(Sve, GatherPrefetch8Bit, -1, -1, false, {INS_sve_prfb, INS_sve_prfb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasImmediateOperand|HW_Flag_HasEnumOperand) HARDWARE_INTRINSIC(Sve, GatherVector, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_sve_ld1w, INS_sve_ld1d, INS_sve_ld1d, INS_sve_ld1w, INS_sve_ld1d}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, GatherVectorByteZeroExtend, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1b, INS_sve_ld1b, INS_sve_ld1b, INS_sve_ld1b, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, GatherVectorFirstFaulting, -1, 3, true, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1w, INS_sve_ldff1w, INS_sve_ldff1d, INS_sve_ldff1d, INS_sve_ldff1w, INS_sve_ldff1d}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, GatherVectorFirstFaulting, -1, 3, true, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1w, INS_sve_ldff1w, INS_sve_ldff1d, INS_sve_ldff1d, INS_sve_ldff1w, INS_sve_ldff1d}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialSideEffectMask) HARDWARE_INTRINSIC(Sve, GatherVectorInt16SignExtend, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sh, INS_sve_ld1sh, INS_sve_ld1sh, INS_sve_ld1sh, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, GatherVectorInt16WithByteOffsetsSignExtend, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sh, INS_sve_ld1sh, INS_sve_ld1sh, INS_sve_ld1sh, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, GatherVectorInt32SignExtend, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sw, INS_sve_ld1sw, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) @@ -123,7 +123,7 @@ HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendToInt64, HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendToUInt16, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendToUInt32, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendToUInt64, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1b, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorFirstFaulting, -1, 2, true, {INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1h, INS_sve_ldff1h, INS_sve_ldff1w, INS_sve_ldff1w, INS_sve_ldff1d, INS_sve_ldff1d, INS_sve_ldff1w, INS_sve_ldff1d}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, LoadVectorFirstFaulting, -1, 2, true, {INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1h, INS_sve_ldff1h, INS_sve_ldff1w, INS_sve_ldff1w, INS_sve_ldff1d, INS_sve_ldff1d, INS_sve_ldff1w, INS_sve_ldff1d}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialSideEffectMask) HARDWARE_INTRINSIC(Sve, LoadVectorInt16NonFaultingSignExtendToInt32, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorInt16NonFaultingSignExtendToInt64, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sh, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorInt16NonFaultingSignExtendToUInt32, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sh, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) @@ -210,7 +210,7 @@ HARDWARE_INTRINSIC(Sve, SaturatingIncrementBy64BitElementCount, HARDWARE_INTRINSIC(Sve, SaturatingIncrementBy8BitElementCount, 0, 3, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqincb, INS_sve_uqincb, INS_sve_sqincb, INS_sve_uqincb, INS_invalid, INS_invalid}, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasEnumOperand|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) HARDWARE_INTRINSIC(Sve, SaturatingIncrementByActiveElementCount, -1, 2, true, {INS_invalid, INS_sve_sqincp, INS_sve_sqincp, INS_sve_sqincp, INS_sve_sqincp, INS_sve_sqincp, INS_sve_sqincp, INS_sve_sqincp, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) HARDWARE_INTRINSIC(Sve, Scale, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fscale, INS_sve_fscale}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve, SetFfr, -1, 1, true, {INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation) +HARDWARE_INTRINSIC(Sve, SetFfr, -1, 1, true, {INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialSideEffectMask|HW_Flag_SpecialCodeGen) HARDWARE_INTRINSIC(Sve, ShiftLeftLogical, -1, -1, false, {INS_sve_lsl, INS_sve_lsl, INS_sve_lsl, INS_sve_lsl, INS_sve_lsl, INS_sve_lsl, INS_sve_lsl, INS_sve_lsl, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) HARDWARE_INTRINSIC(Sve, ShiftRightArithmetic, -1, -1, false, {INS_sve_asr, INS_invalid, INS_sve_asr, INS_invalid, INS_sve_asr, INS_invalid, INS_sve_asr, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) HARDWARE_INTRINSIC(Sve, ShiftRightLogical, -1, -1, false, {INS_invalid, INS_sve_lsr, INS_invalid, INS_sve_lsr, INS_invalid, INS_sve_lsr, INS_invalid, INS_sve_lsr, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveLoadVectorFirstFaultingTest.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveLoadVectorFirstFaultingTest.template index 6db96f274322c..6cf38c1a27b54 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveLoadVectorFirstFaultingTest.template +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveLoadVectorFirstFaultingTest.template @@ -24,12 +24,13 @@ namespace JIT.HardwareIntrinsics.Arm [Fact] public static void {TestName}() { - var test = new LoadUnaryOpTest__{TestName}(); + var test = new Sve__{TestName}(); if (test.IsSupported) { // Validates basic functionality works test.RunBasicScenario_Load(); + test.RunBasicScenario_LoadFirstFaulting(); // Validates calling via reflection works test.RunReflectionScenario_Load(); @@ -56,14 +57,16 @@ namespace JIT.HardwareIntrinsics.Arm } } - public sealed unsafe class LoadUnaryOpTest__{TestName} + public sealed unsafe class Sve__{TestName} { private struct DataTable { private byte[] inArray1; + private byte[] inArray1Half; private byte[] outArray; private GCHandle inHandle1; + private GCHandle inHandle1Half; private GCHandle outHandle; private ulong alignment; @@ -71,6 +74,7 @@ namespace JIT.HardwareIntrinsics.Arm public DataTable({Op2BaseType}[] inArray1, {RetBaseType}[] outArray, int alignment) { int sizeOfinArray1 = inArray1.Length * Unsafe.SizeOf<{Op2BaseType}>(); + int sizeOfinArray1Half = sizeOfinArray1 / 2; int sizeOfoutArray = outArray.Length * Unsafe.SizeOf<{RetBaseType}>(); if ((alignment != 64 && alignment != 16 && alignment != 8) || (alignment * 2) < sizeOfinArray1 || (alignment * 2) < sizeOfoutArray) { @@ -78,23 +82,27 @@ namespace JIT.HardwareIntrinsics.Arm } this.inArray1 = new byte[alignment * 2]; + this.inArray1Half = new byte[(alignment * 2) / 2]; this.outArray = new byte[alignment * 2]; this.inHandle1 = GCHandle.Alloc(this.inArray1, GCHandleType.Pinned); + this.inHandle1Half = GCHandle.Alloc(this.inArray1Half, GCHandleType.Pinned); this.outHandle = GCHandle.Alloc(this.outArray, GCHandleType.Pinned); this.alignment = (ulong)alignment; Unsafe.CopyBlockUnaligned(ref Unsafe.AsRef(inArray1Ptr), ref Unsafe.As<{Op2BaseType}, byte>(ref inArray1[0]), (uint)sizeOfinArray1); - + Unsafe.CopyBlockUnaligned(ref Unsafe.AsRef(inArray1HalfPtr), ref Unsafe.As<{Op2BaseType}, byte>(ref inArray1[0]), (uint)sizeOfinArray1Half); } public void* inArray1Ptr => Align((byte*)(inHandle1.AddrOfPinnedObject().ToPointer()), alignment); + public void* inArray1HalfPtr => Align((byte*)(inHandle1Half.AddrOfPinnedObject().ToPointer()), alignment); public void* outArrayPtr => Align((byte*)(outHandle.AddrOfPinnedObject().ToPointer()), alignment); public void Dispose() { inHandle1.Free(); + inHandle1Half.Free(); outHandle.Free(); } @@ -118,7 +126,7 @@ namespace JIT.HardwareIntrinsics.Arm return testStruct; } - public void RunStructFldScenario(LoadUnaryOpTest__{TestName} testClass) + public void RunStructFldScenario(Sve__{TestName} testClass) { {Op1VectorType}<{Op1BaseType}> loadMask = Sve.CreateTrueMask{RetBaseType}(SveMaskPattern.All); @@ -140,7 +148,7 @@ namespace JIT.HardwareIntrinsics.Arm private DataTable _dataTable; - public LoadUnaryOpTest__{TestName}() + public Sve__{TestName}() { Succeeded = true; @@ -167,6 +175,24 @@ namespace JIT.HardwareIntrinsics.Arm ValidateResult(_dataTable.inArray1Ptr, _dataTable.outArrayPtr); } + public void RunBasicScenario_LoadFirstFaulting() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunBasicScenario_LoadFirstFaulting)); + + {Op1VectorType}<{Op1BaseType}> loadMask = Sve.CreateTrueMask{RetBaseType}(SveMaskPattern.All); + + var op2Ptr = ({Op2BaseType}*)(_dataTable.inArray1HalfPtr); + Sve.SetFfr(Sve.CreateTrueMaskByte(SveMaskPattern.All)); + var result = {Isa}.{Method}( + loadMask, + op2Ptr + ); + var faultResult = Sve.GetFfr(); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateFirstFaultingResult(_dataTable.inArray1HalfPtr, _dataTable.outArrayPtr, faultResult.As()); + } + public void RunReflectionScenario_Load() { TestLibrary.TestFramework.BeginScenario(nameof(RunReflectionScenario_Load)); @@ -266,5 +292,69 @@ namespace JIT.HardwareIntrinsics.Arm Succeeded = false; } } + + private void ValidateFirstFaultingResult(void* firstOp, void* result, Vector<{RetBaseType}> faultResult, [CallerMemberName] string method = "") + { + {Op2BaseType}[] inArray = new {Op2BaseType}[Op2ElementCount / 2]; + {RetBaseType}[] outArray = new {RetBaseType}[RetElementCount]; + + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op2BaseType}, byte>(ref inArray[0]), ref Unsafe.AsRef(firstOp), (uint)Unsafe.SizeOf<{RetVectorType}<{Op2BaseType}>>() / 2); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref outArray[0]), ref Unsafe.AsRef(result), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); + + ValidateFirstFaultingResult(inArray, outArray, faultResult, method); + } + + private void ValidateFirstFaultingResult({Op2BaseType}[] firstOp, {RetBaseType}[] result, Vector<{RetBaseType}> faultResult, [CallerMemberName] string method = "") + { + bool succeeded = true; + + for (var i = 0; i < RetElementCount; i++) + { + if (i < (RetElementCount / 2)) + { + if ({ValidateIterResult}) + { + succeeded = false; + break; + } + } + else + { + if (result[i] != 0) + { + succeeded = false; + break; + } + } + + if (i == (RetElementCount / 2)) + { + if (faultResult[i] != 0) + { + succeeded = false; + break; + } + } + else + { + if (faultResult[i] != 1) + { + succeeded = false; + break; + } + } + } + + if (!succeeded) + { + TestLibrary.TestFramework.LogInformation($"{nameof({Isa})}.{nameof({Isa}.{Method})}({Op1VectorType}<{Op1BaseType}>, {Op1VectorType}<{Op2BaseType}>): {method} failed:"); + TestLibrary.TestFramework.LogInformation($" firstOp: ({string.Join(", ", firstOp)})"); + TestLibrary.TestFramework.LogInformation($" result: ({string.Join(", ", result)})"); + TestLibrary.TestFramework.LogInformation($" faultResult: ({faultResult})"); + TestLibrary.TestFramework.LogInformation(string.Empty); + + Succeeded = false; + } + } } } From a602b245c268fe6c4664c2cf58f0c2841c8248b0 Mon Sep 17 00:00:00 2001 From: TIHan Date: Fri, 5 Jul 2024 19:37:07 -0700 Subject: [PATCH 05/83] Using BoundedMemory to test FirstFaulting behavior for LoadVector. --- .../System/Buffers/BoundedMemory.Windows.cs | 18 ++-- .../SveLoadVectorFirstFaultingTest.template | 92 +++++++++++++------ .../HardwareIntrinsics/Arm/Sve/Sve_r.csproj | 5 + .../HardwareIntrinsics/Arm/Sve/Sve_ro.csproj | 5 + 4 files changed, 81 insertions(+), 39 deletions(-) diff --git a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Windows.cs b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Windows.cs index 96f40d61492e5..82572aaf14a18 100644 --- a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Windows.cs +++ b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Windows.cs @@ -288,38 +288,38 @@ protected override bool ReleaseHandle() => UnsafeNativeMethods.VirtualFree(handle, IntPtr.Zero, VirtualAllocAllocationType.MEM_RELEASE); } - private static partial class UnsafeNativeMethods + private static class UnsafeNativeMethods { private const string KERNEL32_LIB = "kernel32.dll"; // https://msdn.microsoft.com/en-us/library/windows/desktop/aa366887(v=vs.85).aspx - [LibraryImport(KERNEL32_LIB, SetLastError = true)] - public static partial VirtualAllocHandle VirtualAlloc( + [DllImport(KERNEL32_LIB, SetLastError = true)] + public static extern VirtualAllocHandle VirtualAlloc( IntPtr lpAddress, IntPtr dwSize, VirtualAllocAllocationType flAllocationType, VirtualAllocProtection flProtect); // https://msdn.microsoft.com/en-us/library/windows/desktop/aa366892(v=vs.85).aspx - [LibraryImport(KERNEL32_LIB, SetLastError = true)] + [DllImport(KERNEL32_LIB, SetLastError = true)] [return: MarshalAs(UnmanagedType.Bool)] - public static partial bool VirtualFree( + public static extern bool VirtualFree( IntPtr lpAddress, IntPtr dwSize, VirtualAllocAllocationType dwFreeType); // https://msdn.microsoft.com/en-us/library/windows/desktop/aa366898(v=vs.85).aspx - [LibraryImport(KERNEL32_LIB, SetLastError = true)] + [DllImport(KERNEL32_LIB, SetLastError = true)] [return: MarshalAs(UnmanagedType.Bool)] - public static partial bool VirtualProtect( + public static extern bool VirtualProtect( IntPtr lpAddress, IntPtr dwSize, VirtualAllocProtection flNewProtect, out VirtualAllocProtection lpflOldProtect); // https://msdn.microsoft.com/en-us/library/windows/desktop/aa366902(v=vs.85).aspx - [LibraryImport(KERNEL32_LIB, SetLastError = true)] - public static partial IntPtr VirtualQuery( + [DllImport(KERNEL32_LIB, SetLastError = true)] + public static extern IntPtr VirtualQuery( IntPtr lpAddress, out MEMORY_BASIC_INFORMATION lpBuffer, IntPtr dwLength); diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveLoadVectorFirstFaultingTest.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveLoadVectorFirstFaultingTest.template index 6cf38c1a27b54..d92fa79e3c392 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveLoadVectorFirstFaultingTest.template +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveLoadVectorFirstFaultingTest.template @@ -9,6 +9,7 @@ ******************************************************************************/ using System; +using System.Buffers; using System.Numerics; using System.Reflection; using System.Runtime.CompilerServices; @@ -31,6 +32,7 @@ namespace JIT.HardwareIntrinsics.Arm // Validates basic functionality works test.RunBasicScenario_Load(); test.RunBasicScenario_LoadFirstFaulting(); + test.RunBasicScenario_LoadFirstFaulting_IntermittentCalls(); // Validates calling via reflection works test.RunReflectionScenario_Load(); @@ -62,11 +64,9 @@ namespace JIT.HardwareIntrinsics.Arm private struct DataTable { private byte[] inArray1; - private byte[] inArray1Half; private byte[] outArray; private GCHandle inHandle1; - private GCHandle inHandle1Half; private GCHandle outHandle; private ulong alignment; @@ -74,7 +74,7 @@ namespace JIT.HardwareIntrinsics.Arm public DataTable({Op2BaseType}[] inArray1, {RetBaseType}[] outArray, int alignment) { int sizeOfinArray1 = inArray1.Length * Unsafe.SizeOf<{Op2BaseType}>(); - int sizeOfinArray1Half = sizeOfinArray1 / 2; + int sizeOfinBounded = new Random().Next(Unsafe.SizeOf<{Op2BaseType}>(), sizeOfinArray1 - 1); int sizeOfoutArray = outArray.Length * Unsafe.SizeOf<{RetBaseType}>(); if ((alignment != 64 && alignment != 16 && alignment != 8) || (alignment * 2) < sizeOfinArray1 || (alignment * 2) < sizeOfoutArray) { @@ -82,27 +82,27 @@ namespace JIT.HardwareIntrinsics.Arm } this.inArray1 = new byte[alignment * 2]; - this.inArray1Half = new byte[(alignment * 2) / 2]; this.outArray = new byte[alignment * 2]; this.inHandle1 = GCHandle.Alloc(this.inArray1, GCHandleType.Pinned); - this.inHandle1Half = GCHandle.Alloc(this.inArray1Half, GCHandleType.Pinned); this.outHandle = GCHandle.Alloc(this.outArray, GCHandleType.Pinned); this.alignment = (ulong)alignment; + this.inBounded = BoundedMemory.Allocate(sizeOfinBounded, PoisonPagePlacement.After); + Unsafe.CopyBlockUnaligned(ref Unsafe.AsRef(inArray1Ptr), ref Unsafe.As<{Op2BaseType}, byte>(ref inArray1[0]), (uint)sizeOfinArray1); - Unsafe.CopyBlockUnaligned(ref Unsafe.AsRef(inArray1HalfPtr), ref Unsafe.As<{Op2BaseType}, byte>(ref inArray1[0]), (uint)sizeOfinArray1Half); + Unsafe.CopyBlockUnaligned(ref inBounded.Span.GetPinnableReference(), ref Unsafe.As<{Op2BaseType}, byte>(ref inArray1[0]), (uint)sizeOfinBounded); } + public BoundedMemory inBounded; + public void* inArray1Ptr => Align((byte*)(inHandle1.AddrOfPinnedObject().ToPointer()), alignment); - public void* inArray1HalfPtr => Align((byte*)(inHandle1Half.AddrOfPinnedObject().ToPointer()), alignment); public void* outArrayPtr => Align((byte*)(outHandle.AddrOfPinnedObject().ToPointer()), alignment); public void Dispose() { inHandle1.Free(); - inHandle1Half.Free(); outHandle.Free(); } @@ -181,16 +181,44 @@ namespace JIT.HardwareIntrinsics.Arm {Op1VectorType}<{Op1BaseType}> loadMask = Sve.CreateTrueMask{RetBaseType}(SveMaskPattern.All); - var op2Ptr = ({Op2BaseType}*)(_dataTable.inArray1HalfPtr); + ref var op2Ref = ref (_dataTable.inBounded.Span.GetPinnableReference()); + + Sve.SetFfr(Sve.CreateTrueMaskByte(SveMaskPattern.All)); + var result = {Isa}.{Method}( + loadMask, + ({Op2BaseType}*)(Unsafe.AsPointer(ref op2Ref)) + ); + var faultResult = Sve.GetFfr(); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateFirstFaultingResult(ref op2Ref, _dataTable.inBounded.Span.Length, _dataTable.outArrayPtr, faultResult.As()); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static void ConsumeNothing() + { + } + + public void RunBasicScenario_LoadFirstFaulting_IntermittentCalls() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunBasicScenario_LoadFirstFaulting_IntermittentCalls)); + + {Op1VectorType}<{Op1BaseType}> loadMask = Sve.CreateTrueMask{RetBaseType}(SveMaskPattern.All); + + ref var op2Ref = ref (_dataTable.inBounded.Span.GetPinnableReference()); + Sve.SetFfr(Sve.CreateTrueMaskByte(SveMaskPattern.All)); + ConsumeNothing(); var result = {Isa}.{Method}( loadMask, - op2Ptr + ({Op2BaseType}*)(Unsafe.AsPointer(ref op2Ref)) ); + ConsumeNothing(); var faultResult = Sve.GetFfr(); + ConsumeNothing(); Unsafe.Write(_dataTable.outArrayPtr, result); - ValidateFirstFaultingResult(_dataTable.inArray1HalfPtr, _dataTable.outArrayPtr, faultResult.As()); + ValidateFirstFaultingResult(ref op2Ref, _dataTable.inBounded.Span.Length, _dataTable.outArrayPtr, faultResult.As()); } public void RunReflectionScenario_Load() @@ -293,12 +321,12 @@ namespace JIT.HardwareIntrinsics.Arm } } - private void ValidateFirstFaultingResult(void* firstOp, void* result, Vector<{RetBaseType}> faultResult, [CallerMemberName] string method = "") + private void ValidateFirstFaultingResult(ref byte firstOp, int size, void* result, Vector<{RetBaseType}> faultResult, [CallerMemberName] string method = "") { - {Op2BaseType}[] inArray = new {Op2BaseType}[Op2ElementCount / 2]; + {Op2BaseType}[] inArray = new {Op2BaseType}[size / Unsafe.SizeOf<{Op2BaseType}>()]; {RetBaseType}[] outArray = new {RetBaseType}[RetElementCount]; - Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op2BaseType}, byte>(ref inArray[0]), ref Unsafe.AsRef(firstOp), (uint)Unsafe.SizeOf<{RetVectorType}<{Op2BaseType}>>() / 2); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op2BaseType}, byte>(ref inArray[0]), ref firstOp, (uint)(size)); Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref outArray[0]), ref Unsafe.AsRef(result), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); ValidateFirstFaultingResult(inArray, outArray, faultResult, method); @@ -308,39 +336,43 @@ namespace JIT.HardwareIntrinsics.Arm { bool succeeded = true; + bool hitFault = false; + for (var i = 0; i < RetElementCount; i++) { - if (i < (RetElementCount / 2)) + if (hitFault) { - if ({ValidateIterResult}) + if (faultResult[i] != 0) { succeeded = false; break; } - } - else - { + if (result[i] != 0) { succeeded = false; break; } } - - if (i == (RetElementCount / 2)) + else { - if (faultResult[i] != 0) + if (faultResult[i] == 0) { - succeeded = false; - break; + // There has to be a valid value for the first element, so check it. + if (i == 0) + { + succeeded = false; + break; + } + hitFault = true; } - } - else - { - if (faultResult[i] != 1) + else { - succeeded = false; - break; + if ({ValidateIterResult}) + { + succeeded = false; + break; + } } } } diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Sve/Sve_r.csproj b/src/tests/JIT/HardwareIntrinsics/Arm/Sve/Sve_r.csproj index 0918724cfff28..dd4135fc1c866 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Sve/Sve_r.csproj +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Sve/Sve_r.csproj @@ -10,5 +10,10 @@ + + + + + diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Sve/Sve_ro.csproj b/src/tests/JIT/HardwareIntrinsics/Arm/Sve/Sve_ro.csproj index a9d4a43b42ae1..e23748ff74c8b 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Sve/Sve_ro.csproj +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Sve/Sve_ro.csproj @@ -10,5 +10,10 @@ + + + + + From 60d410a2495ad4e813fd604e4dec2997ad00b736 Mon Sep 17 00:00:00 2001 From: TIHan Date: Fri, 5 Jul 2024 20:02:15 -0700 Subject: [PATCH 06/83] Fix size in validation --- .../Arm/Shared/SveLoadVectorFirstFaultingTest.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveLoadVectorFirstFaultingTest.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveLoadVectorFirstFaultingTest.template index d92fa79e3c392..7c0970962d02b 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveLoadVectorFirstFaultingTest.template +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveLoadVectorFirstFaultingTest.template @@ -326,7 +326,7 @@ namespace JIT.HardwareIntrinsics.Arm {Op2BaseType}[] inArray = new {Op2BaseType}[size / Unsafe.SizeOf<{Op2BaseType}>()]; {RetBaseType}[] outArray = new {RetBaseType}[RetElementCount]; - Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op2BaseType}, byte>(ref inArray[0]), ref firstOp, (uint)(size)); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op2BaseType}, byte>(ref inArray[0]), ref firstOp, (uint)(inArray.Length * Unsafe.SizeOf<{Op2BaseType}>())); Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref outArray[0]), ref Unsafe.AsRef(result), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); ValidateFirstFaultingResult(inArray, outArray, faultResult, method); From aee87d70bc863c54c49d052e7c8b5b17c70a6ecd Mon Sep 17 00:00:00 2001 From: TIHan Date: Mon, 8 Jul 2024 12:02:32 -0700 Subject: [PATCH 07/83] Added more helper functions. Added conditional select tests for LoadVectorFirstFaulting. --- .../GenerateHWIntrinsicTests_Arm.cs | 20 +-- .../HardwareIntrinsics/Arm/Shared/Helpers.cs | 109 +++++++++++++ .../Shared/SveGatherVectorIndices.template | 25 +-- .../SveLoadVectorFirstFaultingTest.template | 146 ++++++++---------- 4 files changed, 189 insertions(+), 111 deletions(-) diff --git a/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs b/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs index d6c0179d2afab..c844ef1647fea 100644 --- a/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs +++ b/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs @@ -3502,16 +3502,16 @@ ("SveLoadNonFaultingUnOpTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorNonFaulting_uint", ["Isa"] = "Sve", ["Method"] = "LoadVectorNonFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2BaseType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt32()", ["Cast"] = "(uint*)", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), ("SveLoadNonFaultingUnOpTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorNonFaulting_ulong", ["Isa"] = "Sve", ["Method"] = "LoadVectorNonFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2BaseType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt64()", ["Cast"] = "(ulong*)", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_float", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Single", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSingle()", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_double", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Double", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetDouble()", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_sbyte", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "SByte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "SByte", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "SByte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSByte()", ["NextValueOp2"] = "TestLibrary.Generator.GetSByte()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_short", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int16", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt16()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt16()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt32()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt64()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_byte", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Byte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Byte", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetByte()", ["NextValueOp2"] = "TestLibrary.Generator.GetByte()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_ushort", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt16", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt16()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt16()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_float", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Single", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSingle()", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_double", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Double", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetDouble()", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_sbyte", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "SByte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "SByte", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "SByte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSByte()", ["NextValueOp2"] = "TestLibrary.Generator.GetSByte()"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_short", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int16", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt16()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt16()"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt32()"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt64()"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_byte", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Byte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Byte", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetByte()", ["NextValueOp2"] = "TestLibrary.Generator.GetByte()"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_ushort", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt16", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt16()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt16()"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()"}), ("SveLoadVectorMaskedTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorNonTemporal_float", ["Isa"] = "Sve", ["Method"] = "LoadVectorNonTemporal", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2BaseType"] = "Single", ["LargestVectorSize"] = "64", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), ("SveLoadVectorMaskedTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorNonTemporal_double", ["Isa"] = "Sve", ["Method"] = "LoadVectorNonTemporal", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2BaseType"] = "Double", ["LargestVectorSize"] = "64", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs index 3e987ec461297..efe698688cdad 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs @@ -7699,5 +7699,114 @@ public static T[] CreateBreakBeforePropagateMask(T[] mask, T[] op1, T[] op2) } return result; } + + private static T ConditionalSelectResult(T maskResult, T result, T falseResult) where T : INumberBase + { + return (maskResult != T.Zero) ? result : falseResult; + } + + private static bool CheckLoadVectorBehaviorCore(T[] firstOp, T[] result, Func map) where T : INumberBase + { + for (var i = 0; i < firstOp.Length; i++) + { + T loadResult = firstOp[i]; + loadResult = map(i, loadResult); + if (result[i] != loadResult) + { + return false; + } + } + return true; + } + + public static bool CheckLoadVectorBehavior(T[] firstOp, T[] result) where T : INumberBase + { + return CheckLoadVectorBehaviorCore(firstOp, result, (_, loadResult) => loadResult); + } + + public static bool CheckLoadVectorBehavior(T[] maskOp, T[] firstOp, T[] result, T[] falseOp) where T : INumberBase + { + return CheckLoadVectorBehaviorCore(firstOp, result, (i, loadResult) => ConditionalSelectResult(maskOp[i], loadResult, falseOp[i])); + } + + private static bool CheckGatherVectorBehaviorCore(T[] firstOp, ExtendedElementT[] secondOp, Index[] thirdOp, T[] result, Func map) + where T : INumberBase + where ExtendedElementT : INumberBase + where Index : IBinaryInteger + { + for (var i = 0; i < firstOp.Length; i++) + { + T gatherResult = (firstOp[i] == T.Zero) ? T.Zero : T.CreateTruncating(secondOp[int.CreateChecked(thirdOp[i])]); + gatherResult = map(i, gatherResult); + if (result[i] != gatherResult) + { + return false; + } + } + return true; + } + + public static bool CheckGatherVectorBehavior(T[] firstOp, ExtendedElementT[] secondOp, Index[] thirdOp, T[] result) + where T : INumberBase + where ExtendedElementT : INumberBase + where Index : IBinaryInteger + { + return CheckGatherVectorBehaviorCore(firstOp, secondOp, thirdOp, result, (_, gatherResult) => gatherResult); + } + + public static bool CheckGatherVectorBehavior(T[] maskOp, T[] firstOp, ExtendedElementT[] secondOp, Index[] thirdOp, T[] falseOp, T[] result) + where T : INumberBase + where ExtendedElementT : INumberBase + where Index : IBinaryInteger + { + return CheckGatherVectorBehaviorCore(firstOp, secondOp, thirdOp, result, (i, gatherResult) => ConditionalSelectResult(maskOp[i], gatherResult, falseOp[i])); + } + + private static bool CheckFirstFaultingBehaviorCore(T[] firstOp, T[] result, Vector faultResult, Func checkIter) where T : INumberBase + { + bool hitFault = false; + + for (var i = 0; i < firstOp.Length; i++) + { + if (hitFault) + { + if (faultResult[i] != T.Zero) + { + return false; + } + + if (result[i] != T.Zero) + { + return false; + } + } + else + { + if (faultResult[i] == T.Zero) + { + // There has to be a valid value for the first element, so check it. + if (i == 0) + { + return false; + } + hitFault = true; + } + else + { + if (!checkIter(i)) + { + return false; + } + } + } + } + + return true; + } + + public static bool CheckLoadVectorFirstFaultingBehavior(T[] firstOp, T[] result, Vector faultResult) where T : INumberBase + { + return CheckFirstFaultingBehaviorCore(firstOp, result, faultResult, i => firstOp[i] == result[i]); + } } } diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorIndices.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorIndices.template index 1b97022a3ef58..89777e737ee1c 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorIndices.template +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorIndices.template @@ -443,17 +443,7 @@ namespace JIT.HardwareIntrinsics.Arm private void ValidateResult({Op1BaseType}[] firstOp, {Op2BaseType}[] secondOp, {Op3BaseType}[] thirdOp, {RetBaseType}[] result, [CallerMemberName] string method = "") { - bool succeeded = true; - - for (var i = 0; i < RetElementCount; i++) - { - {RetBaseType} gatherResult = ({RetBaseType})(firstOp[i] == 0 ? 0 : ({ExtendedElementType})secondOp[thirdOp[i]]); - if (result[i] != gatherResult) - { - succeeded = false; - break; - } - } + var succeeded = Helpers.CheckGatherVectorBehavior<{RetBaseType}, {ExtendedElementType}, {Op3BaseType}>(firstOp, secondOp, thirdOp, result); if (!succeeded) { @@ -545,18 +535,7 @@ namespace JIT.HardwareIntrinsics.Arm private void ValidateConditionalSelectResult({RetBaseType}[] maskOp, {Op1BaseType}[] firstOp, {Op2BaseType}[] secondOp, {Op3BaseType}[] thirdOp, {RetBaseType}[] falseOp, {RetBaseType}[] result, [CallerMemberName] string method = "") { - bool succeeded = true; - - for (var i = 0; i < RetElementCount; i++) - { - {RetBaseType} gatherResult = ({RetBaseType})(firstOp[i] == 0 ? 0 : ({ExtendedElementType})secondOp[thirdOp[i]]); - {RetBaseType} iterResult = (maskOp[i] != 0) ? gatherResult : falseOp[i]; - if (iterResult != result[i]) - { - succeeded = false; - break; - } - } + var succeeded = Helpers.CheckGatherVectorBehavior(maskOp, firstOp, secondOp, thirdOp, falseOp, result); if (!succeeded) { diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveLoadVectorFirstFaultingTest.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveLoadVectorFirstFaultingTest.template index 7c0970962d02b..d446a36b50994 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveLoadVectorFirstFaultingTest.template +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveLoadVectorFirstFaultingTest.template @@ -32,7 +32,6 @@ namespace JIT.HardwareIntrinsics.Arm // Validates basic functionality works test.RunBasicScenario_Load(); test.RunBasicScenario_LoadFirstFaulting(); - test.RunBasicScenario_LoadFirstFaulting_IntermittentCalls(); // Validates calling via reflection works test.RunReflectionScenario_Load(); @@ -45,6 +44,9 @@ namespace JIT.HardwareIntrinsics.Arm // Validates passing an instance member of a struct works test.RunStructFldScenario(); + + // Validates using inside ConditionalSelect with zero falseValue + test.ConditionalSelect_ZeroOp(); } else { @@ -144,8 +146,14 @@ namespace JIT.HardwareIntrinsics.Arm private static {Op1BaseType}[] _data = new {Op1BaseType}[Op2ElementCount]; + private static {RetBaseType}[] _maskData = new {RetBaseType}[RetElementCount]; + private static {RetBaseType}[] _falseData = new {RetBaseType}[RetElementCount]; + private {Op1VectorType}<{RetBaseType}> _fld1; + private {Op1VectorType}<{Op1BaseType}> _mask; + private {Op1VectorType}<{Op1BaseType}> _falseFld; + private DataTable _dataTable; public Sve__{TestName}() @@ -154,6 +162,12 @@ namespace JIT.HardwareIntrinsics.Arm for (var i = 0; i < Op2ElementCount; i++) { _data[i] = {NextValueOp2}; } _dataTable = new DataTable(_data, new {RetBaseType}[RetElementCount], LargestVectorSize); + + for (var i = 0; i < RetElementCount; i++) { _maskData[i] = ({RetBaseType})({NextValueOp1}); } + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetVectorType}<{RetBaseType}>, byte>(ref _mask), ref Unsafe.As<{RetBaseType}, byte>(ref _maskData[0]), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); + + for (var i = 0; i < RetElementCount; i++) { _falseData[i] = ({RetBaseType})({NextValueOp2}); } + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetVectorType}<{RetBaseType}>, byte>(ref _falseFld), ref Unsafe.As<{RetBaseType}, byte>(ref _falseData[0]), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); } public bool IsSupported => {Isa}.IsSupported; @@ -194,33 +208,6 @@ namespace JIT.HardwareIntrinsics.Arm ValidateFirstFaultingResult(ref op2Ref, _dataTable.inBounded.Span.Length, _dataTable.outArrayPtr, faultResult.As()); } - [MethodImpl(MethodImplOptions.NoInlining)] - public static void ConsumeNothing() - { - } - - public void RunBasicScenario_LoadFirstFaulting_IntermittentCalls() - { - TestLibrary.TestFramework.BeginScenario(nameof(RunBasicScenario_LoadFirstFaulting_IntermittentCalls)); - - {Op1VectorType}<{Op1BaseType}> loadMask = Sve.CreateTrueMask{RetBaseType}(SveMaskPattern.All); - - ref var op2Ref = ref (_dataTable.inBounded.Span.GetPinnableReference()); - - Sve.SetFfr(Sve.CreateTrueMaskByte(SveMaskPattern.All)); - ConsumeNothing(); - var result = {Isa}.{Method}( - loadMask, - ({Op2BaseType}*)(Unsafe.AsPointer(ref op2Ref)) - ); - ConsumeNothing(); - var faultResult = Sve.GetFfr(); - ConsumeNothing(); - - Unsafe.Write(_dataTable.outArrayPtr, result); - ValidateFirstFaultingResult(ref op2Ref, _dataTable.inBounded.Span.Length, _dataTable.outArrayPtr, faultResult.As()); - } - public void RunReflectionScenario_Load() { TestLibrary.TestFramework.BeginScenario(nameof(RunReflectionScenario_Load)); @@ -270,6 +257,27 @@ namespace JIT.HardwareIntrinsics.Arm test.RunStructFldScenario(this); } + public void ConditionalSelect_ZeroOp() + { + TestLibrary.TestFramework.BeginScenario(nameof(ConditionalSelect_ZeroOp)); + ConditionalSelectScenario(_mask, ref _fld1, {Op1VectorType}<{RetBaseType}>.Zero); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_zero"); + ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}>.Zero, ref _fld1, {Op1VectorType}<{Op1BaseType}>.Zero); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_all"); + ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}>.AllBitsSet, ref _fld1, {Op1VectorType}<{Op1BaseType}>.Zero); + } + + [method: MethodImpl(MethodImplOptions.AggressiveInlining)] + private void ConditionalSelectScenario({RetVectorType}<{RetBaseType}> mask, ref {RetVectorType}<{Op2BaseType}> op1, {Op1VectorType}<{Op1BaseType}> falseOp) + { + var result = Sve.ConditionalSelect(mask, {Isa}.{Method}(Sve.CreateTrueMask{RetBaseType}(SveMaskPattern.All), ({Op2BaseType}*)Unsafe.AsPointer(ref op1)), falseOp); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateConditionalSelectResult(mask, op1, falseOp, _dataTable.outArrayPtr); + } + public void RunUnsupportedScenario() { TestLibrary.TestFramework.BeginScenario(nameof(RunUnsupportedScenario)); @@ -299,21 +307,44 @@ namespace JIT.HardwareIntrinsics.Arm private void ValidateResult({Op2BaseType}[] firstOp, {RetBaseType}[] result, [CallerMemberName] string method = "") { - bool succeeded = true; + var succeeded = Helpers.CheckLoadVectorBehavior(firstOp, result); - for (var i = 0; i < RetElementCount; i++) + if (!succeeded) { - if ({ValidateIterResult}) - { - succeeded = false; - break; - } + TestLibrary.TestFramework.LogInformation($"{nameof({Isa})}.{nameof({Isa}.{Method})}<{RetBaseType}>({Op1VectorType}<{Op1BaseType}>, {Op2BaseType}): {method} failed:"); + TestLibrary.TestFramework.LogInformation($" firstOp: ({string.Join(", ", firstOp)})"); + TestLibrary.TestFramework.LogInformation($" result: ({string.Join(", ", result)})"); + TestLibrary.TestFramework.LogInformation(string.Empty); + + Succeeded = false; } + } + + private void ValidateConditionalSelectResult({RetVectorType}<{RetBaseType}> maskOp, {Op1VectorType}<{Op1BaseType}> op1, {RetVectorType}<{RetBaseType}> falseOp, void* result, [CallerMemberName] string method = "") + { + {RetBaseType}[] maskArray = new {RetBaseType}[RetElementCount]; + {Op1BaseType}[] op1Array = new {Op1BaseType}[RetElementCount]; + {RetBaseType}[] falseValArray = new {RetBaseType}[RetElementCount]; + {RetBaseType}[] resultArray = new {RetBaseType}[RetElementCount]; + + Unsafe.WriteUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref maskArray[0]), maskOp); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref op1Array[0]), op1); + Unsafe.WriteUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref falseValArray[0]), falseOp); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref resultArray[0]), ref Unsafe.AsRef(result), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); + + ValidateConditionalSelectResult(maskArray, op1Array, falseValArray, resultArray, method); + } + + private void ValidateConditionalSelectResult({RetBaseType}[] maskOp, {Op1BaseType}[] firstOp, {RetBaseType}[] falseOp, {RetBaseType}[] result, [CallerMemberName] string method = "") + { + var succeeded = Helpers.CheckLoadVectorBehavior(maskOp, firstOp, result, falseOp); if (!succeeded) { - TestLibrary.TestFramework.LogInformation($"{nameof({Isa})}.{nameof({Isa}.{Method})}<{RetBaseType}>({Op1VectorType}<{Op1BaseType}>, {Op2BaseType}): {method} failed:"); + TestLibrary.TestFramework.LogInformation($"{nameof({Isa})}.{nameof({Isa}.{Method})}({Op1VectorType}<{Op1BaseType}>, {Op1VectorType}<{Op2BaseType}>): {method} failed:"); + TestLibrary.TestFramework.LogInformation($" maskOp: ({string.Join(", ", maskOp)})"); TestLibrary.TestFramework.LogInformation($" firstOp: ({string.Join(", ", firstOp)})"); + TestLibrary.TestFramework.LogInformation($" falseOp: ({string.Join(", ", falseOp)})"); TestLibrary.TestFramework.LogInformation($" result: ({string.Join(", ", result)})"); TestLibrary.TestFramework.LogInformation(string.Empty); @@ -334,48 +365,7 @@ namespace JIT.HardwareIntrinsics.Arm private void ValidateFirstFaultingResult({Op2BaseType}[] firstOp, {RetBaseType}[] result, Vector<{RetBaseType}> faultResult, [CallerMemberName] string method = "") { - bool succeeded = true; - - bool hitFault = false; - - for (var i = 0; i < RetElementCount; i++) - { - if (hitFault) - { - if (faultResult[i] != 0) - { - succeeded = false; - break; - } - - if (result[i] != 0) - { - succeeded = false; - break; - } - } - else - { - if (faultResult[i] == 0) - { - // There has to be a valid value for the first element, so check it. - if (i == 0) - { - succeeded = false; - break; - } - hitFault = true; - } - else - { - if ({ValidateIterResult}) - { - succeeded = false; - break; - } - } - } - } + var succeeded = Helpers.CheckLoadVectorFirstFaultingBehavior(firstOp, result, faultResult); if (!succeeded) { From 7f3bb3c9c8c4b7e9e4b1d4d5e935f6e0fcffefcd Mon Sep 17 00:00:00 2001 From: TIHan Date: Mon, 8 Jul 2024 14:02:15 -0700 Subject: [PATCH 08/83] Added first-faulting behavior tests for GatherVectorFirstFaulting --- src/coreclr/jit/gentree.cpp | 2 +- .../GenerateHWIntrinsicTests_Arm.cs | 24 +- .../HardwareIntrinsics/Arm/Shared/Helpers.cs | 24 +- ...eGatherVectorFirstFaultingIndices.template | 632 ++++++++++++++++++ 4 files changed, 665 insertions(+), 17 deletions(-) create mode 100644 src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorFirstFaultingIndices.template diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 1fcc4215a208d..d4a81e029606d 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -27588,7 +27588,7 @@ bool GenTreeHWIntrinsic::OperIsMemoryLoad(GenTree** pAddr) const NI_Sve_GatherVectorInt32WithByteOffsetsSignExtend, NI_Sve_GatherVectorSByteSignExtend, NI_Sve_GatherVectorUInt16WithByteOffsetsZeroExtend, NI_Sve_GatherVectorUInt16ZeroExtend, NI_Sve_GatherVectorUInt32WithByteOffsetsZeroExtend, NI_Sve_GatherVectorUInt32ZeroExtend)); - assert(varTypeIsI(addr) || (varTypeIsSIMD(addr) && ((intrinsicId >= NI_Sve_GatherVector) && + assert(varTypeIsI(addr) || (varTypeIsSIMD(addr) && ((intrinsicId >= NI_Sve_GatherPrefetch16Bit) && (intrinsicId <= NI_Sve_GatherVectorUInt32ZeroExtend)))); #else assert(varTypeIsI(addr)); diff --git a/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs b/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs index c844ef1647fea..ebc5e9bafae73 100644 --- a/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs +++ b/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs @@ -3430,18 +3430,18 @@ ("SveGatherVectorIndices.template", new Dictionary {["TestName"] = "Sve_GatherVectorUInt32ZeroExtend_Indices_ulong_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorUInt32ZeroExtend", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2BaseType"] = "UInt32", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt64", ["ExtendedElementType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt64()"}), ("SveGatherVectorIndices.template", new Dictionary {["TestName"] = "Sve_GatherVectorUInt32ZeroExtend_Indices_uint_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorUInt32ZeroExtend", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2BaseType"] = "UInt32", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt32", ["ExtendedElementType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt32()"}), - ("SveGatherVectorIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_float_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2BaseType"] = "Single", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int32", ["ExtendedElementType"] = "Single", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskSingle()", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt32()"}), - ("SveGatherVectorIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_int_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2BaseType"] = "Int32", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int32", ["ExtendedElementType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt32()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt32()"}), - ("SveGatherVectorIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_uint_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2BaseType"] = "UInt32", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int32", ["ExtendedElementType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt32()"}), - ("SveGatherVectorIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_float_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2BaseType"] = "Single", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt32", ["ExtendedElementType"] = "Single", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskSingle()", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt32()"}), - ("SveGatherVectorIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_int_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2BaseType"] = "Int32", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt32", ["ExtendedElementType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt32()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt32()"}), - ("SveGatherVectorIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_uint_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2BaseType"] = "UInt32", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt32", ["ExtendedElementType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt32()"}), - ("SveGatherVectorIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_double_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2BaseType"] = "Double", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int64", ["ExtendedElementType"] = "Double", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskDouble()", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt64()"}), - ("SveGatherVectorIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_long_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2BaseType"] = "Int64", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int64", ["ExtendedElementType"] = "Int64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt64()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt64()"}), - ("SveGatherVectorIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_ulong_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2BaseType"] = "UInt64", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int64", ["ExtendedElementType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt64()"}), - ("SveGatherVectorIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_double_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2BaseType"] = "Double", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt64", ["ExtendedElementType"] = "Double", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskDouble()", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt64()"}), - ("SveGatherVectorIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_long_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2BaseType"] = "Int64", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt64", ["ExtendedElementType"] = "Int64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt64()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt64()"}), - ("SveGatherVectorIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_ulong_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2BaseType"] = "UInt64", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt64", ["ExtendedElementType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt64()"}), + ("SveGatherVectorFirstFaultingIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_float_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2BaseType"] = "Single", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int32", ["ExtendedElementType"] = "Single", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskSingle()", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt32()"}), + ("SveGatherVectorFirstFaultingIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_int_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2BaseType"] = "Int32", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int32", ["ExtendedElementType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt32()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt32()"}), + ("SveGatherVectorFirstFaultingIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_uint_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2BaseType"] = "UInt32", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int32", ["ExtendedElementType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt32()"}), + ("SveGatherVectorFirstFaultingIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_float_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2BaseType"] = "Single", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt32", ["ExtendedElementType"] = "Single", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskSingle()", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt32()"}), + ("SveGatherVectorFirstFaultingIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_int_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2BaseType"] = "Int32", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt32", ["ExtendedElementType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt32()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt32()"}), + ("SveGatherVectorFirstFaultingIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_uint_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2BaseType"] = "UInt32", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt32", ["ExtendedElementType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt32()"}), + ("SveGatherVectorFirstFaultingIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_double_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2BaseType"] = "Double", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int64", ["ExtendedElementType"] = "Double", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskDouble()", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt64()"}), + ("SveGatherVectorFirstFaultingIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_long_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2BaseType"] = "Int64", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int64", ["ExtendedElementType"] = "Int64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt64()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt64()"}), + ("SveGatherVectorFirstFaultingIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_ulong_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2BaseType"] = "UInt64", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int64", ["ExtendedElementType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt64()"}), + ("SveGatherVectorFirstFaultingIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_double_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2BaseType"] = "Double", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt64", ["ExtendedElementType"] = "Double", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskDouble()", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt64()"}), + ("SveGatherVectorFirstFaultingIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_long_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2BaseType"] = "Int64", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt64", ["ExtendedElementType"] = "Int64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt64()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt64()"}), + ("SveGatherVectorFirstFaultingIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_ulong_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2BaseType"] = "UInt64", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt64", ["ExtendedElementType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt64()"}), ("SveGatherVectorByteOffsets.template",new Dictionary {["TestName"] = "Sve_GatherVectorWithByteOffsets_float_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorWithByteOffsets", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2BaseType"] = "Single", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int32", ["ExtendedElementType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskSingle()", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt32()"}), ("SveGatherVectorByteOffsets.template",new Dictionary {["TestName"] = "Sve_GatherVectorWithByteOffsets_int_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorWithByteOffsets", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2BaseType"] = "Int32", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int32", ["ExtendedElementType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt32()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt32()"}), diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs index efe698688cdad..23a0c7fbd1d15 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs @@ -7729,6 +7729,14 @@ public static bool CheckLoadVectorBehavior(T[] maskOp, T[] firstOp, T[] resul return CheckLoadVectorBehaviorCore(firstOp, result, (i, loadResult) => ConditionalSelectResult(maskOp[i], loadResult, falseOp[i])); } + private static T GetGatherVectorResultByIndex(int index, T[] firstOp, ExtendedElementT[] secondOp, Index[] thirdOp) + where T : INumberBase + where ExtendedElementT : INumberBase + where Index : IBinaryInteger + { + return (firstOp[index] == T.Zero) ? T.Zero : T.CreateTruncating(secondOp[int.CreateChecked(thirdOp[index])]); + } + private static bool CheckGatherVectorBehaviorCore(T[] firstOp, ExtendedElementT[] secondOp, Index[] thirdOp, T[] result, Func map) where T : INumberBase where ExtendedElementT : INumberBase @@ -7736,7 +7744,7 @@ private static bool CheckGatherVectorBehaviorCore(T[ { for (var i = 0; i < firstOp.Length; i++) { - T gatherResult = (firstOp[i] == T.Zero) ? T.Zero : T.CreateTruncating(secondOp[int.CreateChecked(thirdOp[i])]); + T gatherResult = GetGatherVectorResultByIndex(i, firstOp, secondOp, thirdOp); gatherResult = map(i, gatherResult); if (result[i] != gatherResult) { @@ -7762,11 +7770,11 @@ public static bool CheckGatherVectorBehavior(T[] mas return CheckGatherVectorBehaviorCore(firstOp, secondOp, thirdOp, result, (i, gatherResult) => ConditionalSelectResult(maskOp[i], gatherResult, falseOp[i])); } - private static bool CheckFirstFaultingBehaviorCore(T[] firstOp, T[] result, Vector faultResult, Func checkIter) where T : INumberBase + private static bool CheckFirstFaultingBehaviorCore(T[] result, Vector faultResult, Func checkIter) where T : INumberBase { bool hitFault = false; - for (var i = 0; i < firstOp.Length; i++) + for (var i = 0; i < result.Length; i++) { if (hitFault) { @@ -7806,7 +7814,15 @@ private static bool CheckFirstFaultingBehaviorCore(T[] firstOp, T[] result, V public static bool CheckLoadVectorFirstFaultingBehavior(T[] firstOp, T[] result, Vector faultResult) where T : INumberBase { - return CheckFirstFaultingBehaviorCore(firstOp, result, faultResult, i => firstOp[i] == result[i]); + return CheckFirstFaultingBehaviorCore(result, faultResult, i => firstOp[i] == result[i]); + } + + public static bool CheckGatherVectorFirstFaultingBehavior(T[] firstOp, ExtendedElementT[] secondOp, Index[] thirdOp, T[] result, Vector faultResult) + where T : INumberBase + where ExtendedElementT : INumberBase + where Index : IBinaryInteger + { + return CheckFirstFaultingBehaviorCore(result, faultResult, i => GetGatherVectorResultByIndex(i, firstOp, secondOp, thirdOp) == result[i]); } } } diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorFirstFaultingIndices.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorFirstFaultingIndices.template new file mode 100644 index 0000000000000..2d1a272155663 --- /dev/null +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorFirstFaultingIndices.template @@ -0,0 +1,632 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +/****************************************************************************** + * This file is auto-generated from a template file by the GenerateTests.csx * + * script in tests\src\JIT\HardwareIntrinsics\Arm\Shared. In order to make * + * changes, please update the corresponding template and run according to the * + * directions listed in the file. * + ******************************************************************************/ + +using System; +using System.Buffers; +using System.Numerics; +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.Arm; +using Xunit; + +namespace JIT.HardwareIntrinsics.Arm +{ + public static partial class Program + { + [Fact] + public static void {TestName}() + { + var test = new SveGatherVectorIndices__{TestName}(); + + if (test.IsSupported) + { + // Validates basic functionality works, using Unsafe.Read + test.RunBasicScenario_UnsafeRead(); + + // Validates basic functionality works + test.RunBasicScenario_Load(); + test.RunBasicScenario_LoadFirstFaulting(); + + // Validates fully masked out load works. + test.RunBasicScenario_FalseMask(); + + // Validates fully masked out load with invalid address works. + test.RunBasicScenario_NonFaulting(); + + // Validates calling via reflection works, using Unsafe.Read + test.RunReflectionScenario_UnsafeRead(); + + // Validates passing a local works, using Unsafe.Read + test.RunLclVarScenario_UnsafeRead(); + + // Validates passing an instance member of a class works + test.RunClassFldScenario(); + + // Validates passing the field of a local struct works + test.RunStructLclFldScenario(); + + // Validates passing an instance member of a struct works + test.RunStructFldScenario(); + + // Validates using inside ConditionalSelect with value falseValue + // Currently, using this operation in ConditionalSelect() gives incorrect result + // when falseReg == targetReg because this instruction uses Pg/Z to update the targetReg + // instead of Pg/M to merge it. As such, the value of falseReg is lost. Ideally, such + // instructions should be marked similar to RMW (a different flag name) to make sure that + // we do not assign falseReg/targetReg same. Then, we would do something like this: + // + // ldnf1sh target, pg/z, [x0] + // sel mask, target, target, falseReg + // + // This needs more careful thinking, so disabling it for now. + // test.ConditionalSelect_FalseOp(); + + // Validates using inside ConditionalSelect with zero falseValue + test.ConditionalSelect_ZeroOp(); + } + else + { + // Validates we throw on unsupported hardware + test.RunUnsupportedScenario(); + } + + if (!test.Succeeded) + { + throw new Exception("One or more scenarios did not complete as expected."); + } + } + } + + public sealed unsafe class SveGatherVectorIndices__{TestName} + { + private struct DataTable + { + private byte[] inArray1; + private byte[] inArray2; + private byte[] inArray3; + private byte[] outArray; + + private GCHandle inHandle1; + private GCHandle inHandle2; + private GCHandle inHandle3; + private GCHandle outHandle; + + private ulong alignment; + + public DataTable({Op1BaseType}[] inArray1, {Op2BaseType}[] inArray2, {Op3BaseType}[] inArray3, {RetBaseType}[] outArray, int alignment) + { + int sizeOfinArray1 = inArray1.Length * Unsafe.SizeOf<{Op1BaseType}>(); + int sizeOfinArray2 = inArray2.Length * Unsafe.SizeOf<{Op2BaseType}>(); + int sizeOfinArray3 = inArray3.Length * Unsafe.SizeOf<{Op3BaseType}>(); + int sizeOfinBounded = new Random().Next(Unsafe.SizeOf<{Op2BaseType}>(), sizeOfinArray2 - 1); + int sizeOfoutArray = outArray.Length * Unsafe.SizeOf<{RetBaseType}>(); + + if ((alignment != 64 && alignment != 16 && alignment != 8) || (alignment * 2) < sizeOfinArray1 || (alignment * 2) < sizeOfinArray3 || (alignment * 2) < sizeOfoutArray) + { + throw new ArgumentException($"Invalid value of alignment: {alignment}, sizeOfinArray1: {sizeOfinArray1}, sizeOfinArray2: {sizeOfinArray2}, sizeOfinArray3: {sizeOfinArray3}, sizeOfoutArray: {sizeOfoutArray}"); + } + + this.inArray1 = new byte[alignment * 2]; + this.inArray2 = new byte[alignment + sizeOfinArray2]; + this.inArray3 = new byte[alignment * 2]; + this.outArray = new byte[alignment * 2]; + + this.inHandle1 = GCHandle.Alloc(this.inArray1, GCHandleType.Pinned); + this.inHandle2 = GCHandle.Alloc(this.inArray2, GCHandleType.Pinned); + this.inHandle3 = GCHandle.Alloc(this.inArray3, GCHandleType.Pinned); + this.outHandle = GCHandle.Alloc(this.outArray, GCHandleType.Pinned); + + this.alignment = (ulong)alignment; + + this.inBounded = BoundedMemory.Allocate(sizeOfinBounded, PoisonPagePlacement.After); + + Unsafe.CopyBlockUnaligned(ref Unsafe.AsRef(inArray1Ptr), ref Unsafe.As<{Op1BaseType}, byte>(ref inArray1[0]), (uint)sizeOfinArray1); + Unsafe.CopyBlockUnaligned(ref Unsafe.AsRef(inArray2Ptr), ref Unsafe.As<{Op2BaseType}, byte>(ref inArray2[0]), (uint)sizeOfinArray2); + Unsafe.CopyBlockUnaligned(ref Unsafe.AsRef(inArray3Ptr), ref Unsafe.As<{Op3BaseType}, byte>(ref inArray3[0]), (uint)sizeOfinArray3); + Unsafe.CopyBlockUnaligned(ref inBounded.Span.GetPinnableReference(), ref Unsafe.As<{Op2BaseType}, byte>(ref inArray2[0]), (uint)sizeOfinBounded); + } + + public BoundedMemory inBounded; + + public void* inArray1Ptr => Align((byte*)(inHandle1.AddrOfPinnedObject().ToPointer()), alignment); + public void* inArray2Ptr => Align((byte*)(inHandle2.AddrOfPinnedObject().ToPointer()), alignment); + public void* inArray3Ptr => Align((byte*)(inHandle3.AddrOfPinnedObject().ToPointer()), alignment); + public void* outArrayPtr => Align((byte*)(outHandle.AddrOfPinnedObject().ToPointer()), alignment); + + public void Dispose() + { + inHandle1.Free(); + inHandle2.Free(); + inHandle3.Free(); + outHandle.Free(); + } + + private static unsafe void* Align(byte* buffer, ulong expectedAlignment) + { + return (void*)(((ulong)buffer + expectedAlignment - 1) & ~(expectedAlignment - 1)); + } + } + + private struct TestStruct + { + public {Op1VectorType}<{Op1BaseType}> _fld1; + public {Op2BaseType}* _fld2; + public {Op3VectorType}<{Op3BaseType}> _fld3; + + public static TestStruct Create(DataTable _dataTable) + { + var testStruct = new TestStruct(); + + for (var i = 0; i < Op1ElementCount; i++) { _data1[i] = {NextValueOp1}; } + for (var i = 0; i < Op2ElementCount; i++) { _data2[i] = {NextValueOp2}; } + + // Ensure all values of _data3 fit within the number of _data2 elements + for (var i = 0; i < Op3ElementCount; i++) { _data3[i] = {NextValueOp3} % ({Op3BaseType})Op2ElementCount; } + + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1VectorType}<{Op1BaseType}>, byte>(ref testStruct._fld1), ref Unsafe.As<{Op1BaseType}, byte>(ref _data1[0]), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op3VectorType}<{Op3BaseType}>, byte>(ref testStruct._fld3), ref Unsafe.As<{Op3BaseType}, byte>(ref _data3[0]), (uint)Unsafe.SizeOf<{Op3VectorType}<{Op3BaseType}>>()); + + testStruct._fld2 = ({Op2BaseType}*)_dataTable.inArray2Ptr; + + return testStruct; + } + + public void RunStructFldScenario(SveGatherVectorIndices__{TestName} testClass) + { + var result = {Isa}.{Method}(_fld1, _fld2, _fld3); + + Unsafe.Write(testClass._dataTable.outArrayPtr, result); + testClass.ValidateResult(_fld1, _fld2, _fld3, testClass._dataTable.outArrayPtr); + } + } + + private static readonly int LargestVectorSize = {LargestVectorSize}; + + // A large enough buffer to hold many values. Op3 will index into Op2. + private static readonly int Op2ElementCount = 1024; + + private static readonly int Op1ElementCount = Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>() / sizeof({Op1BaseType}); + private static readonly int Op3ElementCount = Unsafe.SizeOf<{Op3VectorType}<{Op3BaseType}>>() / sizeof({Op3BaseType}); + private static readonly int RetElementCount = Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>() / sizeof({RetBaseType}); + + private static {Op1BaseType}[] _data1 = new {Op1BaseType}[Op1ElementCount]; + private static {Op2BaseType}[] _data2 = new {Op2BaseType}[Op2ElementCount]; + private static {Op3BaseType}[] _data3 = new {Op3BaseType}[Op3ElementCount]; + + private static {RetBaseType}[] _maskData = new {RetBaseType}[RetElementCount]; + private static {RetBaseType}[] _falseData = new {RetBaseType}[RetElementCount]; + + private {Op1VectorType}<{Op1BaseType}> _fld1; + private {Op2BaseType}* _fld2; + private {Op3VectorType}<{Op3BaseType}> _fld3; + + private {Op1VectorType}<{Op1BaseType}> _mask; + private {Op1VectorType}<{Op1BaseType}> _falseFld; + + private DataTable _dataTable; + + public SveGatherVectorIndices__{TestName}() + { + Succeeded = true; + + for (var i = 0; i < Op1ElementCount; i++) { _data1[i] = {NextValueOp1}; } + for (var i = 0; i < Op2ElementCount; i++) { _data2[i] = {NextValueOp2}; } + + // Ensure all values of _data3 fit within the number of _data2 elements + for (var i = 0; i < Op3ElementCount; i++) { _data3[i] = {NextValueOp3} % ({Op3BaseType})Op2ElementCount; } + + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1VectorType}<{Op1BaseType}>, byte>(ref _fld1), ref Unsafe.As<{Op1BaseType}, byte>(ref _data1[0]), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op3VectorType}<{Op3BaseType}>, byte>(ref _fld3), ref Unsafe.As<{Op3BaseType}, byte>(ref _data3[0]), (uint)Unsafe.SizeOf<{Op3VectorType}<{Op3BaseType}>>()); + + for (var i = 0; i < RetElementCount; i++) { _maskData[i] = ({RetBaseType})({NextValueOp1}); } + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetVectorType}<{RetBaseType}>, byte>(ref _mask), ref Unsafe.As<{RetBaseType}, byte>(ref _maskData[0]), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); + + for (var i = 0; i < RetElementCount; i++) { _falseData[i] = ({RetBaseType})({NextValueOp2}); } + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetVectorType}<{RetBaseType}>, byte>(ref _falseFld), ref Unsafe.As<{RetBaseType}, byte>(ref _falseData[0]), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); + + _dataTable = new DataTable(_data1, _data2, _data3, new {RetBaseType}[RetElementCount], LargestVectorSize); + + _fld2 = ({Op2BaseType}*)_dataTable.inArray2Ptr; + } + + public bool IsSupported => {Isa}.IsSupported; + + public bool Succeeded { get; set; } + + public void RunBasicScenario_UnsafeRead() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunBasicScenario_UnsafeRead)); + + var result = {Isa}.{Method}( + Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inArray1Ptr), + ({Op2BaseType}*)_dataTable.inArray2Ptr, + Unsafe.Read<{Op3VectorType}<{Op3BaseType}>>(_dataTable.inArray3Ptr) + ); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateResult(_dataTable.inArray1Ptr, _dataTable.inArray2Ptr, _dataTable.inArray3Ptr, _dataTable.outArrayPtr); + } + + public void RunBasicScenario_Load() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunBasicScenario_Load)); + + {Op1VectorType}<{Op1BaseType}> loadMask1 = Sve.CreateTrueMask{Op1BaseType}(SveMaskPattern.All); + {Op3VectorType}<{Op3BaseType}> loadMask3 = Sve.CreateTrueMask{Op3BaseType}(SveMaskPattern.All); + + var result = {Isa}.{Method}( + {LoadIsa}.Load{Op1VectorType}(loadMask1, ({Op1BaseType}*)(_dataTable.inArray1Ptr)), + ({Op2BaseType}*)_dataTable.inArray2Ptr, + {LoadIsa}.Load{Op3VectorType}(loadMask3, ({Op3BaseType}*)(_dataTable.inArray3Ptr)) + ); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateResult(_dataTable.inArray1Ptr, _dataTable.inArray2Ptr, _dataTable.inArray3Ptr, _dataTable.outArrayPtr); + } + + public void RunBasicScenario_LoadFirstFaulting() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunBasicScenario_LoadFirstFaulting)); + + {Op1VectorType}<{Op1BaseType}> loadMask1 = Sve.CreateTrueMask{Op1BaseType}(SveMaskPattern.All); + {Op3VectorType}<{Op3BaseType}> loadMask3 = Sve.CreateTrueMask{Op3BaseType}(SveMaskPattern.All); + + var op1 = {LoadIsa}.Load{Op1VectorType}(loadMask1, ({Op1BaseType}*)(_dataTable.inArray1Ptr)); + ref var op2Ref = ref (_dataTable.inBounded.Span.GetPinnableReference()); + var op3 = {LoadIsa}.Load{Op3VectorType}(loadMask3, ({Op3BaseType}*)(_dataTable.inArray3Ptr)); + + // When testing first-faulting behavior, we need to make sure we can get the first element. + // So set the first active element of the index vector to 0. + for (var i = 0; i < Vector<{Op3BaseType}>.Count; i++) + { + if (loadMask3[i] != 0) + { + op3 = op3.WithElement<{Op3BaseType}>(i, 0); + break; + } + } + + Sve.SetFfr(Sve.CreateTrueMaskByte(SveMaskPattern.All)); + var result = {Isa}.{Method}( + op1, + ({Op2BaseType}*)(Unsafe.AsPointer(ref op2Ref)), + op3 + ); + var faultResult = Sve.GetFfr(); + + ref var op3Ref = ref op3; + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateFirstFaultingResult(_dataTable.inArray1Ptr, ref op2Ref, _dataTable.inBounded.Span.Length, Unsafe.AsPointer(ref op3Ref), _dataTable.outArrayPtr, faultResult.As()); + } + + public void RunBasicScenario_FalseMask() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunBasicScenario_FalseMask)); + + {Op1VectorType}<{Op1BaseType}> falseMask = Sve.CreateFalseMask{Op1BaseType}(); + + var result = {Isa}.{Method}( + falseMask, + ({Op2BaseType}*)_dataTable.inArray2Ptr, + Unsafe.Read<{Op3VectorType}<{Op3BaseType}>>(_dataTable.inArray3Ptr) + ); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateZeroResult(_dataTable.inArray1Ptr, _dataTable.inArray2Ptr, _dataTable.inArray3Ptr, _dataTable.outArrayPtr); + } + + public void RunBasicScenario_NonFaulting() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunBasicScenario_NonFaulting)); + + {Op1VectorType}<{Op1BaseType}> falseMask = Sve.CreateFalseMask{Op1BaseType}(); + + try + { + var result = {Isa}.{Method}( + falseMask, + default, + Unsafe.Read<{Op3VectorType}<{Op3BaseType}>>(_dataTable.inArray3Ptr) + ); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateZeroResult(_dataTable.inArray1Ptr, _dataTable.inArray2Ptr, _dataTable.inArray3Ptr, _dataTable.outArrayPtr); + } + catch + { + Succeeded = false; + } + } + + public void RunReflectionScenario_UnsafeRead() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunReflectionScenario_UnsafeRead)); + + var result = typeof({Isa}).GetMethod(nameof({Isa}.{Method}), new Type[] { typeof({Op1VectorType}<{Op1BaseType}>), typeof({Op2BaseType}*), typeof({Op3VectorType}<{Op3BaseType}>) }) + .Invoke(null, new object[] { + Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inArray1Ptr), + Pointer.Box(_dataTable.inArray2Ptr, typeof({Op2BaseType}*)), + Unsafe.Read<{Op3VectorType}<{Op3BaseType}>>(_dataTable.inArray3Ptr) + }); + + Unsafe.Write(_dataTable.outArrayPtr, ({RetVectorType}<{RetBaseType}>)(result)); + ValidateResult(_dataTable.inArray1Ptr, _dataTable.inArray2Ptr, _dataTable.inArray3Ptr, _dataTable.outArrayPtr); + } + + public void RunLclVarScenario_UnsafeRead() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunLclVarScenario_UnsafeRead)); + + var op1 = Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inArray1Ptr); + var op2 = ({Op2BaseType}*)_dataTable.inArray2Ptr; + var op3 = Unsafe.Read<{Op3VectorType}<{Op3BaseType}>>(_dataTable.inArray3Ptr); + var result = {Isa}.{Method}(op1, op2, op3); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateResult(op1, op2, op3, _dataTable.outArrayPtr); + } + + public void RunClassFldScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunClassFldScenario)); + + var result = {Isa}.{Method}(_fld1, _fld2, _fld3); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateResult(_fld1, _fld2, _fld3, _dataTable.outArrayPtr); + } + + public void RunStructLclFldScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunStructLclFldScenario)); + + var test = TestStruct.Create(_dataTable); + var result = {Isa}.{Method}(test._fld1, test._fld2, test._fld3); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateResult(test._fld1, test._fld2, test._fld3, _dataTable.outArrayPtr); + } + + public void RunStructFldScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunStructFldScenario)); + + var test = TestStruct.Create(_dataTable); + test.RunStructFldScenario(this); + } + + + public void ConditionalSelect_FalseOp() + { + TestLibrary.TestFramework.BeginScenario(nameof(ConditionalSelect_FalseOp)); + ConditionalSelectScenario(_mask, _fld1, _fld2, _fld3, _falseFld); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_zero"); + ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}>.Zero, _fld1, _fld2, _fld3, _falseFld); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_all"); + ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}>.AllBitsSet, _fld1, _fld2, _fld3, _falseFld); + } + + public void ConditionalSelect_ZeroOp() + { + TestLibrary.TestFramework.BeginScenario(nameof(ConditionalSelect_ZeroOp)); + ConditionalSelectScenario(_mask, _fld1, _fld2, _fld3, {Op1VectorType}<{RetBaseType}>.Zero); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_zero"); + ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}>.Zero, _fld1, _fld2, _fld3, {Op1VectorType}<{Op1BaseType}>.Zero); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_all"); + ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}>.AllBitsSet, _fld1, _fld2, _fld3, {Op1VectorType}<{Op1BaseType}>.Zero); + } + + [method: MethodImpl(MethodImplOptions.AggressiveInlining)] + private void ConditionalSelectScenario({RetVectorType}<{RetBaseType}> mask, {Op1VectorType}<{Op1BaseType}> op1, {Op2BaseType}* op2, {Op3VectorType}<{Op3BaseType}> op3, {Op1VectorType}<{Op1BaseType}> falseOp) + { + var result = Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2, op3), falseOp); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateConditionalSelectResult(mask, op1, op2, op3, falseOp, _dataTable.outArrayPtr); + } + + public void RunUnsupportedScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunUnsupportedScenario)); + + Succeeded = false; + + try + { + RunBasicScenario_Load(); + } + catch (PlatformNotSupportedException) + { + Succeeded = true; + } + } + + private void ValidateResult({Op1VectorType}<{Op1BaseType}> op1, {Op2BaseType}* op2, {Op3VectorType}<{Op3BaseType}> op3, void* result, [CallerMemberName] string method = "") + { + {Op1BaseType}[] inArray1 = new {Op1BaseType}[Op1ElementCount]; + {Op2BaseType}[] inArray2 = new {Op2BaseType}[Op2ElementCount]; + {Op3BaseType}[] inArray3 = new {Op3BaseType}[Op3ElementCount]; + {RetBaseType}[] outArray = new {RetBaseType}[RetElementCount]; + + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref inArray1[0]), op1); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op2BaseType}, byte>(ref inArray2[0]), ref Unsafe.AsRef(op2), (uint)(Unsafe.SizeOf<{Op2BaseType}>() * Op2ElementCount)); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op3BaseType}, byte>(ref inArray3[0]), op3); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref outArray[0]), ref Unsafe.AsRef(result), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); + + ValidateResult(inArray1, inArray2, inArray3, outArray, method); + } + + private void ValidateResult(void* firstOp, void* secondOp, void* thirdOp, void* result, [CallerMemberName] string method = "") + { + {Op1BaseType}[] inArray1 = new {Op1BaseType}[Op1ElementCount]; + {Op2BaseType}[] inArray2 = new {Op2BaseType}[Op2ElementCount]; + {Op3BaseType}[] inArray3 = new {Op3BaseType}[Op3ElementCount]; + {RetBaseType}[] outArray = new {RetBaseType}[RetElementCount]; + + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref inArray1[0]), ref Unsafe.AsRef(firstOp), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op2BaseType}, byte>(ref inArray2[0]), ref Unsafe.AsRef(secondOp), (uint)(Unsafe.SizeOf<{Op2BaseType}>() * Op2ElementCount)); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op3BaseType}, byte>(ref inArray3[0]), ref Unsafe.AsRef(thirdOp), (uint)Unsafe.SizeOf<{Op3VectorType}<{Op3BaseType}>>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref outArray[0]), ref Unsafe.AsRef(result), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); + + ValidateResult(inArray1, inArray2, inArray3, outArray, method); + } + + private void ValidateResult({Op1BaseType}[] firstOp, {Op2BaseType}[] secondOp, {Op3BaseType}[] thirdOp, {RetBaseType}[] result, [CallerMemberName] string method = "") + { + var succeeded = Helpers.CheckGatherVectorBehavior<{RetBaseType}, {ExtendedElementType}, {Op3BaseType}>(firstOp, secondOp, thirdOp, result); + + if (!succeeded) + { + TestLibrary.TestFramework.LogInformation($"{nameof({Isa})}.{nameof({Isa}.{Method})}<{RetBaseType}>({Op1BaseType}, {Op2BaseType}, {Op3BaseType}): {method} failed:"); + TestLibrary.TestFramework.LogInformation($" firstOp: ({string.Join(", ", firstOp)})"); + TestLibrary.TestFramework.LogInformation($" secondOp: ({string.Join(", ", secondOp)})"); + TestLibrary.TestFramework.LogInformation($" thirdOp: ({string.Join(", ", thirdOp)})"); + TestLibrary.TestFramework.LogInformation($" result: ({string.Join(", ", result)})"); + TestLibrary.TestFramework.LogInformation(string.Empty); + + Succeeded = false; + } + } + + private void ValidateZeroResult({Op1VectorType}<{Op1BaseType}> op1, {Op2BaseType}* op2, {Op3VectorType}<{Op3BaseType}> op3, void* result, [CallerMemberName] string method = "") + { + {Op1BaseType}[] inArray1 = new {Op1BaseType}[Op1ElementCount]; + {Op2BaseType}[] inArray2 = new {Op2BaseType}[Op2ElementCount]; + {Op3BaseType}[] inArray3 = new {Op3BaseType}[Op3ElementCount]; + {RetBaseType}[] outArray = new {RetBaseType}[RetElementCount]; + + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref inArray1[0]), op1); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op2BaseType}, byte>(ref inArray2[0]), ref Unsafe.AsRef(op2), (uint)(Unsafe.SizeOf<{Op2BaseType}>() * Op2ElementCount)); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op3BaseType}, byte>(ref inArray3[0]), op3); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref outArray[0]), ref Unsafe.AsRef(result), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); + + ValidateZeroResult(inArray1, inArray2, inArray3, outArray, method); + } + + private void ValidateZeroResult(void* firstOp, void* secondOp, void* thirdOp, void* result, [CallerMemberName] string method = "") + { + {Op1BaseType}[] inArray1 = new {Op1BaseType}[Op1ElementCount]; + {Op2BaseType}[] inArray2 = new {Op2BaseType}[Op2ElementCount]; + {Op3BaseType}[] inArray3 = new {Op3BaseType}[Op3ElementCount]; + {RetBaseType}[] outArray = new {RetBaseType}[RetElementCount]; + + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref inArray1[0]), ref Unsafe.AsRef(firstOp), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op2BaseType}, byte>(ref inArray2[0]), ref Unsafe.AsRef(secondOp), (uint)(Unsafe.SizeOf<{Op2BaseType}>() * Op2ElementCount)); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op3BaseType}, byte>(ref inArray3[0]), ref Unsafe.AsRef(thirdOp), (uint)Unsafe.SizeOf<{Op3VectorType}<{Op3BaseType}>>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref outArray[0]), ref Unsafe.AsRef(result), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); + + ValidateZeroResult(inArray1, inArray2, inArray3, outArray, method); + } + + private void ValidateZeroResult({Op1BaseType}[] firstOp, {Op2BaseType}[] secondOp, {Op3BaseType}[] thirdOp, {RetBaseType}[] result, [CallerMemberName] string method = "") + { + bool succeeded = true; + + for (var i = 0; i < RetElementCount; i++) + { + if (result[i] != 0) + { + succeeded = false; + break; + } + } + + if (!succeeded) + { + TestLibrary.TestFramework.LogInformation($"{nameof({Isa})}.{nameof({Isa}.{Method})}<{RetBaseType}>({Op1BaseType}, {Op2BaseType}, {Op3BaseType}): {method} failed:"); + TestLibrary.TestFramework.LogInformation($" firstOp: ({string.Join(", ", firstOp)})"); + TestLibrary.TestFramework.LogInformation($" secondOp: ({string.Join(", ", secondOp)})"); + TestLibrary.TestFramework.LogInformation($" thirdOp: ({string.Join(", ", thirdOp)})"); + TestLibrary.TestFramework.LogInformation($" result: ({string.Join(", ", result)})"); + TestLibrary.TestFramework.LogInformation(string.Empty); + + Succeeded = false; + } + } + + private void ValidateConditionalSelectResult({RetVectorType}<{RetBaseType}> maskOp, {Op1VectorType}<{Op1BaseType}> op1, {Op2BaseType}* op2, {Op3VectorType}<{Op3BaseType}> op3, {RetVectorType}<{RetBaseType}> falseOp, void* result, [CallerMemberName] string method = "") + { + {RetBaseType}[] maskArray = new {RetBaseType}[RetElementCount]; + {Op1BaseType}[] op1Array = new {Op1BaseType}[Op1ElementCount]; + {Op2BaseType}[] op2Array = new {Op2BaseType}[Op2ElementCount]; + {Op3BaseType}[] op3Array = new {Op3BaseType}[Op3ElementCount]; + {RetBaseType}[] falseValArray = new {RetBaseType}[RetElementCount]; + {RetBaseType}[] resultArray = new {RetBaseType}[RetElementCount]; + + Unsafe.WriteUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref maskArray[0]), maskOp); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref op1Array[0]), op1); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op2BaseType}, byte>(ref op2Array[0]), ref Unsafe.AsRef(op2), (uint)(Unsafe.SizeOf<{Op2BaseType}>() * Op2ElementCount)); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op3BaseType}, byte>(ref op3Array[0]), op3); + Unsafe.WriteUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref falseValArray[0]), falseOp); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref resultArray[0]), ref Unsafe.AsRef(result), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); + + ValidateConditionalSelectResult(maskArray, op1Array, op2Array, op3Array, falseValArray, resultArray, method); + } + + private void ValidateConditionalSelectResult({RetBaseType}[] maskOp, {Op1BaseType}[] firstOp, {Op2BaseType}[] secondOp, {Op3BaseType}[] thirdOp, {RetBaseType}[] falseOp, {RetBaseType}[] result, [CallerMemberName] string method = "") + { + var succeeded = Helpers.CheckGatherVectorBehavior(maskOp, firstOp, secondOp, thirdOp, falseOp, result); + + if (!succeeded) + { + TestLibrary.TestFramework.LogInformation($"{nameof({Isa})}.{nameof({Isa}.{Method})}<{RetBaseType}>({Op1BaseType}, {Op2BaseType}, {Op3BaseType}): {method} failed:"); + TestLibrary.TestFramework.LogInformation($" maskOp: ({string.Join(", ", maskOp)})"); + TestLibrary.TestFramework.LogInformation($" firstOp: ({string.Join(", ", firstOp)})"); + TestLibrary.TestFramework.LogInformation($" secondOp: ({string.Join(", ", secondOp)})"); + TestLibrary.TestFramework.LogInformation($" thirdOp: ({string.Join(", ", thirdOp)})"); + TestLibrary.TestFramework.LogInformation($" falseOp: ({string.Join(", ", falseOp)})"); + TestLibrary.TestFramework.LogInformation($" result: ({string.Join(", ", result)})"); + TestLibrary.TestFramework.LogInformation(string.Empty); + + Succeeded = false; + } + } + + private void ValidateFirstFaultingResult(void* op1, ref byte op2, int op2Size, void* op3, void* result, Vector<{RetBaseType}> faultResult, [CallerMemberName] string method = "") + { + {Op1BaseType}[] inArray1 = new {Op1BaseType}[Op1ElementCount]; + {Op2BaseType}[] inArray2 = new {Op2BaseType}[op2Size / Unsafe.SizeOf<{Op2BaseType}>()]; + {Op3BaseType}[] inArray3 = new {Op3BaseType}[Op3ElementCount]; + {RetBaseType}[] outArray = new {RetBaseType}[RetElementCount]; + + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref inArray1[0]), ref Unsafe.AsRef(op1), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op2BaseType}, byte>(ref inArray2[0]), ref op2, (uint)(inArray2.Length * Unsafe.SizeOf<{Op2BaseType}>())); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op3BaseType}, byte>(ref inArray3[0]), ref Unsafe.AsRef(op3), (uint)Unsafe.SizeOf<{Op3VectorType}<{Op3BaseType}>>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref outArray[0]), ref Unsafe.AsRef(result), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); + + ValidateFirstFaultingResult(inArray1, inArray2, inArray3, outArray, faultResult, method); + } + + private void ValidateFirstFaultingResult({Op1BaseType}[] firstOp, {Op2BaseType}[] secondOp, {Op3BaseType}[] thirdOp, {RetBaseType}[] result, Vector<{RetBaseType}> faultResult, [CallerMemberName] string method = "") + { + var succeeded = Helpers.CheckGatherVectorFirstFaultingBehavior(firstOp, secondOp, thirdOp, result, faultResult); + + if (!succeeded) + { + TestLibrary.TestFramework.LogInformation($"{nameof({Isa})}.{nameof({Isa}.{Method})}<{RetBaseType}>({Op1BaseType}, {Op2BaseType}, {Op3BaseType}): {method} failed:"); + TestLibrary.TestFramework.LogInformation($" firstOp: ({string.Join(", ", firstOp)})"); + TestLibrary.TestFramework.LogInformation($" secondOp: ({string.Join(", ", secondOp)})"); + TestLibrary.TestFramework.LogInformation($" thirdOp: ({string.Join(", ", thirdOp)})"); + TestLibrary.TestFramework.LogInformation($" result: ({string.Join(", ", result)})"); + TestLibrary.TestFramework.LogInformation($" faultResult: ({faultResult})"); + TestLibrary.TestFramework.LogInformation(string.Empty); + + Succeeded = false; + } + } + } +} \ No newline at end of file From 3923946de956b4cad85f86f152208a853c6196d5 Mon Sep 17 00:00:00 2001 From: TIHan Date: Mon, 8 Jul 2024 15:06:15 -0700 Subject: [PATCH 09/83] Added GetFfr suffix-style APIs --- src/coreclr/jit/hwintrinsiclistarm64sve.h | 9 +++- .../Arm/Sve.PlatformNotSupported.cs | 44 ++++++++++++++++++- .../src/System/Runtime/Intrinsics/Arm/Sve.cs | 44 ++++++++++++++++++- .../ref/System.Runtime.Intrinsics.cs | 9 +++- .../GenerateHWIntrinsicTests_Arm.cs | 44 +++++++++---------- ...eGatherVectorFirstFaultingIndices.template | 4 +- .../SveLoadVectorFirstFaultingTest.template | 4 +- 7 files changed, 128 insertions(+), 30 deletions(-) diff --git a/src/coreclr/jit/hwintrinsiclistarm64sve.h b/src/coreclr/jit/hwintrinsiclistarm64sve.h index 094fe270b3b6d..5102a097da93e 100644 --- a/src/coreclr/jit/hwintrinsiclistarm64sve.h +++ b/src/coreclr/jit/hwintrinsiclistarm64sve.h @@ -117,7 +117,14 @@ HARDWARE_INTRINSIC(Sve, GatherVectorUInt32WithByteOffsetsZeroExtend, HARDWARE_INTRINSIC(Sve, GatherVectorUInt32ZeroExtend, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_sve_ld1w, INS_sve_ld1w, INS_sve_ld1w, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, GatherVectorWithByteOffsets, -1, 3, true, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_sve_ld1w, INS_sve_ld1d, INS_sve_ld1d, INS_sve_ld1w, INS_sve_ld1d}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, GetActiveElementCount, -1, 2, true, {INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ExplicitMaskedOperation) -HARDWARE_INTRINSIC(Sve, GetFfr, -1, 0, false, {INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, GetFfrByte, -1, 0, false, {INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, GetFfrInt16, -1, 0, false, {INS_invalid, INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, GetFfrInt32, -1, 0, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, GetFfrInt64, -1, 0, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, GetFfrSByte, -1, 0, false, {INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, GetFfrUInt16, -1, 0, false, {INS_invalid, INS_invalid, INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, GetFfrUInt32, -1, 0, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, GetFfrUInt64, -1, 0, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) HARDWARE_INTRINSIC(Sve, InsertIntoShiftedVector, -1, 2, true, {INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics) HARDWARE_INTRINSIC(Sve, LeadingSignCount, -1, -1, false, {INS_sve_cls, INS_invalid, INS_sve_cls, INS_invalid, INS_sve_cls, INS_invalid, INS_sve_cls, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LeadingZeroCount, -1, -1, false, {INS_sve_clz, INS_sve_clz, INS_sve_clz, INS_sve_clz, INS_sve_clz, INS_sve_clz, INS_sve_clz, INS_sve_clz, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs index e3e1f756275de..52082ba51a45b 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs @@ -4207,7 +4207,49 @@ internal Arm64() { } /// svbool_t svrdffr() /// RDFFR Presult.B /// - public static unsafe Vector GetFfr() { throw new PlatformNotSupportedException(); } + public static Vector GetFfrByte() { throw new PlatformNotSupportedException(); } + + /// + /// svbool_t svrdffr() + /// RDFFR Presult.B + /// + public static Vector GetFfrInt16() { throw new PlatformNotSupportedException(); } + + /// + /// svbool_t svrdffr() + /// RDFFR Presult.B + /// + public static Vector GetFfrInt32() { throw new PlatformNotSupportedException(); } + + /// + /// svbool_t svrdffr() + /// RDFFR Presult.B + /// + public static Vector GetFfrInt64() { throw new PlatformNotSupportedException(); } + + /// + /// svbool_t svrdffr() + /// RDFFR Presult.B + /// + public static Vector GetFfrSByte() { throw new PlatformNotSupportedException(); } + + /// + /// svbool_t svrdffr() + /// RDFFR Presult.B + /// + public static Vector GetFfrUInt16() { throw new PlatformNotSupportedException(); } + + /// + /// svbool_t svrdffr() + /// RDFFR Presult.B + /// + public static Vector GetFfrUInt32() { throw new PlatformNotSupportedException(); } + + /// + /// svbool_t svrdffr() + /// RDFFR Presult.B + /// + public static Vector GetFfrUInt64() { throw new PlatformNotSupportedException(); } /// Insert scalar into shifted vector diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs index 9b7b1be04767c..82fe0681234d9 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs @@ -4266,7 +4266,49 @@ internal Arm64() { } /// svbool_t svrdffr() /// RDFFR Presult.B /// - public static unsafe Vector GetFfr() => GetFfr(); + public static Vector GetFfrByte() => GetFfrByte(); + + /// + /// svbool_t svrdffr() + /// RDFFR Presult.B + /// + public static Vector GetFfrInt16() => GetFfrInt16(); + + /// + /// svbool_t svrdffr() + /// RDFFR Presult.B + /// + public static Vector GetFfrInt32() => GetFfrInt32(); + + /// + /// svbool_t svrdffr() + /// RDFFR Presult.B + /// + public static Vector GetFfrInt64() => GetFfrInt64(); + + /// + /// svbool_t svrdffr() + /// RDFFR Presult.B + /// + public static Vector GetFfrSByte() => GetFfrSByte(); + + /// + /// svbool_t svrdffr() + /// RDFFR Presult.B + /// + public static Vector GetFfrUInt16() => GetFfrUInt16(); + + /// + /// svbool_t svrdffr() + /// RDFFR Presult.B + /// + public static Vector GetFfrUInt32() => GetFfrUInt32(); + + /// + /// svbool_t svrdffr() + /// RDFFR Presult.B + /// + public static Vector GetFfrUInt64() => GetFfrUInt64(); /// Insert scalar into shifted vector diff --git a/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs b/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs index ec7d7a187d2a6..1c11d841ea1a5 100644 --- a/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs +++ b/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs @@ -4853,7 +4853,14 @@ internal Arm64() { } public static ulong GetActiveElementCount(System.Numerics.Vector mask, System.Numerics.Vector from) { throw null; } public static ulong GetActiveElementCount(System.Numerics.Vector mask, System.Numerics.Vector from) { throw null; } - public static unsafe System.Numerics.Vector GetFfr() { throw null; } + public static System.Numerics.Vector GetFfrByte() { throw null; } + public static System.Numerics.Vector GetFfrInt16() { throw null; } + public static System.Numerics.Vector GetFfrInt32() { throw null; } + public static System.Numerics.Vector GetFfrInt64() { throw null; } + public static System.Numerics.Vector GetFfrSByte() { throw null; } + public static System.Numerics.Vector GetFfrUInt16() { throw null; } + public static System.Numerics.Vector GetFfrUInt32() { throw null; } + public static System.Numerics.Vector GetFfrUInt64() { throw null; } public static System.Numerics.Vector LeadingSignCount(System.Numerics.Vector value) { throw null; } public static System.Numerics.Vector LeadingSignCount(System.Numerics.Vector value) { throw null; } diff --git a/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs b/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs index 4c7b0941fab11..0ea300ca19a51 100644 --- a/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs +++ b/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs @@ -3604,18 +3604,18 @@ ("SveGatherVectorIndices.template", new Dictionary {["TestName"] = "Sve_GatherVectorUInt32ZeroExtend_Indices_ulong_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorUInt32ZeroExtend", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2BaseType"] = "UInt32", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt64", ["ExtendedElementType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt64()"}), ("SveGatherVectorIndices.template", new Dictionary {["TestName"] = "Sve_GatherVectorUInt32ZeroExtend_Indices_uint_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorUInt32ZeroExtend", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2BaseType"] = "UInt32", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt32", ["ExtendedElementType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt32()"}), - ("SveGatherVectorFirstFaultingIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_float_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2BaseType"] = "Single", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int32", ["ExtendedElementType"] = "Single", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskSingle()", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt32()"}), - ("SveGatherVectorFirstFaultingIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_int_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2BaseType"] = "Int32", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int32", ["ExtendedElementType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt32()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt32()"}), - ("SveGatherVectorFirstFaultingIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_uint_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2BaseType"] = "UInt32", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int32", ["ExtendedElementType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt32()"}), - ("SveGatherVectorFirstFaultingIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_float_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2BaseType"] = "Single", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt32", ["ExtendedElementType"] = "Single", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskSingle()", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt32()"}), - ("SveGatherVectorFirstFaultingIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_int_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2BaseType"] = "Int32", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt32", ["ExtendedElementType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt32()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt32()"}), - ("SveGatherVectorFirstFaultingIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_uint_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2BaseType"] = "UInt32", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt32", ["ExtendedElementType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt32()"}), - ("SveGatherVectorFirstFaultingIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_double_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2BaseType"] = "Double", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int64", ["ExtendedElementType"] = "Double", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskDouble()", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt64()"}), - ("SveGatherVectorFirstFaultingIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_long_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2BaseType"] = "Int64", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int64", ["ExtendedElementType"] = "Int64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt64()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt64()"}), - ("SveGatherVectorFirstFaultingIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_ulong_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2BaseType"] = "UInt64", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int64", ["ExtendedElementType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt64()"}), - ("SveGatherVectorFirstFaultingIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_double_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2BaseType"] = "Double", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt64", ["ExtendedElementType"] = "Double", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskDouble()", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt64()"}), - ("SveGatherVectorFirstFaultingIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_long_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2BaseType"] = "Int64", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt64", ["ExtendedElementType"] = "Int64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt64()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt64()"}), - ("SveGatherVectorFirstFaultingIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_ulong_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2BaseType"] = "UInt64", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt64", ["ExtendedElementType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt64()"}), + ("SveGatherVectorFirstFaultingIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_float_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2BaseType"] = "Single", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int32", ["ExtendedElementType"] = "Single", ["GetFfrType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskSingle()", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt32()"}), + ("SveGatherVectorFirstFaultingIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_int_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2BaseType"] = "Int32", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int32", ["ExtendedElementType"] = "Int32", ["GetFfrType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt32()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt32()"}), + ("SveGatherVectorFirstFaultingIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_uint_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2BaseType"] = "UInt32", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int32", ["ExtendedElementType"] = "UInt32", ["GetFfrType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt32()"}), + ("SveGatherVectorFirstFaultingIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_float_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2BaseType"] = "Single", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt32", ["ExtendedElementType"] = "Single", ["GetFfrType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskSingle()", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt32()"}), + ("SveGatherVectorFirstFaultingIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_int_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2BaseType"] = "Int32", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt32", ["ExtendedElementType"] = "Int32", ["GetFfrType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt32()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt32()"}), + ("SveGatherVectorFirstFaultingIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_uint_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2BaseType"] = "UInt32", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt32", ["ExtendedElementType"] = "UInt32", ["GetFfrType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt32()"}), + ("SveGatherVectorFirstFaultingIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_double_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2BaseType"] = "Double", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int64", ["ExtendedElementType"] = "Double", ["GetFfrType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskDouble()", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt64()"}), + ("SveGatherVectorFirstFaultingIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_long_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2BaseType"] = "Int64", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int64", ["ExtendedElementType"] = "Int64", ["GetFfrType"] = "Int64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt64()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt64()"}), + ("SveGatherVectorFirstFaultingIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_ulong_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2BaseType"] = "UInt64", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int64", ["ExtendedElementType"] = "UInt64", ["GetFfrType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt64()"}), + ("SveGatherVectorFirstFaultingIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_double_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2BaseType"] = "Double", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt64", ["ExtendedElementType"] = "Double", ["GetFfrType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskDouble()", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt64()"}), + ("SveGatherVectorFirstFaultingIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_long_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2BaseType"] = "Int64", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt64", ["ExtendedElementType"] = "Int64", ["GetFfrType"] = "Int64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt64()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt64()"}), + ("SveGatherVectorFirstFaultingIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_ulong_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2BaseType"] = "UInt64", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt64", ["ExtendedElementType"] = "UInt64", ["GetFfrType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt64()"}), ("SveGatherVectorByteOffsets.template",new Dictionary {["TestName"] = "Sve_GatherVectorWithByteOffsets_float_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorWithByteOffsets", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2BaseType"] = "Single", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int32", ["ExtendedElementType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskSingle()", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt32()"}), ("SveGatherVectorByteOffsets.template",new Dictionary {["TestName"] = "Sve_GatherVectorWithByteOffsets_int_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorWithByteOffsets", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2BaseType"] = "Int32", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int32", ["ExtendedElementType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt32()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt32()"}), @@ -3676,16 +3676,16 @@ ("SveLoadNonFaultingUnOpTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorNonFaulting_uint", ["Isa"] = "Sve", ["Method"] = "LoadVectorNonFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2BaseType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt32()", ["Cast"] = "(uint*)", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), ("SveLoadNonFaultingUnOpTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorNonFaulting_ulong", ["Isa"] = "Sve", ["Method"] = "LoadVectorNonFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2BaseType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt64()", ["Cast"] = "(ulong*)", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_float", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Single", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSingle()", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()"}), - ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_double", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Double", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetDouble()", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()"}), - ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_sbyte", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "SByte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "SByte", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "SByte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSByte()", ["NextValueOp2"] = "TestLibrary.Generator.GetSByte()"}), - ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_short", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int16", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt16()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt16()"}), - ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt32()"}), - ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt64()"}), - ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_byte", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Byte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Byte", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetByte()", ["NextValueOp2"] = "TestLibrary.Generator.GetByte()"}), - ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_ushort", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt16", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt16()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt16()"}), - ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()"}), - ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_float", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Single", ["GetFfrType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSingle()", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_double", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Double", ["GetFfrType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetDouble()", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_sbyte", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "SByte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "SByte", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "SByte", ["GetFfrType"] = "SByte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSByte()", ["NextValueOp2"] = "TestLibrary.Generator.GetSByte()"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_short", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int16", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int16", ["GetFfrType"] = "Int16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt16()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt16()"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int32", ["GetFfrType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt32()"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int64", ["GetFfrType"] = "Int64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt64()"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_byte", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Byte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Byte", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Byte", ["GetFfrType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetByte()", ["NextValueOp2"] = "TestLibrary.Generator.GetByte()"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_ushort", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt16", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt16", ["GetFfrType"] = "UInt16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt16()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt16()"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt32", ["GetFfrType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt64", ["GetFfrType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()"}), ("SveLoadVectorMaskedTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorNonTemporal_float", ["Isa"] = "Sve", ["Method"] = "LoadVectorNonTemporal", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2BaseType"] = "Single", ["LargestVectorSize"] = "64", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), ("SveLoadVectorMaskedTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorNonTemporal_double", ["Isa"] = "Sve", ["Method"] = "LoadVectorNonTemporal", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2BaseType"] = "Double", ["LargestVectorSize"] = "64", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorFirstFaultingIndices.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorFirstFaultingIndices.template index 2d1a272155663..20a372556c748 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorFirstFaultingIndices.template +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorFirstFaultingIndices.template @@ -301,12 +301,12 @@ namespace JIT.HardwareIntrinsics.Arm ({Op2BaseType}*)(Unsafe.AsPointer(ref op2Ref)), op3 ); - var faultResult = Sve.GetFfr(); + var faultResult = Sve.GetFfr{GetFfrType}(); ref var op3Ref = ref op3; Unsafe.Write(_dataTable.outArrayPtr, result); - ValidateFirstFaultingResult(_dataTable.inArray1Ptr, ref op2Ref, _dataTable.inBounded.Span.Length, Unsafe.AsPointer(ref op3Ref), _dataTable.outArrayPtr, faultResult.As()); + ValidateFirstFaultingResult(_dataTable.inArray1Ptr, ref op2Ref, _dataTable.inBounded.Span.Length, Unsafe.AsPointer(ref op3Ref), _dataTable.outArrayPtr, faultResult.As<{GetFfrType}, {RetBaseType}>()); } public void RunBasicScenario_FalseMask() diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveLoadVectorFirstFaultingTest.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveLoadVectorFirstFaultingTest.template index d446a36b50994..9b004863c9f29 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveLoadVectorFirstFaultingTest.template +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveLoadVectorFirstFaultingTest.template @@ -202,10 +202,10 @@ namespace JIT.HardwareIntrinsics.Arm loadMask, ({Op2BaseType}*)(Unsafe.AsPointer(ref op2Ref)) ); - var faultResult = Sve.GetFfr(); + var faultResult = Sve.GetFfr{GetFfrType}(); Unsafe.Write(_dataTable.outArrayPtr, result); - ValidateFirstFaultingResult(ref op2Ref, _dataTable.inBounded.Span.Length, _dataTable.outArrayPtr, faultResult.As()); + ValidateFirstFaultingResult(ref op2Ref, _dataTable.inBounded.Span.Length, _dataTable.outArrayPtr, faultResult.As<{GetFfrType}, {RetBaseType}>()); } public void RunReflectionScenario_Load() From 461b6a3a32056e28979a8b6c86459fb1aa4b187b Mon Sep 17 00:00:00 2001 From: TIHan Date: Mon, 8 Jul 2024 16:06:32 -0700 Subject: [PATCH 10/83] Fixing GatherVector tests --- .../Intrinsics/Arm/Sve.PlatformNotSupported.cs | 2 +- .../SveGatherVectorFirstFaultingIndices.template | 13 +++++++------ .../Shared/SveLoadVectorFirstFaultingTest.template | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs index 52082ba51a45b..0ee8104c15d59 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs @@ -4208,7 +4208,7 @@ internal Arm64() { } /// RDFFR Presult.B /// public static Vector GetFfrByte() { throw new PlatformNotSupportedException(); } - + /// /// svbool_t svrdffr() /// RDFFR Presult.B diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorFirstFaultingIndices.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorFirstFaultingIndices.template index 20a372556c748..86c2827817d6a 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorFirstFaultingIndices.template +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorFirstFaultingIndices.template @@ -107,7 +107,7 @@ namespace JIT.HardwareIntrinsics.Arm int sizeOfinArray1 = inArray1.Length * Unsafe.SizeOf<{Op1BaseType}>(); int sizeOfinArray2 = inArray2.Length * Unsafe.SizeOf<{Op2BaseType}>(); int sizeOfinArray3 = inArray3.Length * Unsafe.SizeOf<{Op3BaseType}>(); - int sizeOfinBounded = new Random().Next(Unsafe.SizeOf<{Op2BaseType}>(), sizeOfinArray2 - 1); + int sizeOfinBounded = new Random().Next(Unsafe.SizeOf<{Op2BaseType}>(), Vector<{Op2BaseType}>.Count * Unsafe.SizeOf<{Op2BaseType}>() - 1); int sizeOfoutArray = outArray.Length * Unsafe.SizeOf<{RetBaseType}>(); if ((alignment != 64 && alignment != 16 && alignment != 8) || (alignment * 2) < sizeOfinArray1 || (alignment * 2) < sizeOfinArray3 || (alignment * 2) < sizeOfoutArray) @@ -288,7 +288,8 @@ namespace JIT.HardwareIntrinsics.Arm // So set the first active element of the index vector to 0. for (var i = 0; i < Vector<{Op3BaseType}>.Count; i++) { - if (loadMask3[i] != 0) + // op1 is the mask for GatherVector. + if (op1[i] != 0) { op3 = op3.WithElement<{Op3BaseType}>(i, 0); break; @@ -618,10 +619,10 @@ namespace JIT.HardwareIntrinsics.Arm if (!succeeded) { TestLibrary.TestFramework.LogInformation($"{nameof({Isa})}.{nameof({Isa}.{Method})}<{RetBaseType}>({Op1BaseType}, {Op2BaseType}, {Op3BaseType}): {method} failed:"); - TestLibrary.TestFramework.LogInformation($" firstOp: ({string.Join(", ", firstOp)})"); - TestLibrary.TestFramework.LogInformation($" secondOp: ({string.Join(", ", secondOp)})"); - TestLibrary.TestFramework.LogInformation($" thirdOp: ({string.Join(", ", thirdOp)})"); - TestLibrary.TestFramework.LogInformation($" result: ({string.Join(", ", result)})"); + TestLibrary.TestFramework.LogInformation($" firstOp: ({string.Join(", ", firstOp)})"); + TestLibrary.TestFramework.LogInformation($" secondOp: ({string.Join(", ", secondOp)})"); + TestLibrary.TestFramework.LogInformation($" thirdOp: ({string.Join(", ", thirdOp)})"); + TestLibrary.TestFramework.LogInformation($" result: ({string.Join(", ", result)})"); TestLibrary.TestFramework.LogInformation($" faultResult: ({faultResult})"); TestLibrary.TestFramework.LogInformation(string.Empty); diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveLoadVectorFirstFaultingTest.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveLoadVectorFirstFaultingTest.template index 9b004863c9f29..4e7cb213d63d1 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveLoadVectorFirstFaultingTest.template +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveLoadVectorFirstFaultingTest.template @@ -76,7 +76,7 @@ namespace JIT.HardwareIntrinsics.Arm public DataTable({Op2BaseType}[] inArray1, {RetBaseType}[] outArray, int alignment) { int sizeOfinArray1 = inArray1.Length * Unsafe.SizeOf<{Op2BaseType}>(); - int sizeOfinBounded = new Random().Next(Unsafe.SizeOf<{Op2BaseType}>(), sizeOfinArray1 - 1); + int sizeOfinBounded = new Random().Next(Unsafe.SizeOf<{Op2BaseType}>(), Vector<{Op2BaseType}>.Count * Unsafe.SizeOf<{Op2BaseType}>() - 1); int sizeOfoutArray = outArray.Length * Unsafe.SizeOf<{RetBaseType}>(); if ((alignment != 64 && alignment != 16 && alignment != 8) || (alignment * 2) < sizeOfinArray1 || (alignment * 2) < sizeOfoutArray) { From d5b86757c11dcb879d97b2cf50d1ed331e585f62 Mon Sep 17 00:00:00 2001 From: TIHan Date: Mon, 8 Jul 2024 16:25:58 -0700 Subject: [PATCH 11/83] Formatting --- src/coreclr/jit/gentree.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index c76cf2cd25189..ffc0e51b83e1a 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -26395,11 +26395,12 @@ bool GenTreeHWIntrinsic::OperIsMemoryLoad(GenTree** pAddr) const { #ifdef TARGET_ARM64 static_assert_no_msg( - AreContiguous(NI_Sve_GatherVector, NI_Sve_GatherVectorByteZeroExtend, NI_Sve_GatherVectorFirstFaulting, NI_Sve_GatherVectorInt16SignExtend, - NI_Sve_GatherVectorInt16WithByteOffsetsSignExtend, NI_Sve_GatherVectorInt32SignExtend, - NI_Sve_GatherVectorInt32WithByteOffsetsSignExtend, NI_Sve_GatherVectorSByteSignExtend, - NI_Sve_GatherVectorUInt16WithByteOffsetsZeroExtend, NI_Sve_GatherVectorUInt16ZeroExtend, - NI_Sve_GatherVectorUInt32WithByteOffsetsZeroExtend, NI_Sve_GatherVectorUInt32ZeroExtend)); + AreContiguous(NI_Sve_GatherVector, NI_Sve_GatherVectorByteZeroExtend, NI_Sve_GatherVectorFirstFaulting, + NI_Sve_GatherVectorInt16SignExtend, NI_Sve_GatherVectorInt16WithByteOffsetsSignExtend, + NI_Sve_GatherVectorInt32SignExtend, NI_Sve_GatherVectorInt32WithByteOffsetsSignExtend, + NI_Sve_GatherVectorSByteSignExtend, NI_Sve_GatherVectorUInt16WithByteOffsetsZeroExtend, + NI_Sve_GatherVectorUInt16ZeroExtend, NI_Sve_GatherVectorUInt32WithByteOffsetsZeroExtend, + NI_Sve_GatherVectorUInt32ZeroExtend)); assert(varTypeIsI(addr) || (varTypeIsSIMD(addr) && ((intrinsicId >= NI_Sve_GatherPrefetch16Bit) && (intrinsicId <= NI_Sve_GatherVectorUInt32ZeroExtend)))); #else From 07833e3d0a8577753f50c9aeabca55b15d76a6b6 Mon Sep 17 00:00:00 2001 From: TIHan Date: Tue, 9 Jul 2024 12:28:14 -0700 Subject: [PATCH 12/83] Feedback --- src/coreclr/jit/fgdiagnostic.cpp | 2 -- src/coreclr/jit/gentree.cpp | 4 ---- src/coreclr/jit/hwintrinsiccodegenarm64.cpp | 6 +----- 3 files changed, 1 insertion(+), 11 deletions(-) diff --git a/src/coreclr/jit/fgdiagnostic.cpp b/src/coreclr/jit/fgdiagnostic.cpp index c8e669e3d0a04..322aae30ec500 100644 --- a/src/coreclr/jit/fgdiagnostic.cpp +++ b/src/coreclr/jit/fgdiagnostic.cpp @@ -3467,8 +3467,6 @@ void Compiler::fgDebugCheckFlags(GenTree* tree, BasicBlock* block) case NI_Sve_GatherPrefetch32Bit: case NI_Sve_GatherPrefetch64Bit: case NI_Sve_GatherPrefetch8Bit: - case NI_Sve_GatherVectorFirstFaulting: - case NI_Sve_LoadVectorFirstFaulting: case NI_Sve_SetFfr: { assert(tree->OperRequiresCallFlag(this)); diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index ffc0e51b83e1a..fc400064b7cd9 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -26804,8 +26804,6 @@ bool GenTreeHWIntrinsic::OperRequiresCallFlag() const case NI_Sve_GatherPrefetch32Bit: case NI_Sve_GatherPrefetch64Bit: case NI_Sve_GatherPrefetch8Bit: - case NI_Sve_GatherVectorFirstFaulting: - case NI_Sve_LoadVectorFirstFaulting: case NI_Sve_SetFfr: { return true; @@ -26997,8 +26995,6 @@ void GenTreeHWIntrinsic::Initialize(NamedIntrinsic intrinsicId) case NI_Sve_GatherPrefetch32Bit: case NI_Sve_GatherPrefetch64Bit: case NI_Sve_GatherPrefetch8Bit: - case NI_Sve_GatherVectorFirstFaulting: - case NI_Sve_LoadVectorFirstFaulting: case NI_Sve_SetFfr: { // Mark as a call and global reference, much as is done for GT_KEEPALIVE diff --git a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp index dd24c268e1cb1..306cddea26e79 100644 --- a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp @@ -2255,11 +2255,7 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) case NI_Sve_LoadVectorFirstFaulting: { - insScalableOpts sopt = INS_SCALABLE_OPTS_LSL_N; - if (opt == INS_OPTS_SCALABLE_B) - { - sopt = INS_SCALABLE_OPTS_NONE; - } + insScalableOpts sopt = (opt == INS_OPTS_SCALABLE_B) ? INS_SCALABLE_OPTS_NONE : INS_SCALABLE_OPTS_LSL_N; GetEmitter()->emitIns_R_R_R_R(ins, emitSize, targetReg, op1Reg, op2Reg, REG_ZR, opt, sopt); break; } From 05fb46d44d34e8ae85a5fc406b096cc1ed49ad5c Mon Sep 17 00:00:00 2001 From: TIHan Date: Tue, 9 Jul 2024 12:29:56 -0700 Subject: [PATCH 13/83] Feedback --- src/coreclr/jit/gentree.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index fc400064b7cd9..d579f8e10f800 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -26401,7 +26401,7 @@ bool GenTreeHWIntrinsic::OperIsMemoryLoad(GenTree** pAddr) const NI_Sve_GatherVectorSByteSignExtend, NI_Sve_GatherVectorUInt16WithByteOffsetsZeroExtend, NI_Sve_GatherVectorUInt16ZeroExtend, NI_Sve_GatherVectorUInt32WithByteOffsetsZeroExtend, NI_Sve_GatherVectorUInt32ZeroExtend)); - assert(varTypeIsI(addr) || (varTypeIsSIMD(addr) && ((intrinsicId >= NI_Sve_GatherPrefetch16Bit) && + assert(varTypeIsI(addr) || (varTypeIsSIMD(addr) && ((intrinsicId >= NI_Sve_GatherVector) && (intrinsicId <= NI_Sve_GatherVectorUInt32ZeroExtend)))); #else assert(varTypeIsI(addr)); From c63f8785be42c3dd3a55c3f4365ec037968240f1 Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Wed, 10 Jul 2024 10:02:03 -0700 Subject: [PATCH 14/83] Ensure the P/Invokes are blittable --- .../System/Buffers/BoundedMemory.Windows.cs | 43 ++++++++----------- 1 file changed, 18 insertions(+), 25 deletions(-) diff --git a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Windows.cs b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Windows.cs index 82572aaf14a18..d6dc353ed2daa 100644 --- a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Windows.cs +++ b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Windows.cs @@ -33,7 +33,7 @@ private static WindowsImplementation AllocateWithoutDataPopulationWindows( // Reserve and commit the entire range as NOACCESS. - VirtualAllocHandle handle = UnsafeNativeMethods.VirtualAlloc( + VirtualAllocHandle handle = VirtualAllocHandle.Allocate( lpAddress: IntPtr.Zero, dwSize: (IntPtr)totalBytesToAllocate /* cast throws OverflowException if out of range */, flAllocationType: VirtualAllocAllocationType.MEM_RESERVE | VirtualAllocAllocationType.MEM_COMMIT, @@ -91,9 +91,10 @@ internal VirtualAllocProtection Protection try { _handle.DangerousAddRef(ref refAdded); + MEMORY_BASIC_INFORMATION memoryInfo; if (UnsafeNativeMethods.VirtualQuery( lpAddress: _handle.DangerousGetHandle() + _byteOffsetIntoHandle, - lpBuffer: out MEMORY_BASIC_INFORMATION memoryInfo, + lpBuffer: &memoryInfo, dwLength: (IntPtr)sizeof(MEMORY_BASIC_INFORMATION)) == IntPtr.Zero) { Marshal.ThrowExceptionForHR(Marshal.GetHRForLastWin32Error()); @@ -117,11 +118,12 @@ internal VirtualAllocProtection Protection try { _handle.DangerousAddRef(ref refAdded); - if (!UnsafeNativeMethods.VirtualProtect( + VirtualAllocProtection flOldProtect; + if (UnsafeNativeMethods.VirtualProtect( lpAddress: _handle.DangerousGetHandle() + _byteOffsetIntoHandle, dwSize: (IntPtr)(&((T*)null)[_elementCount]), flNewProtect: value, - lpflOldProtect: out _)) + lpflOldProtect: &flOldProtect) == 0) { Marshal.ThrowExceptionForHR(Marshal.GetHRForLastWin32Error()); throw new InvalidOperationException("VirtualProtect failed unexpectedly."); @@ -279,13 +281,20 @@ public VirtualAllocHandle() { } + internal static VirtualAllocHandle Allocate(IntPtr lpAddress, IntPtr dwSize, VirtualAllocAllocationType flAllocationType, VirtualAllocProtection flProtect) + { + VirtualAllocHandle retVal = new VirtualAllocHandle(); + retVal.SetHandle(UnsafeNativeMethods.VirtualAlloc(lpAddress, dwSize, flAllocationType, flProtect)); + return retVal; + } + // Do not provide a finalizer - SafeHandle's critical finalizer will // call ReleaseHandle for you. public override bool IsInvalid => (handle == IntPtr.Zero); protected override bool ReleaseHandle() => - UnsafeNativeMethods.VirtualFree(handle, IntPtr.Zero, VirtualAllocAllocationType.MEM_RELEASE); + UnsafeNativeMethods.VirtualFree(handle, IntPtr.Zero, VirtualAllocAllocationType.MEM_RELEASE) != 0; } private static class UnsafeNativeMethods @@ -294,35 +303,19 @@ private static class UnsafeNativeMethods // https://msdn.microsoft.com/en-us/library/windows/desktop/aa366887(v=vs.85).aspx [DllImport(KERNEL32_LIB, SetLastError = true)] - public static extern VirtualAllocHandle VirtualAlloc( - IntPtr lpAddress, - IntPtr dwSize, - VirtualAllocAllocationType flAllocationType, - VirtualAllocProtection flProtect); + public static extern IntPtr VirtualAlloc(IntPtr lpAddress, IntPtr dwSize, VirtualAllocAllocationType flAllocationType, VirtualAllocProtection flProtect); // https://msdn.microsoft.com/en-us/library/windows/desktop/aa366892(v=vs.85).aspx [DllImport(KERNEL32_LIB, SetLastError = true)] - [return: MarshalAs(UnmanagedType.Bool)] - public static extern bool VirtualFree( - IntPtr lpAddress, - IntPtr dwSize, - VirtualAllocAllocationType dwFreeType); + public static extern int VirtualFree(IntPtr lpAddress, IntPtr dwSize, VirtualAllocAllocationType dwFreeType); // https://msdn.microsoft.com/en-us/library/windows/desktop/aa366898(v=vs.85).aspx [DllImport(KERNEL32_LIB, SetLastError = true)] - [return: MarshalAs(UnmanagedType.Bool)] - public static extern bool VirtualProtect( - IntPtr lpAddress, - IntPtr dwSize, - VirtualAllocProtection flNewProtect, - out VirtualAllocProtection lpflOldProtect); + public static extern int VirtualProtect(IntPtr lpAddress, IntPtr dwSize, VirtualAllocProtection flNewProtect, VirtualAllocProtection* lpflOldProtect); // https://msdn.microsoft.com/en-us/library/windows/desktop/aa366902(v=vs.85).aspx [DllImport(KERNEL32_LIB, SetLastError = true)] - public static extern IntPtr VirtualQuery( - IntPtr lpAddress, - out MEMORY_BASIC_INFORMATION lpBuffer, - IntPtr dwLength); + public static extern IntPtr VirtualQuery(IntPtr lpAddress, MEMORY_BASIC_INFORMATION* lpBuffer, IntPtr dwLength); } } } From 6c28927435cdcdec16b2d79fc9a25f743e8f97b8 Mon Sep 17 00:00:00 2001 From: TIHan Date: Fri, 12 Jul 2024 18:22:58 -0700 Subject: [PATCH 15/83] Fix build --- src/coreclr/jit/hwintrinsiclistarm64sve.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/jit/hwintrinsiclistarm64sve.h b/src/coreclr/jit/hwintrinsiclistarm64sve.h index 4dae38fd42bce..6ae1c7e76fadc 100644 --- a/src/coreclr/jit/hwintrinsiclistarm64sve.h +++ b/src/coreclr/jit/hwintrinsiclistarm64sve.h @@ -233,8 +233,8 @@ HARDWARE_INTRINSIC(Sve, SaturatingIncrementBy64BitElementCount, HARDWARE_INTRINSIC(Sve, SaturatingIncrementBy8BitElementCount, 0, 3, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqincb, INS_sve_uqincb, INS_sve_sqincb, INS_sve_uqincb, INS_invalid, INS_invalid}, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasEnumOperand|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) HARDWARE_INTRINSIC(Sve, SaturatingIncrementByActiveElementCount, -1, 2, true, {INS_invalid, INS_sve_sqincp, INS_sve_sqincp, INS_sve_sqincp, INS_sve_sqincp, INS_sve_sqincp, INS_sve_sqincp, INS_sve_sqincp, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) HARDWARE_INTRINSIC(Sve, Scale, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fscale, INS_sve_fscale}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve, SetFfr, -1, 1, true, {INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialSideEffectMask|HW_Flag_SpecialCodeGen) HARDWARE_INTRINSIC(Sve, Scatter, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_st1w, INS_sve_st1w, INS_sve_st1d, INS_sve_st1d, INS_sve_st1w, INS_sve_st1d}, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, SetFfr, -1, 1, true, {INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialSideEffectMask|HW_Flag_SpecialCodeGen) HARDWARE_INTRINSIC(Sve, ShiftLeftLogical, -1, -1, false, {INS_sve_lsl, INS_sve_lsl, INS_sve_lsl, INS_sve_lsl, INS_sve_lsl, INS_sve_lsl, INS_sve_lsl, INS_sve_lsl, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) HARDWARE_INTRINSIC(Sve, ShiftRightArithmetic, -1, -1, false, {INS_sve_asr, INS_invalid, INS_sve_asr, INS_invalid, INS_sve_asr, INS_invalid, INS_sve_asr, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) HARDWARE_INTRINSIC(Sve, ShiftRightArithmeticForDivide, -1, -1, false, {INS_sve_asrd, INS_invalid, INS_sve_asrd, INS_invalid, INS_sve_asrd, INS_invalid, INS_sve_asrd, INS_invalid, INS_invalid, INS_invalid}, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand) From fb2012ec8722870b48ebfa6b38ed952dbfe36a98 Mon Sep 17 00:00:00 2001 From: TIHan Date: Fri, 12 Jul 2024 21:56:27 -0700 Subject: [PATCH 16/83] Remove checking for zeroes after the fault --- src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs index d1d5c06b8e1fe..0b2cc688458c2 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs @@ -8196,11 +8196,6 @@ private static bool CheckFirstFaultingBehaviorCore(T[] result, Vector faul { return false; } - - if (result[i] != T.Zero) - { - return false; - } } else { From aca67599d748033a671b6d9c26d7095cc2c2fb44 Mon Sep 17 00:00:00 2001 From: TIHan Date: Mon, 15 Jul 2024 18:43:59 -0700 Subject: [PATCH 17/83] Added GatherVectorFirstFaultingVectorBases test template, but currently without the FirstFaulting test. Added SveFfrTest template. --- src/coreclr/jit/hwintrinsiccodegenarm64.cpp | 1 + src/coreclr/jit/hwintrinsiclistarm64sve.h | 2 +- .../Arm/Sve.PlatformNotSupported.cs | 33 +- .../src/System/Runtime/Intrinsics/Arm/Sve.cs | 33 +- .../ref/System.Runtime.Intrinsics.cs | 6 +- .../GenerateHWIntrinsicTests_Arm.cs | 13 + .../HardwareIntrinsics/Arm/Shared/Helpers.cs | 94 ++- .../Arm/Shared/SveFfrTest.template | 325 ++++++++++ ...eGatherVectorFirstFaultingIndices.template | 4 +- ...herVectorFirstFaultingVectorBases.template | 600 ++++++++++++++++++ .../Shared/SveGatherVectorIndices.template | 18 +- .../SveGatherVectorVectorBases.template | 41 +- 12 files changed, 1070 insertions(+), 100 deletions(-) create mode 100644 src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveFfrTest.template create mode 100644 src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorFirstFaultingVectorBases.template diff --git a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp index 24d75c6820c62..05e333eb335c8 100644 --- a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp @@ -2288,6 +2288,7 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) case NI_Sve_LoadVectorFirstFaulting: { + assert(op3Reg == REG_NA); insScalableOpts sopt = (opt == INS_OPTS_SCALABLE_B) ? INS_SCALABLE_OPTS_NONE : INS_SCALABLE_OPTS_LSL_N; GetEmitter()->emitIns_R_R_R_R(ins, emitSize, targetReg, op1Reg, op2Reg, REG_ZR, opt, sopt); break; diff --git a/src/coreclr/jit/hwintrinsiclistarm64sve.h b/src/coreclr/jit/hwintrinsiclistarm64sve.h index 6ae1c7e76fadc..b7420689beb45 100644 --- a/src/coreclr/jit/hwintrinsiclistarm64sve.h +++ b/src/coreclr/jit/hwintrinsiclistarm64sve.h @@ -106,7 +106,7 @@ HARDWARE_INTRINSIC(Sve, GatherPrefetch64Bit, HARDWARE_INTRINSIC(Sve, GatherPrefetch8Bit, -1, -1, false, {INS_sve_prfb, INS_sve_prfb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Special, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasImmediateOperand|HW_Flag_HasEnumOperand|HW_Flag_SpecialSideEffect_Other) HARDWARE_INTRINSIC(Sve, GatherVector, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_sve_ld1w, INS_sve_ld1d, INS_sve_ld1d, INS_sve_ld1w, INS_sve_ld1d}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, GatherVectorByteZeroExtend, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1b, INS_sve_ld1b, INS_sve_ld1b, INS_sve_ld1b, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, GatherVectorFirstFaulting, -1, 3, true, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1w, INS_sve_ldff1w, INS_sve_ldff1d, INS_sve_ldff1d, INS_sve_ldff1w, INS_sve_ldff1d}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, GatherVectorFirstFaulting, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1w, INS_sve_ldff1w, INS_sve_ldff1d, INS_sve_ldff1d, INS_sve_ldff1w, INS_sve_ldff1d}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialSideEffectMask) HARDWARE_INTRINSIC(Sve, GatherVectorInt16SignExtend, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sh, INS_sve_ld1sh, INS_sve_ld1sh, INS_sve_ld1sh, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, GatherVectorInt16WithByteOffsetsSignExtend, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sh, INS_sve_ld1sh, INS_sve_ld1sh, INS_sve_ld1sh, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, GatherVectorInt32SignExtend, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sw, INS_sve_ld1sw, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs index 43e6b508ccb04..6d0adc1367708 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs @@ -3462,11 +3462,12 @@ internal Arm64() { } /// public static unsafe Vector GatherVectorFirstFaulting(Vector mask, double* address, Vector indices) { throw new PlatformNotSupportedException(); } - /// - /// svint32_t svldff1_gather[_u32base]_s32(svbool_t pg, svuint32_t bases) - /// LDFF1W Zresult.S, Pg/Z, [Zbases.S, #0] - /// - public static unsafe Vector GatherVectorFirstFaulting(Vector mask, Vector addresses) { throw new PlatformNotSupportedException(); } + // + // svint32_t svldff1_gather[_u32base]_s32(svbool_t pg, svuint32_t bases) + // LDFF1W Zresult.S, Pg/Z, [Zbases.S, #0] + // + // Removed as per #103297 + // public static unsafe Vector GatherVectorFirstFaulting(Vector mask, Vector addresses) { throw new PlatformNotSupportedException(); } /// /// svint32_t svldff1_gather_[s32]index[_s32](svbool_t pg, const int32_t *base, svint32_t indices) @@ -3504,11 +3505,12 @@ internal Arm64() { } /// public static unsafe Vector GatherVectorFirstFaulting(Vector mask, float* address, Vector indices) { throw new PlatformNotSupportedException(); } - /// - /// svfloat32_t svldff1_gather[_u32base]_f32(svbool_t pg, svuint32_t bases) - /// LDFF1W Zresult.S, Pg/Z, [Zbases.S, #0] - /// - public static unsafe Vector GatherVectorFirstFaulting(Vector mask, Vector addresses) { throw new PlatformNotSupportedException(); } + // + // svfloat32_t svldff1_gather[_u32base]_f32(svbool_t pg, svuint32_t bases) + // LDFF1W Zresult.S, Pg/Z, [Zbases.S, #0] + // + // Removed as per #103297 + // public static unsafe Vector GatherVectorFirstFaulting(Vector mask, Vector addresses) { throw new PlatformNotSupportedException(); } /// /// svfloat32_t svldff1_gather_[u32]index[_f32](svbool_t pg, const float32_t *base, svuint32_t indices) @@ -3516,11 +3518,12 @@ internal Arm64() { } /// public static unsafe Vector GatherVectorFirstFaulting(Vector mask, float* address, Vector indices) { throw new PlatformNotSupportedException(); } - /// - /// svuint32_t svldff1_gather[_u32base]_u32(svbool_t pg, svuint32_t bases) - /// LDFF1W Zresult.S, Pg/Z, [Zbases.S, #0] - /// - public static unsafe Vector GatherVectorFirstFaulting(Vector mask, Vector addresses) { throw new PlatformNotSupportedException(); } + // + // svuint32_t svldff1_gather[_u32base]_u32(svbool_t pg, svuint32_t bases) + // LDFF1W Zresult.S, Pg/Z, [Zbases.S, #0] + // + // Removed as per #103297 + // public static unsafe Vector GatherVectorFirstFaulting(Vector mask, Vector addresses) { throw new PlatformNotSupportedException(); } /// /// svuint32_t svldff1_gather_[s32]index[_u32](svbool_t pg, const uint32_t *base, svint32_t indices) diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs index b21998104287c..ed64f367321ef 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs @@ -3521,11 +3521,12 @@ internal Arm64() { } /// public static unsafe Vector GatherVectorFirstFaulting(Vector mask, double* address, Vector indices) => GatherVectorFirstFaulting(mask, address, indices); - /// - /// svint32_t svldff1_gather[_u32base]_s32(svbool_t pg, svuint32_t bases) - /// LDFF1W Zresult.S, Pg/Z, [Zbases.S, #0] - /// - public static unsafe Vector GatherVectorFirstFaulting(Vector mask, Vector addresses) => GatherVectorFirstFaulting(mask, addresses); + // + // svint32_t svldff1_gather[_u32base]_s32(svbool_t pg, svuint32_t bases) + // LDFF1W Zresult.S, Pg/Z, [Zbases.S, #0] + // + // Removed as per #103297 + // public static unsafe Vector GatherVectorFirstFaulting(Vector mask, Vector addresses) => GatherVectorFirstFaulting(mask, addresses); /// /// svint32_t svldff1_gather_[s32]index[_s32](svbool_t pg, const int32_t *base, svint32_t indices) @@ -3563,11 +3564,12 @@ internal Arm64() { } /// public static unsafe Vector GatherVectorFirstFaulting(Vector mask, float* address, Vector indices) => GatherVectorFirstFaulting(mask, address, indices); - /// - /// svfloat32_t svldff1_gather[_u32base]_f32(svbool_t pg, svuint32_t bases) - /// LDFF1W Zresult.S, Pg/Z, [Zbases.S, #0] - /// - public static unsafe Vector GatherVectorFirstFaulting(Vector mask, Vector addresses) => GatherVectorFirstFaulting(mask, addresses); + // + // svfloat32_t svldff1_gather[_u32base]_f32(svbool_t pg, svuint32_t bases) + // LDFF1W Zresult.S, Pg/Z, [Zbases.S, #0] + // + // Removed as per #103297 + // public static unsafe Vector GatherVectorFirstFaulting(Vector mask, Vector addresses) => GatherVectorFirstFaulting(mask, addresses); /// /// svfloat32_t svldff1_gather_[u32]index[_f32](svbool_t pg, const float32_t *base, svuint32_t indices) @@ -3575,11 +3577,12 @@ internal Arm64() { } /// public static unsafe Vector GatherVectorFirstFaulting(Vector mask, float* address, Vector indices) => GatherVectorFirstFaulting(mask, address, indices); - /// - /// svuint32_t svldff1_gather[_u32base]_u32(svbool_t pg, svuint32_t bases) - /// LDFF1W Zresult.S, Pg/Z, [Zbases.S, #0] - /// - public static unsafe Vector GatherVectorFirstFaulting(Vector mask, Vector addresses) => GatherVectorFirstFaulting(mask, addresses); + // + // svuint32_t svldff1_gather[_u32base]_u32(svbool_t pg, svuint32_t bases) + // LDFF1W Zresult.S, Pg/Z, [Zbases.S, #0] + // + // Removed as per #103297 + // public static unsafe Vector GatherVectorFirstFaulting(Vector mask, Vector addresses) => GatherVectorFirstFaulting(mask, addresses); /// /// svuint32_t svldff1_gather_[s32]index[_u32](svbool_t pg, const uint32_t *base, svint32_t indices) diff --git a/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs b/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs index 9c059c533564f..71263bd49943d 100644 --- a/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs +++ b/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs @@ -4853,16 +4853,16 @@ internal Arm64() { } public static unsafe System.Numerics.Vector GatherVectorFirstFaulting(System.Numerics.Vector mask, double* address, System.Numerics.Vector indices) { throw null; } public static unsafe System.Numerics.Vector GatherVectorFirstFaulting(System.Numerics.Vector mask, System.Numerics.Vector addresses) { throw null; } public static unsafe System.Numerics.Vector GatherVectorFirstFaulting(System.Numerics.Vector mask, double* address, System.Numerics.Vector indices) { throw null; } - public static unsafe System.Numerics.Vector GatherVectorFirstFaulting(System.Numerics.Vector mask, System.Numerics.Vector addresses) { throw null; } + // public static unsafe System.Numerics.Vector GatherVectorFirstFaulting(System.Numerics.Vector mask, System.Numerics.Vector addresses) { throw null; } public static unsafe System.Numerics.Vector GatherVectorFirstFaulting(System.Numerics.Vector mask, int* address, System.Numerics.Vector indices) { throw null; } public static unsafe System.Numerics.Vector GatherVectorFirstFaulting(System.Numerics.Vector mask, int* address, System.Numerics.Vector indices) { throw null; } public static unsafe System.Numerics.Vector GatherVectorFirstFaulting(System.Numerics.Vector mask, System.Numerics.Vector addresses) { throw null; } public static unsafe System.Numerics.Vector GatherVectorFirstFaulting(System.Numerics.Vector mask, long* address, System.Numerics.Vector indices) { throw null; } public static unsafe System.Numerics.Vector GatherVectorFirstFaulting(System.Numerics.Vector mask, long* address, System.Numerics.Vector indices) { throw null; } public static unsafe System.Numerics.Vector GatherVectorFirstFaulting(System.Numerics.Vector mask, float* address, System.Numerics.Vector indices) { throw null; } - public static unsafe System.Numerics.Vector GatherVectorFirstFaulting(System.Numerics.Vector mask, System.Numerics.Vector addresses) { throw null; } + // public static unsafe System.Numerics.Vector GatherVectorFirstFaulting(System.Numerics.Vector mask, System.Numerics.Vector addresses) { throw null; } public static unsafe System.Numerics.Vector GatherVectorFirstFaulting(System.Numerics.Vector mask, float* address, System.Numerics.Vector indices) { throw null; } - public static unsafe System.Numerics.Vector GatherVectorFirstFaulting(System.Numerics.Vector mask, System.Numerics.Vector addresses) { throw null; } + // public static unsafe System.Numerics.Vector GatherVectorFirstFaulting(System.Numerics.Vector mask, System.Numerics.Vector addresses) { throw null; } public static unsafe System.Numerics.Vector GatherVectorFirstFaulting(System.Numerics.Vector mask, uint* address, System.Numerics.Vector indices) { throw null; } public static unsafe System.Numerics.Vector GatherVectorFirstFaulting(System.Numerics.Vector mask, uint* address, System.Numerics.Vector indices) { throw null; } public static unsafe System.Numerics.Vector GatherVectorFirstFaulting(System.Numerics.Vector mask, System.Numerics.Vector addresses) { throw null; } diff --git a/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs b/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs index a002de75109e8..50b052a2fd5c6 100644 --- a/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs +++ b/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs @@ -3484,6 +3484,15 @@ ("SveExtractVectorTest.template", new Dictionary { ["TestName"] = "SveExtractVector_UInt32_1", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "ExtractVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["ElementIndex"] = "1", ["ValidateIterResult"] = "Helpers.ExtractVector(firstOp, secondOp, ElementIndex, i) != result[i]"}), ("SveExtractVectorTest.template", new Dictionary { ["TestName"] = "SveExtractVector_UInt64_1", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "ExtractVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()", ["ElementIndex"] = "1", ["ValidateIterResult"] = "Helpers.ExtractVector(firstOp, secondOp, ElementIndex, i) != result[i]"}), + ("SveFfrTest.template", new Dictionary { ["TestName"] = "Sve_Ffr_byte", ["VectorBaseType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskByte()"}), + ("SveFfrTest.template", new Dictionary { ["TestName"] = "Sve_Ffr_short", ["VectorBaseType"] = "Int16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt16()"}), + ("SveFfrTest.template", new Dictionary { ["TestName"] = "Sve_Ffr_int", ["VectorBaseType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt32()"}), + ("SveFfrTest.template", new Dictionary { ["TestName"] = "Sve_Ffr_long", ["VectorBaseType"] = "Int64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt64()"}), + ("SveFfrTest.template", new Dictionary { ["TestName"] = "Sve_Ffr_sbyte", ["VectorBaseType"] = "SByte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskSByte()"}), + ("SveFfrTest.template", new Dictionary { ["TestName"] = "Sve_Ffr_ushort", ["VectorBaseType"] = "UInt16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt16()"}), + ("SveFfrTest.template", new Dictionary { ["TestName"] = "Sve_Ffr_uint", ["VectorBaseType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt32()"}), + ("SveFfrTest.template", new Dictionary { ["TestName"] = "Sve_Ffr_ulong", ["VectorBaseType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt64()"}), + // ("SveGatherVectorVectorBases.template", new Dictionary { ["TestName"] = "Sve_GatherVector_Bases_float_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt32", ["ExtendedElementType"] = "Single", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskSingle()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["NextValueBase"] = "TestLibrary.Generator.GetSingle()"}), // ("SveGatherVectorVectorBases.template", new Dictionary { ["TestName"] = "Sve_GatherVector_Bases_int_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt32", ["ExtendedElementType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["NextValueBase"] = "TestLibrary.Generator.GetInt32()"}), // ("SveGatherVectorVectorBases.template", new Dictionary { ["TestName"] = "Sve_GatherVector_Bases_uint_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt32", ["ExtendedElementType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["NextValueBase"] = "TestLibrary.Generator.GetUInt32()"}), @@ -3607,6 +3616,10 @@ ("SveGatherVectorIndices.template", new Dictionary {["TestName"] = "Sve_GatherVectorUInt32ZeroExtend_Indices_ulong_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorUInt32ZeroExtend", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2BaseType"] = "UInt32", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt64", ["ExtendedElementType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt64()"}), ("SveGatherVectorIndices.template", new Dictionary {["TestName"] = "Sve_GatherVectorUInt32ZeroExtend_Indices_uint_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorUInt32ZeroExtend", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2BaseType"] = "UInt32", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt32", ["ExtendedElementType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt32()"}), + ("SveGatherVectorFirstFaultingVectorBases.template", new Dictionary {["TestName"] = "Sve_GatherVectorFirstFaulting_Bases_double_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt64", ["ExtendedElementType"] = "Double", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskDouble()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()", ["NextValueBase"] = "TestLibrary.Generator.GetDouble()"}), + ("SveGatherVectorFirstFaultingVectorBases.template", new Dictionary {["TestName"] = "Sve_GatherVectorFirstFaulting_Bases_long_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt64", ["ExtendedElementType"] = "Int64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()", ["NextValueBase"] = "TestLibrary.Generator.GetInt64()"}), + ("SveGatherVectorFirstFaultingVectorBases.template", new Dictionary {["TestName"] = "Sve_GatherVectorFirstFaulting_Bases_ulong_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt64", ["ExtendedElementType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()", ["NextValueBase"] = "TestLibrary.Generator.GetUInt64()"}), + ("SveGatherVectorFirstFaultingIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_float_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2BaseType"] = "Single", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int32", ["ExtendedElementType"] = "Single", ["GetFfrType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskSingle()", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt32()"}), ("SveGatherVectorFirstFaultingIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_int_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2BaseType"] = "Int32", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int32", ["ExtendedElementType"] = "Int32", ["GetFfrType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt32()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt32()"}), ("SveGatherVectorFirstFaultingIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_uint_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2BaseType"] = "UInt32", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int32", ["ExtendedElementType"] = "UInt32", ["GetFfrType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt32()"}), diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs index 0b2cc688458c2..a4a57bc9e89b0 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs @@ -8119,6 +8119,12 @@ private static T ConditionalSelectResult(T maskResult, T result, T falseResul return (maskResult != T.Zero) ? result : falseResult; } + private static T ConditionalSelectTrueResult(T maskResult, T result, T trueResult) where T : INumberBase + { + return (maskResult != T.Zero) ? trueResult : result; + } + + private static bool CheckLoadVectorBehaviorCore(T[] firstOp, T[] result, Func map) where T : INumberBase { for (var i = 0; i < firstOp.Length; i++) @@ -8143,22 +8149,30 @@ public static bool CheckLoadVectorBehavior(T[] maskOp, T[] firstOp, T[] resul return CheckLoadVectorBehaviorCore(firstOp, result, (i, loadResult) => ConditionalSelectResult(maskOp[i], loadResult, falseOp[i])); } - private static T GetGatherVectorResultByIndex(int index, T[] firstOp, ExtendedElementT[] secondOp, Index[] thirdOp) + private static T GetGatherVectorResultByIndex(int index, T[] mask, ExtendedElementT[] data, Index[] indices) where T : INumberBase where ExtendedElementT : INumberBase where Index : IBinaryInteger { - return (firstOp[index] == T.Zero) ? T.Zero : T.CreateTruncating(secondOp[int.CreateChecked(thirdOp[index])]); + return (mask[index] == T.Zero) ? T.Zero : T.CreateTruncating(data[int.CreateChecked(indices[index])]); } - private static bool CheckGatherVectorBehaviorCore(T[] firstOp, ExtendedElementT[] secondOp, Index[] thirdOp, T[] result, Func map) + private static unsafe T GetGatherVectorBasesResultByIndex(int index, T[] mask, AddressT[] data) where T : INumberBase + where AddressT : unmanaged, INumberBase + where ExtendedElementT : unmanaged, INumberBase + { + return (mask[index] == T.Zero) ? T.Zero : T.CreateTruncating(*(ExtendedElementT*)Unsafe.BitCast(data[index])); + } + + private static bool CheckGatherVectorBehaviorCore(T[] mask, ExtendedElementT[] data, Index[] indices, T[] result, Func map) + where T : INumberBase where ExtendedElementT : INumberBase where Index : IBinaryInteger { - for (var i = 0; i < firstOp.Length; i++) + for (var i = 0; i < mask.Length; i++) { - T gatherResult = GetGatherVectorResultByIndex(i, firstOp, secondOp, thirdOp); + T gatherResult = GetGatherVectorResultByIndex(i, mask, data, indices); gatherResult = map(i, gatherResult); if (result[i] != gatherResult) { @@ -8168,20 +8182,70 @@ private static bool CheckGatherVectorBehaviorCore(T[ return true; } - public static bool CheckGatherVectorBehavior(T[] firstOp, ExtendedElementT[] secondOp, Index[] thirdOp, T[] result) + private static bool CheckGatherVectorBasesBehaviorCore(T[] mask, AddressT[] data, T[] result, Func map) + where T : INumberBase + where AddressT : unmanaged, INumberBase + where ExtendedElementT : unmanaged, INumberBase + { + for (var i = 0; i < mask.Length; i++) + { + T gatherResult = GetGatherVectorBasesResultByIndex(i, mask, data); + gatherResult = map(i, gatherResult); + if (result[i] != gatherResult) + { + return false; + } + } + return true; + } + + public static bool CheckGatherVectorBehavior(T[] mask, ExtendedElementT[] data, Index[] indices, T[] result) where T : INumberBase where ExtendedElementT : INumberBase where Index : IBinaryInteger { - return CheckGatherVectorBehaviorCore(firstOp, secondOp, thirdOp, result, (_, gatherResult) => gatherResult); + return CheckGatherVectorBehaviorCore(mask, data, indices, result, (_, gatherResult) => gatherResult); } - public static bool CheckGatherVectorBehavior(T[] maskOp, T[] firstOp, ExtendedElementT[] secondOp, Index[] thirdOp, T[] falseOp, T[] result) + public static bool CheckGatherVectorConditionalSelectBehavior(T[] cndSelMask, T[] mask, ExtendedElementT[] data, Index[] indices, T[] cndSelFalse, T[] result) where T : INumberBase where ExtendedElementT : INumberBase where Index : IBinaryInteger { - return CheckGatherVectorBehaviorCore(firstOp, secondOp, thirdOp, result, (i, gatherResult) => ConditionalSelectResult(maskOp[i], gatherResult, falseOp[i])); + return CheckGatherVectorBehaviorCore(mask, data, indices, result, (i, gatherResult) => ConditionalSelectResult(cndSelMask[i], gatherResult, cndSelFalse[i])); + } + + public static bool CheckGatherVectorConditionalSelectTrueBehavior(T[] cndSelMask, T[] mask, ExtendedElementT[] data, Index[] indices, T[] cndSelTrue, T[] result) + where T : INumberBase + where ExtendedElementT : INumberBase + where Index : IBinaryInteger + { + return CheckGatherVectorBehaviorCore(mask, data, indices, result, (i, gatherResult) => ConditionalSelectTrueResult(cndSelMask[i], gatherResult, cndSelTrue[i])); + } + + + public static bool CheckGatherVectorBasesBehavior(T[] mask, AddressT[] data, T[] result) + where T : INumberBase + where AddressT : unmanaged, INumberBase + where ExtendedElementT : unmanaged, INumberBase + { + return CheckGatherVectorBasesBehaviorCore(mask, data, result, (_, gatherResult) => gatherResult); + } + + public static bool CheckGatherVectorBasesConditionalSelectBehavior(T[] cndSelMask, T[] mask, AddressT[] data, T[] cndSelFalse, T[] result) + where T : INumberBase + where AddressT : unmanaged, INumberBase + where ExtendedElementT : unmanaged, INumberBase + { + return CheckGatherVectorBasesBehaviorCore(mask, data, result, (i, gatherResult) => ConditionalSelectResult(cndSelMask[i], gatherResult, cndSelFalse[i])); + } + + public static bool CheckGatherVectorBasesConditionalSelectTrueBehavior(T[] cndSelMask, T[] mask, AddressT[] data, T[] cndSelTrue, T[] result) + where T : INumberBase + where AddressT : unmanaged, INumberBase + where ExtendedElementT : unmanaged, INumberBase + { + return CheckGatherVectorBasesBehaviorCore(mask, data, result, (i, gatherResult) => ConditionalSelectTrueResult(cndSelMask[i], gatherResult, cndSelTrue[i])); } private static bool CheckFirstFaultingBehaviorCore(T[] result, Vector faultResult, Func checkIter) where T : INumberBase @@ -8226,12 +8290,20 @@ public static bool CheckLoadVectorFirstFaultingBehavior(T[] firstOp, T[] resu return CheckFirstFaultingBehaviorCore(result, faultResult, i => firstOp[i] == result[i]); } - public static bool CheckGatherVectorFirstFaultingBehavior(T[] firstOp, ExtendedElementT[] secondOp, Index[] thirdOp, T[] result, Vector faultResult) + public static bool CheckGatherVectorFirstFaultingBehavior(T[] mask, ExtendedElementT[] data, Index[] indices, T[] result, Vector faultResult) where T : INumberBase where ExtendedElementT : INumberBase where Index : IBinaryInteger { - return CheckFirstFaultingBehaviorCore(result, faultResult, i => GetGatherVectorResultByIndex(i, firstOp, secondOp, thirdOp) == result[i]); + return CheckFirstFaultingBehaviorCore(result, faultResult, i => GetGatherVectorResultByIndex(i, mask, data, indices) == result[i]); + } + + public static bool CheckGatherVectorBasesFirstFaultingBehavior(T[] mask, AddressT[] data, T[] result, Vector faultResult) + where T : INumberBase + where AddressT : unmanaged, INumberBase + where ExtendedElementT : unmanaged, INumberBase + { + return CheckFirstFaultingBehaviorCore(result, faultResult, i => GetGatherVectorBasesResultByIndex(i, mask, data) == result[i]); } private static byte ConditionalExtract(byte[] op1, byte op2, byte[] op3, bool after) diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveFfrTest.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveFfrTest.template new file mode 100644 index 0000000000000..dfb7e6a2b2629 --- /dev/null +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveFfrTest.template @@ -0,0 +1,325 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +/****************************************************************************** + * This file is auto-generated from a template file by the GenerateTests.csx * + * script in tests\src\JIT\HardwareIntrinsics.Arm\Shared. In order to make * + * changes, please update the corresponding template and run according to the * + * directions listed in the file. * + ******************************************************************************/ + +using System; +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.Arm; +using Xunit; + +namespace JIT.HardwareIntrinsics.Arm +{ + public static partial class Program + { + [Fact] + public static void {TestName}() + { + var test = new SveFfrTest__{TestName}(); + + if (test.IsSupported) + { + // Validates basic functionality works, using Unsafe.Read + test.RunBasicScenario_UnsafeRead(); + + if (Sve.IsSupported) + { + // Validates basic functionality works, using Load + test.RunBasicScenario_Load(); + } + + // Validates calling via reflection works, using Unsafe.Read + test.RunReflectionScenario_UnsafeRead(); + + // Validates passing a local works, using Unsafe.Read + test.RunLclVarScenario_UnsafeRead(); + + // Validates passing an instance member of a class works + test.RunClassFldScenario(); + + // Validates passing the field of a local struct works + test.RunStructLclFldScenario(); + + // Validates passing an instance member of a struct works + test.RunStructFldScenario(); + } + else + { + // Validates we throw on unsupported hardware + test.RunUnsupportedScenario(); + } + + if (!test.Succeeded) + { + throw new Exception("One or more scenarios did not complete as expected."); + } + } + } + + public sealed unsafe class SveFfrTest__{TestName} + { + private struct DataTable + { + private byte[] inArray1; + private byte[] outArray; + + private GCHandle inHandle1; + private GCHandle outHandle; + + private ulong alignment; + + public DataTable({VectorBaseType}[] inArray1, {VectorBaseType}[] outArray, int alignment) + { + int sizeOfinArray1 = inArray1.Length * Unsafe.SizeOf<{VectorBaseType}>(); + int sizeOfoutArray = outArray.Length * Unsafe.SizeOf<{VectorBaseType}>(); + if ((alignment != 64 && alignment != 16 && alignment != 8) || (alignment * 2) < sizeOfinArray1 || (alignment * 2) < sizeOfoutArray) + { + throw new ArgumentException($"Invalid value of alignment: {alignment}, sizeOfinArray1: {sizeOfinArray1}, sizeOfoutArray: {sizeOfoutArray}"); + } + + this.inArray1 = new byte[alignment * 2]; + this.outArray = new byte[alignment * 2]; + + this.inHandle1 = GCHandle.Alloc(this.inArray1, GCHandleType.Pinned); + this.outHandle = GCHandle.Alloc(this.outArray, GCHandleType.Pinned); + + this.alignment = (ulong)alignment; + + Unsafe.CopyBlockUnaligned(ref Unsafe.AsRef(inArray1Ptr), ref Unsafe.As<{VectorBaseType}, byte>(ref inArray1[0]), (uint)sizeOfinArray1); + } + + public void* inArray1Ptr => Align((byte*)(inHandle1.AddrOfPinnedObject().ToPointer()), alignment); + public void* outArrayPtr => Align((byte*)(outHandle.AddrOfPinnedObject().ToPointer()), alignment); + + public void Dispose() + { + inHandle1.Free(); + outHandle.Free(); + } + + private static unsafe void* Align(byte* buffer, ulong expectedAlignment) + { + return (void*)(((ulong)buffer + expectedAlignment - 1) & ~(expectedAlignment - 1)); + } + } + + private struct TestStruct + { + public Vector<{VectorBaseType}> _fld1; + + public static TestStruct Create() + { + var testStruct = new TestStruct(); + + for (var i = 0; i < Op1ElementCount; i++) { _data1[i] = {NextValueOp1}; } + Unsafe.CopyBlockUnaligned(ref Unsafe.As, byte>(ref testStruct._fld1), ref Unsafe.As<{VectorBaseType}, byte>(ref _data1[0]), (uint)Unsafe.SizeOf>()); + + return testStruct; + } + + public void RunStructFldScenario(SveFfrTest__{TestName} testClass) + { + Sve.SetFfr(_fld1); + var result = Sve.GetFfr{VectorBaseType}(); + + Unsafe.Write(testClass._dataTable.outArrayPtr, result); + testClass.ValidateResult(_fld1, testClass._dataTable.outArrayPtr); + } + } + + private static readonly int LargestVectorSize = {LargestVectorSize}; + + private static readonly int Op1ElementCount = Unsafe.SizeOf>() / sizeof({VectorBaseType}); + private static readonly int RetElementCount = Unsafe.SizeOf>() / sizeof({VectorBaseType}); + + private static {VectorBaseType}[] _maskData = new {VectorBaseType}[Op1ElementCount]; + private static {VectorBaseType}[] _data1 = new {VectorBaseType}[Op1ElementCount]; + + private Vector<{VectorBaseType}> _mask; + private Vector<{VectorBaseType}> _fld1; + private Vector<{VectorBaseType}> _falseFld; + + private DataTable _dataTable; + + public SveFfrTest__{TestName}() + { + Succeeded = true; + + for (var i = 0; i < Op1ElementCount; i++) { _maskData[i] = ({VectorBaseType})({NextValueOp1} % 2); } + Unsafe.CopyBlockUnaligned(ref Unsafe.As, byte>(ref _mask), ref Unsafe.As<{VectorBaseType}, byte>(ref _maskData[0]), (uint)Unsafe.SizeOf>()); + for (var i = 0; i < Op1ElementCount; i++) { _data1[i] = {NextValueOp1}; } + Unsafe.CopyBlockUnaligned(ref Unsafe.As, byte>(ref _fld1), ref Unsafe.As<{VectorBaseType}, byte>(ref _data1[0]), (uint)Unsafe.SizeOf>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As, byte>(ref _falseFld), ref Unsafe.As<{VectorBaseType}, byte>(ref _data1[0]), (uint)Unsafe.SizeOf>()); + + for (var i = 0; i < Op1ElementCount; i++) { _data1[i] = {NextValueOp1}; } + _dataTable = new DataTable(_data1, new {VectorBaseType}[RetElementCount], LargestVectorSize); + } + + public bool IsSupported => Sve.IsSupported; + + public bool Succeeded { get; set; } + + public void RunBasicScenario_UnsafeRead() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunBasicScenario_UnsafeRead)); + + Sve.SetFfr( + Unsafe.Read>(_dataTable.inArray1Ptr) + ); + var result = Sve.GetFfr{VectorBaseType}(); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateResult(_dataTable.inArray1Ptr, _dataTable.outArrayPtr); + } + + public void RunBasicScenario_Load() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunBasicScenario_Load)); + + Vector<{VectorBaseType}> loadMask = Sve.CreateTrueMask{VectorBaseType}(SveMaskPattern.All); + + Sve.SetFfr( + Sve.LoadVector(loadMask, ({VectorBaseType}*)(_dataTable.inArray1Ptr)) + ); + var result = Sve.GetFfr{VectorBaseType}(); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateResult(_dataTable.inArray1Ptr, _dataTable.outArrayPtr); + } + + public void RunReflectionScenario_UnsafeRead() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunReflectionScenario_UnsafeRead)); + + typeof(Sve).GetMethod(nameof(Sve.SetFfr), new Type[] { typeof(Vector<{VectorBaseType}>) }) + .Invoke(null, new object[] { + Unsafe.Read>(_dataTable.inArray1Ptr) + }); + + var result = typeof(Sve).GetMethod(nameof(Sve.GetFfr{VectorBaseType}), new Type[] { }) + .Invoke(null, new object[] { }); + + // We cannot validate the results because the FFR register is trashed. + // Unsafe.Write(_dataTable.outArrayPtr, (Vector<{VectorBaseType}>)(result)); + // ValidateResult(_dataTable.inArray1Ptr, _dataTable.outArrayPtr); + } + + public void RunLclVarScenario_UnsafeRead() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunLclVarScenario_UnsafeRead)); + + var op1 = Unsafe.Read>(_dataTable.inArray1Ptr); + Sve.SetFfr(op1); + var result = Sve.GetFfr{VectorBaseType}(); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateResult(op1, _dataTable.outArrayPtr); + } + + public void RunClassFldScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunClassFldScenario)); + + Sve.SetFfr(_fld1); + var result = Sve.GetFfr{VectorBaseType}(); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateResult(_fld1, _dataTable.outArrayPtr); + } + + public void RunStructLclFldScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunStructLclFldScenario)); + + var test = TestStruct.Create(); + Sve.SetFfr(test._fld1); + var result = Sve.GetFfr{VectorBaseType}(); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateResult(test._fld1, _dataTable.outArrayPtr); + } + + public void RunStructFldScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunStructFldScenario)); + + var test = TestStruct.Create(); + test.RunStructFldScenario(this); + } + + public void RunUnsupportedScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunUnsupportedScenario)); + + bool succeeded = false; + + try + { + RunBasicScenario_UnsafeRead(); + } + catch (PlatformNotSupportedException) + { + succeeded = true; + } + + if (!succeeded) + { + Succeeded = false; + } + } + + private void ValidateResult(Vector<{VectorBaseType}> op1, void* result, [CallerMemberName] string method = "") + { + {VectorBaseType}[] inArray1 = new {VectorBaseType}[Op1ElementCount]; + {VectorBaseType}[] outArray = new {VectorBaseType}[RetElementCount]; + + Unsafe.WriteUnaligned(ref Unsafe.As<{VectorBaseType}, byte>(ref inArray1[0]), op1); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{VectorBaseType}, byte>(ref outArray[0]), ref Unsafe.AsRef(result), (uint)Unsafe.SizeOf>()); + + ValidateResult(inArray1, outArray, method); + } + + private void ValidateResult(void* op1, void* result, [CallerMemberName] string method = "") + { + {VectorBaseType}[] inArray1 = new {VectorBaseType}[Op1ElementCount]; + {VectorBaseType}[] outArray = new {VectorBaseType}[RetElementCount]; + + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{VectorBaseType}, byte>(ref inArray1[0]), ref Unsafe.AsRef(op1), (uint)Unsafe.SizeOf>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{VectorBaseType}, byte>(ref outArray[0]), ref Unsafe.AsRef(result), (uint)Unsafe.SizeOf>()); + + ValidateResult(inArray1, outArray, method); + } + + private void ValidateResult({VectorBaseType}[] firstOp, {VectorBaseType}[] result, [CallerMemberName] string method = "") + { + bool succeeded = true; + + for (var i = 0; i < firstOp.Length; i++) + { + if (firstOp[i] != result[i]) + { + succeeded = false; + break; + } + } + + if (!succeeded) + { + TestLibrary.TestFramework.LogInformation($"{nameof(Sve)}.{nameof(Sve.SetFfr)}<{VectorBaseType}>(Vector<{VectorBaseType}>): SetFfr failed:"); + TestLibrary.TestFramework.LogInformation($" firstOp: ({string.Join(", ", firstOp)})"); + TestLibrary.TestFramework.LogInformation($" result: ({string.Join(", ", result)})"); + TestLibrary.TestFramework.LogInformation(string.Empty); + + Succeeded = false; + } + } + } +} diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorFirstFaultingIndices.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorFirstFaultingIndices.template index 86c2827817d6a..c45dfc62d86d2 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorFirstFaultingIndices.template +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorFirstFaultingIndices.template @@ -34,6 +34,8 @@ namespace JIT.HardwareIntrinsics.Arm // Validates basic functionality works test.RunBasicScenario_Load(); + + // Validates basic functionality of first-faulting behavior test.RunBasicScenario_LoadFirstFaulting(); // Validates fully masked out load works. @@ -580,7 +582,7 @@ namespace JIT.HardwareIntrinsics.Arm private void ValidateConditionalSelectResult({RetBaseType}[] maskOp, {Op1BaseType}[] firstOp, {Op2BaseType}[] secondOp, {Op3BaseType}[] thirdOp, {RetBaseType}[] falseOp, {RetBaseType}[] result, [CallerMemberName] string method = "") { - var succeeded = Helpers.CheckGatherVectorBehavior(maskOp, firstOp, secondOp, thirdOp, falseOp, result); + var succeeded = Helpers.CheckGatherVectorConditionalSelectBehavior(maskOp, firstOp, secondOp, thirdOp, falseOp, result); if (!succeeded) { diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorFirstFaultingVectorBases.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorFirstFaultingVectorBases.template new file mode 100644 index 0000000000000..d4bc14542bef4 --- /dev/null +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorFirstFaultingVectorBases.template @@ -0,0 +1,600 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +/****************************************************************************** + * This file is auto-generated from a template file by the GenerateTests.csx * + * script in tests\src\JIT\HardwareIntrinsics\Arm\Shared. In order to make * + * changes, please update the corresponding template and run according to the * + * directions listed in the file. * + ******************************************************************************/ + +using System; +using System.Numerics; +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.Arm; +using Xunit; + +namespace JIT.HardwareIntrinsics.Arm +{ + public static partial class Program + { + [Fact] + public static void {TestName}() + { + var test = new SveGatherVectorVectorBasesTest__{TestName}(); + + if (test.IsSupported) + { + // Validates basic functionality works, using Unsafe.Read + test.RunBasicScenario_UnsafeRead(); + + // Validates basic functionality works + test.RunBasicScenario_Load(); + + // Validates fully masked out load works. + test.RunBasicScenario_FalseMask(); + + // Validates fully masked out load with invalid address works. + test.RunBasicScenario_NonFaulting(); + + // Validates calling via reflection works, using Unsafe.Read + test.RunReflectionScenario_UnsafeRead(); + + // Validates passing a local works, using Unsafe.Read + test.RunLclVarScenario_UnsafeRead(); + + // Validates passing an instance member of a class works + test.RunClassFldScenario(); + + // Validates passing the field of a local struct works + test.RunStructLclFldScenario(); + + // Validates passing an instance member of a struct works + test.RunStructFldScenario(); + + // Validates using inside ConditionalSelect with value falseValue + // Currently, using this operation in ConditionalSelect() gives incorrect result + // when falseReg == targetReg because this instruction uses Pg/Z to update the targetReg + // instead of Pg/M to merge it. As such, the value of falseReg is lost. Ideally, such + // instructions should be marked similar to RMW (a different flag name) to make sure that + // we do not assign falseReg/targetReg same. Then, we would do something like this: + // + // ldnf1sh target, pg/z, [x0] + // sel mask, target, target, falseReg + // + // This needs more careful thinking, so disabling it for now. + // test.ConditionalSelect_FalseOp(); + + // Validates using inside ConditionalSelect with zero falseValue + test.ConditionalSelect_ZeroOp(); + } + else + { + // Validates we throw on unsupported hardware + test.RunUnsupportedScenario(); + } + + if (!test.Succeeded) + { + throw new Exception("One or more scenarios did not complete as expected."); + } + } + } + + public sealed unsafe class SveGatherVectorVectorBasesTest__{TestName} + { + private struct DataTable + { + private byte[] inArray1; + private byte[] inArray2; + private byte[] baseArray; + private byte[] outArray; + + private GCHandle inHandle1; + private GCHandle inHandle2; + private GCHandle baseHandle; + private GCHandle outHandle; + + private ulong alignment; + + public DataTable({Op1BaseType}[] inArray1, {Op2BaseType}[] inArray2, {RetBaseType}[] baseArray, {RetBaseType}[] outArray, int alignment) + { + int sizeOfinArray1 = inArray1.Length * Unsafe.SizeOf<{Op1BaseType}>(); + int sizeOfinArray2 = inArray2.Length * Unsafe.SizeOf<{Op2BaseType}>(); + int sizeOfBaseArray = baseArray.Length * Unsafe.SizeOf<{RetBaseType}>(); + int sizeOfoutArray = outArray.Length * Unsafe.SizeOf<{RetBaseType}>(); + + if ((alignment != 64 && alignment != 16 && alignment != 8) || (alignment * 2) < sizeOfinArray1 || (alignment * 2) < sizeOfinArray2 || (alignment * 2) < sizeOfoutArray) + { + throw new ArgumentException($"Invalid value of alignment: {alignment}, sizeOfinArray1: {sizeOfinArray1}, sizeOfinArray2: {sizeOfinArray2}, sizeOfoutArray: {sizeOfoutArray}"); + } + + this.inArray1 = new byte[alignment * 2]; + this.inArray2 = new byte[alignment * 2]; + this.baseArray = new byte[alignment + sizeOfBaseArray]; + this.outArray = new byte[alignment * 2]; + + this.inHandle1 = GCHandle.Alloc(this.inArray1, GCHandleType.Pinned); + this.inHandle2 = GCHandle.Alloc(this.inArray2, GCHandleType.Pinned); + this.baseHandle = GCHandle.Alloc(this.baseArray, GCHandleType.Pinned); + this.outHandle = GCHandle.Alloc(this.outArray, GCHandleType.Pinned); + + this.alignment = (ulong)alignment; + + Unsafe.CopyBlockUnaligned(ref Unsafe.AsRef(inArray1Ptr), ref Unsafe.As<{Op1BaseType}, byte>(ref inArray1[0]), (uint)sizeOfinArray1); + Unsafe.CopyBlockUnaligned(ref Unsafe.AsRef(baseArrayPtr), ref Unsafe.As<{RetBaseType}, byte>(ref baseArray[0]), (uint)sizeOfBaseArray); + + // Add the base pointer to the offsets within inArray2 to create complete pointers. + for (var i = 0; i < inArray2.Length; i++) { inArray2[i] += ({Op2BaseType})baseArrayPtr; } + Unsafe.CopyBlockUnaligned(ref Unsafe.AsRef(inArray2Ptr), ref Unsafe.As<{Op2BaseType}, byte>(ref inArray2[0]), (uint)sizeOfinArray2); + } + + public void* inArray1Ptr => Align((byte*)(inHandle1.AddrOfPinnedObject().ToPointer()), alignment); + public void* inArray2Ptr => Align((byte*)(inHandle2.AddrOfPinnedObject().ToPointer()), alignment); + public void* baseArrayPtr => Align((byte*)(baseHandle.AddrOfPinnedObject().ToPointer()), alignment); + public void* outArrayPtr => Align((byte*)(outHandle.AddrOfPinnedObject().ToPointer()), alignment); + + public void Dispose() + { + inHandle1.Free(); + inHandle2.Free(); + baseHandle.Free(); + outHandle.Free(); + } + + private static unsafe void* Align(byte* buffer, ulong expectedAlignment) + { + return (void*)(((ulong)buffer + expectedAlignment - 1) & ~(expectedAlignment - 1)); + } + } + + private struct TestStruct + { + public {Op1VectorType}<{Op1BaseType}> _fld1; + public {Op2VectorType}<{Op2BaseType}> _fld2; + + public static TestStruct Create(DataTable _dataTable) + { + var testStruct = new TestStruct(); + + for (var i = 0; i < BaseElementCount; i++) { _datab[i] = {NextValueBase}; } + for (var i = 0; i < Op1ElementCount; i++) { _data1[i] = {NextValueOp1}; } + for (var i = 0; i < Op2ElementCount; i++) { _data2[i] = {NextValueOp2}; } + + // Fill full of offsets into the data buffer. + for (var i = 0; i < Op2ElementCount; i++) { _data2[i] = {NextValueOp2} % ({Op2BaseType})BaseElementCount; } + + // Add the base pointer to the offsets within inArray2 to create complete pointers. + for (var i = 0; i < Op2ElementCount; i++) { _data2[i] += ({Op2BaseType})_dataTable.baseArrayPtr; } + + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1VectorType}<{Op1BaseType}>, byte>(ref testStruct._fld1), ref Unsafe.As<{Op1BaseType}, byte>(ref _data1[0]), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op2VectorType}<{Op2BaseType}>, byte>(ref testStruct._fld2), ref Unsafe.As<{Op2BaseType}, byte>(ref _data2[0]), (uint)Unsafe.SizeOf<{Op2VectorType}<{Op2BaseType}>>()); + + return testStruct; + } + + public void RunStructFldScenario(SveGatherVectorVectorBasesTest__{TestName} testClass) + { + var result = {Isa}.{Method}(_fld1, _fld2); + + Unsafe.Write(testClass._dataTable.outArrayPtr, result); + testClass.ValidateResult(_fld1, _fld2, testClass._dataTable.outArrayPtr); + } + } + + private static readonly int LargestVectorSize = {LargestVectorSize}; + + // A large enough buffer to hold many values. + // Values in Op2 will point to locations within this buffer. + private static readonly int BaseElementCount = 1024; + private static {RetBaseType}[] _datab = new {RetBaseType}[BaseElementCount]; + + private static readonly int Op1ElementCount = Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>() / sizeof({Op1BaseType}); + private static readonly int Op2ElementCount = Unsafe.SizeOf<{Op2VectorType}<{Op2BaseType}>>() / sizeof({Op2BaseType}); + private static readonly int RetElementCount = Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>() / sizeof({RetBaseType}); + + private static {Op1BaseType}[] _data1 = new {Op1BaseType}[Op1ElementCount]; + private static {Op2BaseType}[] _data2 = new {Op2BaseType}[Op2ElementCount]; + + private static {RetBaseType}[] _maskData = new {RetBaseType}[RetElementCount]; + private static {RetBaseType}[] _falseData = new {RetBaseType}[RetElementCount]; + + private {Op1VectorType}<{Op1BaseType}> _fld1; + private {Op2VectorType}<{Op2BaseType}> _fld2; + + private {Op1VectorType}<{Op1BaseType}> _mask; + private {Op1VectorType}<{Op1BaseType}> _falseFld; + + private DataTable _dataTable; + + public SveGatherVectorVectorBasesTest__{TestName}() + { + Succeeded = true; + + for (var i = 0; i < BaseElementCount; i++) { _datab[i] = {NextValueBase}; } + for (var i = 0; i < Op1ElementCount; i++) { _data1[i] = {NextValueOp1}; } + + // Fill full of offsets into the data buffer. They wil be expanded to full pointers inside the DataTable constructor. + for (var i = 0; i < Op2ElementCount; i++) { _data2[i] = {NextValueOp2} % ({Op2BaseType})BaseElementCount; } + + for (var i = 0; i < RetElementCount; i++) { _maskData[i] = ({RetBaseType})({NextValueOp1}); } + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetVectorType}<{RetBaseType}>, byte>(ref _mask), ref Unsafe.As<{RetBaseType}, byte>(ref _maskData[0]), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); + + for (var i = 0; i < RetElementCount; i++) { _falseData[i] = ({RetBaseType})({NextValueOp2}); } + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetVectorType}<{RetBaseType}>, byte>(ref _falseFld), ref Unsafe.As<{RetBaseType}, byte>(ref _falseData[0]), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); + + _dataTable = new DataTable(_data1, _data2, _datab, new {RetBaseType}[RetElementCount], LargestVectorSize); + + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1VectorType}<{Op1BaseType}>, byte>(ref _fld1), ref Unsafe.As<{Op1BaseType}, byte>(ref _data1[0]), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op2VectorType}<{Op2BaseType}>, byte>(ref _fld2), ref Unsafe.As<{Op2BaseType}, byte>(ref _data2[0]), (uint)Unsafe.SizeOf<{Op2VectorType}<{Op2BaseType}>>()); + } + + public bool IsSupported => {Isa}.IsSupported; + + public bool Succeeded { get; set; } + + public void RunBasicScenario_UnsafeRead() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunBasicScenario_UnsafeRead)); + + var result = {Isa}.{Method}( + Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inArray1Ptr), + Unsafe.Read<{Op2VectorType}<{Op2BaseType}>>(_dataTable.inArray2Ptr) + ); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateResult(_dataTable.inArray1Ptr, _dataTable.inArray2Ptr, _dataTable.outArrayPtr); + } + + public void RunBasicScenario_Load() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunBasicScenario_Load)); + + {Op1VectorType}<{Op1BaseType}> loadMask1 = Sve.CreateTrueMask{Op1BaseType}(SveMaskPattern.All); + {Op2VectorType}<{Op2BaseType}> loadMask2 = Sve.CreateTrueMask{Op2BaseType}(SveMaskPattern.All); + + var result = {Isa}.{Method}( + {LoadIsa}.Load{Op1VectorType}(loadMask1, ({Op1BaseType}*)(_dataTable.inArray1Ptr)), + {LoadIsa}.Load{Op2VectorType}(loadMask2, ({Op2BaseType}*)(_dataTable.inArray2Ptr)) + ); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateResult(_dataTable.inArray1Ptr, _dataTable.inArray2Ptr, _dataTable.outArrayPtr); + } + + + public void RunBasicScenario_FalseMask() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunBasicScenario_FalseMask)); + + {Op1VectorType}<{Op1BaseType}> falseMask = Sve.CreateFalseMask{Op1BaseType}(); + + var result = {Isa}.{Method}( + falseMask, + Unsafe.Read<{Op2VectorType}<{Op2BaseType}>>(_dataTable.inArray2Ptr) + ); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateZeroResult(_dataTable.inArray1Ptr, _dataTable.inArray2Ptr, _dataTable.outArrayPtr); + } + + public void RunBasicScenario_NonFaulting() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunBasicScenario_NonFaulting)); + + {Op1VectorType}<{Op1BaseType}> falseMask = Sve.CreateFalseMask{Op1BaseType}(); + + try + { + var result = {Isa}.{Method}( + falseMask, + {Op2VectorType}<{Op2BaseType}>.Zero + ); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateZeroResult(_dataTable.inArray1Ptr, _dataTable.inArray2Ptr, _dataTable.outArrayPtr); + } + catch + { + Succeeded = false; + } + } + + public void RunReflectionScenario_UnsafeRead() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunReflectionScenario_UnsafeRead)); + + var result = typeof({Isa}).GetMethod(nameof({Isa}.{Method}), new Type[] { typeof({Op1VectorType}<{Op1BaseType}>), typeof({Op1VectorType}<{Op2BaseType}>) }) + .Invoke(null, new object[] { + Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inArray1Ptr), + Unsafe.Read<{Op2VectorType}<{Op2BaseType}>>(_dataTable.inArray2Ptr), + }); + + Unsafe.Write(_dataTable.outArrayPtr, ({RetVectorType}<{RetBaseType}>)(result)); + ValidateResult(_dataTable.inArray1Ptr, _dataTable.inArray2Ptr, _dataTable.outArrayPtr); + } + + public void RunLclVarScenario_UnsafeRead() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunLclVarScenario_UnsafeRead)); + + var op1 = Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inArray1Ptr); + var op2 = Unsafe.Read<{Op2VectorType}<{Op2BaseType}>>(_dataTable.inArray2Ptr); + var result = {Isa}.{Method}(op1, op2); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateResult(op1, op2, _dataTable.outArrayPtr); + } + + public void RunClassFldScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunClassFldScenario)); + + var result = {Isa}.{Method}(_fld1, _fld2); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateResult(_fld1, _fld2, _dataTable.outArrayPtr); + } + + public void RunStructLclFldScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunStructLclFldScenario)); + + var test = TestStruct.Create(_dataTable); + var result = {Isa}.{Method}(test._fld1, test._fld2); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateResult(test._fld1, test._fld2, _dataTable.outArrayPtr); + } + + public void RunStructFldScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunStructFldScenario)); + + var test = TestStruct.Create(_dataTable); + test.RunStructFldScenario(this); + } + + public void ConditionalSelect_FalseOp() + { + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_mask - operation in TrueValue"); + ConditionalSelectScenario_TrueValue(_mask, _fld1, _fld2, _falseFld); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_zero - operation in TrueValue"); + ConditionalSelectScenario_TrueValue({Op1VectorType}<{Op1BaseType}>.Zero, _fld1, _fld2, _falseFld); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_all - operation in TrueValue"); + ConditionalSelectScenario_TrueValue({Op1VectorType}<{Op1BaseType}>.AllBitsSet, _fld1, _fld2, _falseFld); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_mask - operation in FalseValue"); + ConditionalSelectScenario_FalseValue(_mask, _fld1, _fld2, _falseFld); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_zero - operation in FalseValue"); + ConditionalSelectScenario_FalseValue({Op1VectorType}<{Op1BaseType}>.Zero, _fld1, _fld2, _falseFld); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_all - operation in FalseValue"); + ConditionalSelectScenario_FalseValue({Op1VectorType}<{Op1BaseType}>.AllBitsSet, _fld1, _fld2, _falseFld); + } + + public void ConditionalSelect_ZeroOp() + { + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_mask - operation in TrueValue"); + ConditionalSelectScenario_TrueValue(_mask, _fld1, _fld2, {Op1VectorType}<{RetBaseType}>.Zero); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_zero - operation in TrueValue"); + ConditionalSelectScenario_TrueValue({Op1VectorType}<{Op1BaseType}>.Zero, _fld1, _fld2, {Op1VectorType}<{Op1BaseType}>.Zero); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_all - operation in TrueValue"); + ConditionalSelectScenario_TrueValue({Op1VectorType}<{Op1BaseType}>.AllBitsSet, _fld1, _fld2, {Op1VectorType}<{Op1BaseType}>.Zero); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_mask - operation in FalseValue"); + ConditionalSelectScenario_FalseValue(_mask, _fld1, _fld2, {Op1VectorType}<{RetBaseType}>.Zero); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_zero - operation in FalseValue"); + ConditionalSelectScenario_FalseValue({Op1VectorType}<{Op1BaseType}>.Zero, _fld1, _fld2, {Op1VectorType}<{Op1BaseType}>.Zero); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_all - operation in FalseValue"); + ConditionalSelectScenario_FalseValue({Op1VectorType}<{Op1BaseType}>.AllBitsSet, _fld1, _fld2, {Op1VectorType}<{Op1BaseType}>.Zero); + } + + [method: MethodImpl(MethodImplOptions.AggressiveInlining)] + private void ConditionalSelectScenario_TrueValue({RetVectorType}<{RetBaseType}> mask, {Op1VectorType}<{Op1BaseType}> op1, {Op2VectorType}<{Op2BaseType}> op2, {Op1VectorType}<{Op1BaseType}> falseOp) + { + var result = Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2), falseOp); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateConditionalSelectResult_TrueValue(mask, op1, op2, falseOp, _dataTable.outArrayPtr); + } + + [method: MethodImpl(MethodImplOptions.AggressiveInlining)] + private void ConditionalSelectScenario_FalseValue({RetVectorType}<{RetBaseType}> mask, {Op1VectorType}<{Op1BaseType}> op1, {Op2VectorType}<{Op2BaseType}> op2, {Op1VectorType}<{Op1BaseType}> trueOp) + { + var result = Sve.ConditionalSelect(mask, trueOp, {Isa}.{Method}(op1, op2)); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateConditionalSelectResult_FalseValue(mask, op1, op2, trueOp, _dataTable.outArrayPtr); + } + + public void RunUnsupportedScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunUnsupportedScenario)); + + Succeeded = false; + + try + { + RunBasicScenario_Load(); + } + catch (PlatformNotSupportedException) + { + Succeeded = true; + } + } + + private void ValidateResult({Op1VectorType}<{Op1BaseType}> op1, {Op2VectorType}<{Op2BaseType}> op2, void* result, [CallerMemberName] string method = "") + { + {Op1BaseType}[] inArray1 = new {Op1BaseType}[Op1ElementCount]; + {Op2BaseType}[] inArray2 = new {Op2BaseType}[Op2ElementCount]; + {RetBaseType}[] outArray = new {RetBaseType}[RetElementCount]; + + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref inArray1[0]), op1); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op2BaseType}, byte>(ref inArray2[0]), op2); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref outArray[0]), ref Unsafe.AsRef(result), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); + + ValidateResult(inArray1, inArray2, outArray, method); + } + + private void ValidateResult(void* firstOp, void* secondOp, void* result, [CallerMemberName] string method = "") + { + {Op1BaseType}[] inArray1 = new {Op1BaseType}[Op1ElementCount]; + {Op2BaseType}[] inArray2 = new {Op2BaseType}[Op2ElementCount]; + {RetBaseType}[] outArray = new {RetBaseType}[RetElementCount]; + + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref inArray1[0]), ref Unsafe.AsRef(firstOp), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op2BaseType}, byte>(ref inArray2[0]), ref Unsafe.AsRef(secondOp), (uint)(Unsafe.SizeOf<{Op2VectorType}<{Op2BaseType}>>())); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref outArray[0]), ref Unsafe.AsRef(result), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); + + ValidateResult(inArray1, inArray2, outArray, method); + } + + private void ValidateResult({Op1BaseType}[] firstOp, {Op2BaseType}[] secondOp, {RetBaseType}[] result, [CallerMemberName] string method = "") + { + var succeeded = Helpers.CheckGatherVectorBasesBehavior<{Op1BaseType}, {Op2BaseType}, {ExtendedElementType}>(firstOp, secondOp, result); + + if (!succeeded) + { + TestLibrary.TestFramework.LogInformation($"{nameof({Isa})}.{nameof({Isa}.{Method})}<{RetBaseType}>({Op1BaseType}, {Op2BaseType}, {Op2BaseType}): {method} failed:"); + TestLibrary.TestFramework.LogInformation($" firstOp: ({string.Join(", ", firstOp)})"); + TestLibrary.TestFramework.LogInformation($" secondOp: ({string.Join(", ", secondOp)})"); + TestLibrary.TestFramework.LogInformation($" result: ({string.Join(", ", result)})"); + TestLibrary.TestFramework.LogInformation(string.Empty); + + Succeeded = false; + } + } + + private void ValidateZeroResult({Op1VectorType}<{Op1BaseType}> op1, {Op2VectorType}<{Op2BaseType}> op2, void* result, [CallerMemberName] string method = "") + { + {Op1BaseType}[] inArray1 = new {Op1BaseType}[Op1ElementCount]; + {Op2BaseType}[] inArray2 = new {Op2BaseType}[Op2ElementCount]; + {RetBaseType}[] outArray = new {RetBaseType}[RetElementCount]; + + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref inArray1[0]), op1); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op2BaseType}, byte>(ref inArray2[0]), op2); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref outArray[0]), ref Unsafe.AsRef(result), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); + + ValidateZeroResult(inArray1, inArray2, outArray, method); + } + + private void ValidateZeroResult(void* firstOp, void* secondOp, void* result, [CallerMemberName] string method = "") + { + {Op1BaseType}[] inArray1 = new {Op1BaseType}[Op1ElementCount]; + {Op2BaseType}[] inArray2 = new {Op2BaseType}[Op2ElementCount]; + {RetBaseType}[] outArray = new {RetBaseType}[RetElementCount]; + + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref inArray1[0]), ref Unsafe.AsRef(firstOp), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op2BaseType}, byte>(ref inArray2[0]), ref Unsafe.AsRef(secondOp), (uint)(Unsafe.SizeOf<{Op2VectorType}<{Op2BaseType}>>())); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref outArray[0]), ref Unsafe.AsRef(result), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); + + ValidateZeroResult(inArray1, inArray2, outArray, method); + } + + private void ValidateZeroResult({Op1BaseType}[] firstOp, {Op2BaseType}[] secondOp, {RetBaseType}[] result, [CallerMemberName] string method = "") + { + bool succeeded = true; + + for (var i = 0; i < RetElementCount; i++) + { + if (result[i] != 0) + { + succeeded = false; + break; + } + } + + if (!succeeded) + { + TestLibrary.TestFramework.LogInformation($"{nameof({Isa})}.{nameof({Isa}.{Method})}<{RetBaseType}>({Op1BaseType}, {Op2BaseType}, {Op2BaseType}): {method} failed:"); + TestLibrary.TestFramework.LogInformation($" firstOp: ({string.Join(", ", firstOp)})"); + TestLibrary.TestFramework.LogInformation($" secondOp: ({string.Join(", ", secondOp)})"); + TestLibrary.TestFramework.LogInformation($" result: ({string.Join(", ", result)})"); + TestLibrary.TestFramework.LogInformation(string.Empty); + + Succeeded = false; + } + } + + private void ValidateConditionalSelectResult_TrueValue({RetVectorType}<{RetBaseType}> maskOp, {Op1VectorType}<{Op1BaseType}> op1, {Op2VectorType}<{Op2BaseType}> op2, {RetVectorType}<{RetBaseType}> falseOp, void* result, [CallerMemberName] string method = "") + { + {RetBaseType}[] maskArray = new {RetBaseType}[RetElementCount]; + {Op1BaseType}[] op1Array = new {Op1BaseType}[Op1ElementCount]; + {Op2BaseType}[] op2Array = new {Op2BaseType}[Op2ElementCount]; + {RetBaseType}[] falseValArray = new {RetBaseType}[RetElementCount]; + {RetBaseType}[] resultArray = new {RetBaseType}[RetElementCount]; + + Unsafe.WriteUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref maskArray[0]), maskOp); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref op1Array[0]), op1); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op2BaseType}, byte>(ref op2Array[0]), op2); + Unsafe.WriteUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref falseValArray[0]), falseOp); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref resultArray[0]), ref Unsafe.AsRef(result), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); + + ValidateConditionalSelectResult_TrueValue(maskArray, op1Array, op2Array, falseValArray, resultArray, method); + } + + private void ValidateConditionalSelectResult_TrueValue({RetBaseType}[] maskOp, {Op1BaseType}[] firstOp, {Op2BaseType}[] secondOp, {RetBaseType}[] falseOp, {RetBaseType}[] result, [CallerMemberName] string method = "") + { + var succeeded = Helpers.CheckGatherVectorBasesConditionalSelectBehavior<{Op1BaseType}, {Op2BaseType}, {ExtendedElementType}>(maskOp, firstOp, secondOp, falseOp, result); + + if (!succeeded) + { + TestLibrary.TestFramework.LogInformation($"{nameof({Isa})}.{nameof({Isa}.{Method})}<{RetBaseType}>({Op1BaseType}, {Op2BaseType}, {Op2BaseType}): {method} failed:"); + TestLibrary.TestFramework.LogInformation($" maskOp: ({string.Join(", ", maskOp)})"); + TestLibrary.TestFramework.LogInformation($" firstOp: ({string.Join(", ", firstOp)})"); + TestLibrary.TestFramework.LogInformation($" secondOp: ({string.Join(", ", secondOp)})"); + TestLibrary.TestFramework.LogInformation($" falseOp: ({string.Join(", ", falseOp)})"); + TestLibrary.TestFramework.LogInformation($" result: ({string.Join(", ", result)})"); + TestLibrary.TestFramework.LogInformation(string.Empty); + + Succeeded = false; + } + } + + private void ValidateConditionalSelectResult_FalseValue({RetVectorType}<{RetBaseType}> maskOp, {Op1VectorType}<{Op1BaseType}> op1, {Op2VectorType}<{Op2BaseType}> op2, {RetVectorType}<{RetBaseType}> trueOp, void* result, [CallerMemberName] string method = "") + { + {RetBaseType}[] maskArray = new {RetBaseType}[RetElementCount]; + {Op1BaseType}[] op1Array = new {Op1BaseType}[Op1ElementCount]; + {Op2BaseType}[] op2Array = new {Op2BaseType}[Op2ElementCount]; + {RetBaseType}[] trueValArray = new {RetBaseType}[RetElementCount]; + {RetBaseType}[] resultArray = new {RetBaseType}[RetElementCount]; + + Unsafe.WriteUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref maskArray[0]), maskOp); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref op1Array[0]), op1); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op2BaseType}, byte>(ref op2Array[0]), op2); + Unsafe.WriteUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref trueValArray[0]), trueOp); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref resultArray[0]), ref Unsafe.AsRef(result), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); + + ValidateConditionalSelectResult_FalseValue(maskArray, op1Array, op2Array, trueValArray, resultArray, method); + } + + private void ValidateConditionalSelectResult_FalseValue({RetBaseType}[] maskOp, {Op1BaseType}[] firstOp, {Op2BaseType}[] secondOp, {RetBaseType}[] trueOp, {RetBaseType}[] result, [CallerMemberName] string method = "") + { + var succeeded = Helpers.CheckGatherVectorBasesConditionalSelectTrueBehavior<{Op1BaseType}, {Op2BaseType}, {ExtendedElementType}>(maskOp, firstOp, secondOp, trueOp, result); + + if (!succeeded) + { + TestLibrary.TestFramework.LogInformation($"{nameof({Isa})}.{nameof({Isa}.{Method})}<{RetBaseType}>({Op1BaseType}, {Op2BaseType}, {Op2BaseType}): {method} failed:"); + TestLibrary.TestFramework.LogInformation($" maskOp: ({string.Join(", ", maskOp)})"); + TestLibrary.TestFramework.LogInformation($" firstOp: ({string.Join(", ", firstOp)})"); + TestLibrary.TestFramework.LogInformation($" secondOp: ({string.Join(", ", secondOp)})"); + TestLibrary.TestFramework.LogInformation($" trueOp: ({string.Join(", ", trueOp)})"); + TestLibrary.TestFramework.LogInformation($" result: ({string.Join(", ", result)})"); + TestLibrary.TestFramework.LogInformation(string.Empty); + + Succeeded = false; + } + } + } +} \ No newline at end of file diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorIndices.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorIndices.template index f31b9b63bd52d..094cb1eec218f 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorIndices.template +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorIndices.template @@ -562,7 +562,7 @@ namespace JIT.HardwareIntrinsics.Arm private void ValidateConditionalSelectResult_TrueValue({RetBaseType}[] maskOp, {Op1BaseType}[] firstOp, {Op2BaseType}[] secondOp, {Op3BaseType}[] thirdOp, {RetBaseType}[] falseOp, {RetBaseType}[] result, [CallerMemberName] string method = "") { - var succeeded = Helpers.CheckGatherVectorBehavior(maskOp, firstOp, secondOp, thirdOp, falseOp, result); + var succeeded = Helpers.CheckGatherVectorConditionalSelectBehavior(maskOp, firstOp, secondOp, thirdOp, falseOp, result); if (!succeeded) { @@ -600,21 +600,7 @@ namespace JIT.HardwareIntrinsics.Arm private void ValidateConditionalSelectResult_FalseValue({RetBaseType}[] maskOp, {Op1BaseType}[] firstOp, {Op2BaseType}[] secondOp, {Op3BaseType}[] thirdOp, {RetBaseType}[] trueOp, {RetBaseType}[] result, [CallerMemberName] string method = "") { - bool succeeded = true; - - for (var i = 0; i < RetElementCount; i++) - { - {RetBaseType} gatherResult = ({RetBaseType})(firstOp[i] == 0 ? 0 : ({ExtendedElementType})secondOp[thirdOp[i]]); - {RetBaseType} iterResult = (maskOp[i] != 0) ? trueOp[i] : gatherResult; - if (maskOp[i] != 0) - { - if (iterResult != result[i]) - { - succeeded = false; - break; - } - } - } + var succeeded = Helpers.CheckGatherVectorConditionalSelectTrueBehavior(maskOp, firstOp, secondOp, thirdOp, trueOp, result); if (!succeeded) { diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorVectorBases.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorVectorBases.template index eb5a2b2b65e83..d4bc14542bef4 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorVectorBases.template +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorVectorBases.template @@ -462,17 +462,7 @@ namespace JIT.HardwareIntrinsics.Arm private void ValidateResult({Op1BaseType}[] firstOp, {Op2BaseType}[] secondOp, {RetBaseType}[] result, [CallerMemberName] string method = "") { - bool succeeded = true; - - for (var i = 0; i < RetElementCount; i++) - { - {RetBaseType} gatherResult = ({RetBaseType})(firstOp[i] == 0 ? 0 : *({ExtendedElementType}*)(secondOp[i])); - if (result[i] != gatherResult) - { - succeeded = false; - break; - } - } + var succeeded = Helpers.CheckGatherVectorBasesBehavior<{Op1BaseType}, {Op2BaseType}, {ExtendedElementType}>(firstOp, secondOp, result); if (!succeeded) { @@ -556,18 +546,7 @@ namespace JIT.HardwareIntrinsics.Arm private void ValidateConditionalSelectResult_TrueValue({RetBaseType}[] maskOp, {Op1BaseType}[] firstOp, {Op2BaseType}[] secondOp, {RetBaseType}[] falseOp, {RetBaseType}[] result, [CallerMemberName] string method = "") { - bool succeeded = true; - - for (var i = 0; i < RetElementCount; i++) - { - {RetBaseType} gatherResult = ({RetBaseType})(firstOp[i] == 0 ? 0 : *({ExtendedElementType}*)(secondOp[i])); - {RetBaseType} iterResult = (maskOp[i] != 0) ? gatherResult : falseOp[i]; - if (iterResult != result[i]) - { - succeeded = false; - break; - } - } + var succeeded = Helpers.CheckGatherVectorBasesConditionalSelectBehavior<{Op1BaseType}, {Op2BaseType}, {ExtendedElementType}>(maskOp, firstOp, secondOp, falseOp, result); if (!succeeded) { @@ -602,21 +581,7 @@ namespace JIT.HardwareIntrinsics.Arm private void ValidateConditionalSelectResult_FalseValue({RetBaseType}[] maskOp, {Op1BaseType}[] firstOp, {Op2BaseType}[] secondOp, {RetBaseType}[] trueOp, {RetBaseType}[] result, [CallerMemberName] string method = "") { - bool succeeded = true; - - for (var i = 0; i < RetElementCount; i++) - { - {RetBaseType} gatherResult = ({RetBaseType})(firstOp[i] == 0 ? 0 : *({ExtendedElementType}*)(secondOp[i])); - {RetBaseType} iterResult = (maskOp[i] != 0) ? trueOp[i] : gatherResult; - if (maskOp[i] != 0) - { - if (iterResult != result[i]) - { - succeeded = false; - break; - } - } - } + var succeeded = Helpers.CheckGatherVectorBasesConditionalSelectTrueBehavior<{Op1BaseType}, {Op2BaseType}, {ExtendedElementType}>(maskOp, firstOp, secondOp, trueOp, result); if (!succeeded) { From fc90b2e7c28c1bcbe10fbd0e61c6111e82fad343 Mon Sep 17 00:00:00 2001 From: Mikhail Ablakatov Date: Mon, 15 Jul 2024 15:38:31 +0000 Subject: [PATCH 18/83] JIT ARM64-SVE: Add Sve.LoadVector*FirstFaulting APIs --- src/coreclr/jit/gentree.cpp | 6 + src/coreclr/jit/hwintrinsiccodegenarm64.cpp | 11 ++ src/coreclr/jit/hwintrinsiclistarm64sve.h | 6 + .../Arm/Sve.PlatformNotSupported.cs | 184 ++++++++++++++++++ .../src/System/Runtime/Intrinsics/Arm/Sve.cs | 179 +++++++++++++++++ .../ref/System.Runtime.Intrinsics.cs | 24 +++ .../GenerateHWIntrinsicTests_Arm.cs | 30 +++ .../HardwareIntrinsics/Arm/Shared/Helpers.cs | 39 +++- ...eGatherVectorFirstFaultingIndices.template | 4 +- .../SveLoadVectorFirstFaultingTest.template | 49 +++-- 10 files changed, 496 insertions(+), 36 deletions(-) diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 9aedea5052d6c..c0d631db254b6 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -26586,6 +26586,7 @@ bool GenTreeHWIntrinsic::OperIsMemoryLoad(GenTree** pAddr) const case NI_Sve_LoadVector: case NI_Sve_LoadVectorNonTemporal: case NI_Sve_LoadVector128AndReplicateToVector: + case NI_Sve_LoadVectorByteZeroExtendFirstFaulting: case NI_Sve_LoadVectorByteZeroExtendToInt16: case NI_Sve_LoadVectorByteZeroExtendToInt32: case NI_Sve_LoadVectorByteZeroExtendToInt64: @@ -26593,22 +26594,27 @@ bool GenTreeHWIntrinsic::OperIsMemoryLoad(GenTree** pAddr) const case NI_Sve_LoadVectorByteZeroExtendToUInt32: case NI_Sve_LoadVectorByteZeroExtendToUInt64: case NI_Sve_LoadVectorFirstFaulting: + case NI_Sve_LoadVectorInt16SignExtendFirstFaulting: case NI_Sve_LoadVectorInt16SignExtendToInt32: case NI_Sve_LoadVectorInt16SignExtendToInt64: case NI_Sve_LoadVectorInt16SignExtendToUInt32: case NI_Sve_LoadVectorInt16SignExtendToUInt64: + case NI_Sve_LoadVectorInt32SignExtendFirstFaulting: case NI_Sve_LoadVectorInt32SignExtendToInt64: case NI_Sve_LoadVectorInt32SignExtendToUInt64: + case NI_Sve_LoadVectorSByteSignExtendFirstFaulting: case NI_Sve_LoadVectorSByteSignExtendToInt16: case NI_Sve_LoadVectorSByteSignExtendToInt32: case NI_Sve_LoadVectorSByteSignExtendToInt64: case NI_Sve_LoadVectorSByteSignExtendToUInt16: case NI_Sve_LoadVectorSByteSignExtendToUInt32: case NI_Sve_LoadVectorSByteSignExtendToUInt64: + case NI_Sve_LoadVectorUInt16ZeroExtendFirstFaulting: case NI_Sve_LoadVectorUInt16ZeroExtendToInt32: case NI_Sve_LoadVectorUInt16ZeroExtendToInt64: case NI_Sve_LoadVectorUInt16ZeroExtendToUInt32: case NI_Sve_LoadVectorUInt16ZeroExtendToUInt64: + case NI_Sve_LoadVectorUInt32ZeroExtendFirstFaulting: case NI_Sve_LoadVectorUInt32ZeroExtendToInt64: case NI_Sve_LoadVectorUInt32ZeroExtendToUInt64: case NI_Sve_Load2xVectorAndUnzip: diff --git a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp index 05e333eb335c8..7dd2dae57e9c8 100644 --- a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp @@ -2287,6 +2287,10 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) } case NI_Sve_LoadVectorFirstFaulting: + case NI_Sve_LoadVectorInt16SignExtendFirstFaulting: + case NI_Sve_LoadVectorInt32SignExtendFirstFaulting: + case NI_Sve_LoadVectorUInt16ZeroExtendFirstFaulting: + case NI_Sve_LoadVectorUInt32ZeroExtendFirstFaulting: { assert(op3Reg == REG_NA); insScalableOpts sopt = (opt == INS_OPTS_SCALABLE_B) ? INS_SCALABLE_OPTS_NONE : INS_SCALABLE_OPTS_LSL_N; @@ -2294,6 +2298,13 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) break; } + case NI_Sve_LoadVectorByteZeroExtendFirstFaulting: + case NI_Sve_LoadVectorSByteSignExtendFirstFaulting: + { + GetEmitter()->emitIns_R_R_R_R(ins, emitSize, targetReg, op1Reg, op2Reg, REG_ZR, opt); + break; + } + case NI_Sve_SetFfr: { assert(targetReg == REG_NA); diff --git a/src/coreclr/jit/hwintrinsiclistarm64sve.h b/src/coreclr/jit/hwintrinsiclistarm64sve.h index b7420689beb45..d4d3105e5b4ca 100644 --- a/src/coreclr/jit/hwintrinsiclistarm64sve.h +++ b/src/coreclr/jit/hwintrinsiclistarm64sve.h @@ -140,6 +140,7 @@ HARDWARE_INTRINSIC(Sve, LoadVectorByteNonFaultingZeroExtendToInt64, HARDWARE_INTRINSIC(Sve, LoadVectorByteNonFaultingZeroExtendToUInt16, -1, 1, false, {INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorByteNonFaultingZeroExtendToUInt32, -1, 1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorByteNonFaultingZeroExtendToUInt64, -1, 1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1b, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendFirstFaulting, -1, 2, true, {INS_invalid, INS_invalid, INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1b, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialSideEffectMask) HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendToInt16, -1, 2, false, {INS_invalid, INS_invalid, INS_sve_ld1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendToInt32, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendToInt64, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1b, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) @@ -151,12 +152,14 @@ HARDWARE_INTRINSIC(Sve, LoadVectorInt16NonFaultingSignExtendToInt32, HARDWARE_INTRINSIC(Sve, LoadVectorInt16NonFaultingSignExtendToInt64, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sh, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorInt16NonFaultingSignExtendToUInt32, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sh, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorInt16NonFaultingSignExtendToUInt64, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sh, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorInt16SignExtendFirstFaulting, -1, 2, true, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1sh, INS_sve_ldff1sh, INS_sve_ldff1sh, INS_sve_ldff1sh, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialSideEffectMask) HARDWARE_INTRINSIC(Sve, LoadVectorInt16SignExtendToInt32, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorInt16SignExtendToInt64, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sh, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorInt16SignExtendToUInt32, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sh, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorInt16SignExtendToUInt64, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sh, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorInt32NonFaultingSignExtendToInt64, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sw, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorInt32NonFaultingSignExtendToUInt64, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sw, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorInt32SignExtendFirstFaulting, -1, 2, true, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1sw, INS_sve_ldff1sw, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialSideEffectMask) HARDWARE_INTRINSIC(Sve, LoadVectorInt32SignExtendToInt64, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sw, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorInt32SignExtendToUInt64, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sw, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorNonFaulting, -1, 1, true, {INS_sve_ldnf1b, INS_sve_ldnf1b, INS_sve_ldnf1h, INS_sve_ldnf1h, INS_sve_ldnf1w, INS_sve_ldnf1w, INS_sve_ldnf1d, INS_sve_ldnf1d, INS_sve_ldnf1w, INS_sve_ldnf1d}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) @@ -167,6 +170,7 @@ HARDWARE_INTRINSIC(Sve, LoadVectorSByteNonFaultingSignExtendToInt64, HARDWARE_INTRINSIC(Sve, LoadVectorSByteNonFaultingSignExtendToUInt16, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorSByteNonFaultingSignExtendToUInt32, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorSByteNonFaultingSignExtendToUInt64, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sb, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorSByteSignExtendFirstFaulting, -1, 2, true, {INS_invalid, INS_invalid, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialSideEffectMask) HARDWARE_INTRINSIC(Sve, LoadVectorSByteSignExtendToInt16, -1, 2, false, {INS_invalid, INS_invalid, INS_sve_ld1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorSByteSignExtendToInt32, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorSByteSignExtendToInt64, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sb, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) @@ -177,12 +181,14 @@ HARDWARE_INTRINSIC(Sve, LoadVectorUInt16NonFaultingZeroExtendToInt32, HARDWARE_INTRINSIC(Sve, LoadVectorUInt16NonFaultingZeroExtendToInt64, -1, 1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1h, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorUInt16NonFaultingZeroExtendToUInt32, -1, 1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1h, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorUInt16NonFaultingZeroExtendToUInt64, -1, 1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1h, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorUInt16ZeroExtendFirstFaulting, -1, 2, true, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1h, INS_sve_ldff1h, INS_sve_ldff1h, INS_sve_ldff1h, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialSideEffectMask) HARDWARE_INTRINSIC(Sve, LoadVectorUInt16ZeroExtendToInt32, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1h, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorUInt16ZeroExtendToInt64, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1h, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorUInt16ZeroExtendToUInt32, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1h, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorUInt16ZeroExtendToUInt64, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1h, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorUInt32NonFaultingZeroExtendToInt64, -1, 1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1w, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorUInt32NonFaultingZeroExtendToUInt64, -1, 1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1w, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorUInt32ZeroExtendFirstFaulting, -1, 2, true, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1w, INS_sve_ldff1w, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialSideEffectMask) HARDWARE_INTRINSIC(Sve, LoadVectorUInt32ZeroExtendToInt64, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorUInt32ZeroExtendToUInt64, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, Max, -1, -1, false, {INS_sve_smax, INS_sve_umax, INS_sve_smax, INS_sve_umax, INS_sve_smax, INS_sve_umax, INS_sve_smax, INS_sve_umax, INS_sve_fmax, INS_sve_fmax}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs index 6d0adc1367708..1867400403c3f 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs @@ -5013,6 +5013,48 @@ internal Arm64() { } /// public static unsafe Vector LoadVectorByteNonFaultingZeroExtendToUInt64(byte* address) { throw new PlatformNotSupportedException(); } + /// + /// svint16_t svldff1ub_s16(svbool_t pg, const uint8_t *base) + /// LDFF1B Zresult.H, Pg/Z, [Xarray, Xindex] + /// LDFF1B Zresult.H, Pg/Z, [Xbase, XZR] + /// + public static unsafe Vector LoadVectorByteZeroExtendFirstFaulting(Vector mask, byte* address) { throw new PlatformNotSupportedException(); } + + /// + /// svint32_t svldff1ub_s32(svbool_t pg, const uint8_t *base) + /// LDFF1B Zresult.S, Pg/Z, [Xarray, Xindex] + /// LDFF1B Zresult.S, Pg/Z, [Xbase, XZR] + /// + public static unsafe Vector LoadVectorByteZeroExtendFirstFaulting(Vector mask, byte* address) { throw new PlatformNotSupportedException(); } + + /// + /// svint64_t svldff1ub_s64(svbool_t pg, const uint8_t *base) + /// LDFF1B Zresult.D, Pg/Z, [Xarray, Xindex] + /// LDFF1B Zresult.D, Pg/Z, [Xbase, XZR] + /// + public static unsafe Vector LoadVectorByteZeroExtendFirstFaulting(Vector mask, byte* address) { throw new PlatformNotSupportedException(); } + + /// + /// svuint16_t svldff1ub_u16(svbool_t pg, const uint8_t *base) + /// LDFF1B Zresult.H, Pg/Z, [Xarray, Xindex] + /// LDFF1B Zresult.H, Pg/Z, [Xbase, XZR] + /// + public static unsafe Vector LoadVectorByteZeroExtendFirstFaulting(Vector mask, byte* address) { throw new PlatformNotSupportedException(); } + + /// + /// svuint32_t svldff1ub_u32(svbool_t pg, const uint8_t *base) + /// LDFF1B Zresult.S, Pg/Z, [Xarray, Xindex] + /// LDFF1B Zresult.S, Pg/Z, [Xbase, XZR] + /// + public static unsafe Vector LoadVectorByteZeroExtendFirstFaulting(Vector mask, byte* address) { throw new PlatformNotSupportedException(); } + + /// + /// svuint64_t svldff1ub_u64(svbool_t pg, const uint8_t *base) + /// LDFF1B Zresult.D, Pg/Z, [Xarray, Xindex] + /// LDFF1B Zresult.D, Pg/Z, [Xbase, XZR] + /// + public static unsafe Vector LoadVectorByteZeroExtendFirstFaulting(Vector mask, byte* address) { throw new PlatformNotSupportedException(); } + /// /// svint32_t svldnf1uh_s32(svbool_t pg, const uint16_t *base) /// LDNF1H Zresult.S, Pg/Z, [Xbase, #0, MUL VL] @@ -5037,6 +5079,36 @@ internal Arm64() { } /// public static unsafe Vector LoadVectorUInt16NonFaultingZeroExtendToUInt64(ushort* address) { throw new PlatformNotSupportedException(); } + /// Load 16-bit data and zero-extend, first-faulting + + /// + /// svint32_t svldff1uh_s32(svbool_t pg, const uint16_t *base) + /// LDFF1H Zresult.S, Pg/Z, [Xarray, Xindex, LSL #1] + /// LDFF1H Zresult.S, Pg/Z, [Xbase, XZR, LSL #1] + /// + public static unsafe Vector LoadVectorUInt16ZeroExtendFirstFaulting(Vector mask, ushort* address) { throw new PlatformNotSupportedException(); } + + /// + /// svint64_t svldff1uh_s64(svbool_t pg, const uint16_t *base) + /// LDFF1H Zresult.D, Pg/Z, [Xarray, Xindex, LSL #1] + /// LDFF1H Zresult.D, Pg/Z, [Xbase, XZR, LSL #1] + /// + public static unsafe Vector LoadVectorUInt16ZeroExtendFirstFaulting(Vector mask, ushort* address) { throw new PlatformNotSupportedException(); } + + /// + /// svuint32_t svldff1uh_u32(svbool_t pg, const uint16_t *base) + /// LDFF1H Zresult.S, Pg/Z, [Xarray, Xindex, LSL #1] + /// LDFF1H Zresult.S, Pg/Z, [Xbase, XZR, LSL #1] + /// + public static unsafe Vector LoadVectorUInt16ZeroExtendFirstFaulting(Vector mask, ushort* address) { throw new PlatformNotSupportedException(); } + + /// + /// svuint64_t svldff1uh_u64(svbool_t pg, const uint16_t *base) + /// LDFF1H Zresult.D, Pg/Z, [Xarray, Xindex, LSL #1] + /// LDFF1H Zresult.D, Pg/Z, [Xbase, XZR, LSL #1] + /// + public static unsafe Vector LoadVectorUInt16ZeroExtendFirstFaulting(Vector mask, ushort* address) { throw new PlatformNotSupportedException(); } + /// /// svint64_t svldnf1uw_s64(svbool_t pg, const uint32_t *base) /// LDNF1W Zresult.D, Pg/Z, [Xbase, #0, MUL VL] @@ -5049,6 +5121,22 @@ internal Arm64() { } /// public static unsafe Vector LoadVectorUInt32NonFaultingZeroExtendToUInt64(uint* address) { throw new PlatformNotSupportedException(); } + /// Load 32-bit data and zero-extend, first-faulting + + /// + /// svint64_t svldff1uw_s64(svbool_t pg, const uint32_t *base) + /// LDFF1W Zresult.D, Pg/Z, [Xarray, Xindex, LSL #2] + /// LDFF1W Zresult.D, Pg/Z, [Xbase, XZR, LSL #2] + /// + public static unsafe Vector LoadVectorUInt32ZeroExtendFirstFaulting(Vector mask, uint* address) { throw new PlatformNotSupportedException(); } + + /// + /// svuint64_t svldff1uw_u64(svbool_t pg, const uint32_t *base) + /// LDFF1W Zresult.D, Pg/Z, [Xarray, Xindex, LSL #2] + /// LDFF1W Zresult.D, Pg/Z, [Xbase, XZR, LSL #2] + /// + public static unsafe Vector LoadVectorUInt32ZeroExtendFirstFaulting(Vector mask, uint* address) { throw new PlatformNotSupportedException(); } + /// /// svuint8x2_t svld2[_u8](svbool_t pg, const uint8_t *base) /// LD2B {Zresult0.B, Zresult1.B}, Pg/Z, [Xbase, #0, MUL VL] @@ -5264,6 +5352,38 @@ internal Arm64() { } /// public static unsafe Vector LoadVectorInt16NonFaultingSignExtendToUInt64(short* address) { throw new PlatformNotSupportedException(); } + + /// Load 16-bit data and sign-extend, first-faulting + + /// + /// svint32_t svldff1sh_s32(svbool_t pg, const int16_t *base) + /// LDFF1SH Zresult.S, Pg/Z, [Xarray, Xindex, LSL #1] + /// LDFF1SH Zresult.S, Pg/Z, [Xbase, XZR, LSL #1] + /// + public static unsafe Vector LoadVectorInt16SignExtendFirstFaulting(Vector mask, short* address) { throw new PlatformNotSupportedException(); } + + /// + /// svint64_t svldff1sh_s64(svbool_t pg, const int16_t *base) + /// LDFF1SH Zresult.D, Pg/Z, [Xarray, Xindex, LSL #1] + /// LDFF1SH Zresult.D, Pg/Z, [Xbase, XZR, LSL #1] + /// + public static unsafe Vector LoadVectorInt16SignExtendFirstFaulting(Vector mask, short* address) { throw new PlatformNotSupportedException(); } + + /// + /// svuint32_t svldff1sh_u32(svbool_t pg, const int16_t *base) + /// LDFF1SH Zresult.S, Pg/Z, [Xarray, Xindex, LSL #1] + /// LDFF1SH Zresult.S, Pg/Z, [Xbase, XZR, LSL #1] + /// + public static unsafe Vector LoadVectorInt16SignExtendFirstFaulting(Vector mask, short* address) { throw new PlatformNotSupportedException(); } + + /// + /// svuint64_t svldff1sh_u64(svbool_t pg, const int16_t *base) + /// LDFF1SH Zresult.D, Pg/Z, [Xarray, Xindex, LSL #1] + /// LDFF1SH Zresult.D, Pg/Z, [Xbase, XZR, LSL #1] + /// + public static unsafe Vector LoadVectorInt16SignExtendFirstFaulting(Vector mask, short* address) { throw new PlatformNotSupportedException(); } + + /// Load 32-bit data and sign-extend, non-faulting /// @@ -5280,6 +5400,24 @@ internal Arm64() { } /// public static unsafe Vector LoadVectorInt32NonFaultingSignExtendToUInt64(int* address) { throw new PlatformNotSupportedException(); } + + /// Load 32-bit data and sign-extend, first-faulting + + /// + /// svint64_t svldff1sw_s64(svbool_t pg, const int32_t *base) + /// LDFF1SW Zresult.D, Pg/Z, [Xarray, Xindex, LSL #2] + /// LDFF1SW Zresult.D, Pg/Z, [Xbase, XZR, LSL #2] + /// + public static unsafe Vector LoadVectorInt32SignExtendFirstFaulting(Vector mask, int* address) { throw new PlatformNotSupportedException(); } + + /// + /// svuint64_t svldff1sw_u64(svbool_t pg, const int32_t *base) + /// LDFF1SW Zresult.D, Pg/Z, [Xarray, Xindex, LSL #2] + /// LDFF1SW Zresult.D, Pg/Z, [Xbase, XZR, LSL #2] + /// + public static unsafe Vector LoadVectorInt32SignExtendFirstFaulting(Vector mask, int* address) { throw new PlatformNotSupportedException(); } + + /// Load 8-bit data and sign-extend, non-faulting /// @@ -5332,6 +5470,52 @@ internal Arm64() { } /// public static unsafe Vector LoadVectorSByteNonFaultingSignExtendToUInt64(sbyte* address) { throw new PlatformNotSupportedException(); } + + /// Load 8-bit data and sign-extend, first-faulting + + /// + /// svint16_t svldff1sb_s16(svbool_t pg, const int8_t *base) + /// LDFF1SB Zresult.H, Pg/Z, [Xarray, Xindex] + /// LDFF1SB Zresult.H, Pg/Z, [Xbase, XZR] + /// + public static unsafe Vector LoadVectorSByteSignExtendFirstFaulting(Vector mask, sbyte* address) { throw new PlatformNotSupportedException(); } + + /// + /// svint32_t svldff1sb_s32(svbool_t pg, const int8_t *base) + /// LDFF1SB Zresult.S, Pg/Z, [Xarray, Xindex] + /// LDFF1SB Zresult.S, Pg/Z, [Xbase, XZR] + /// + public static unsafe Vector LoadVectorSByteSignExtendFirstFaulting(Vector mask, sbyte* address) { throw new PlatformNotSupportedException(); } + + /// + /// svint64_t svldff1sb_s64(svbool_t pg, const int8_t *base) + /// LDFF1SB Zresult.D, Pg/Z, [Xarray, Xindex] + /// LDFF1SB Zresult.D, Pg/Z, [Xbase, XZR] + /// + public static unsafe Vector LoadVectorSByteSignExtendFirstFaulting(Vector mask, sbyte* address) { throw new PlatformNotSupportedException(); } + + /// + /// svuint16_t svldff1sb_u16(svbool_t pg, const int8_t *base) + /// LDFF1SB Zresult.H, Pg/Z, [Xarray, Xindex] + /// LDFF1SB Zresult.H, Pg/Z, [Xbase, XZR] + /// + public static unsafe Vector LoadVectorSByteSignExtendFirstFaulting(Vector mask, sbyte* address) { throw new PlatformNotSupportedException(); } + + /// + /// svuint32_t svldff1sb_u32(svbool_t pg, const int8_t *base) + /// LDFF1SB Zresult.S, Pg/Z, [Xarray, Xindex] + /// LDFF1SB Zresult.S, Pg/Z, [Xbase, XZR] + /// + public static unsafe Vector LoadVectorSByteSignExtendFirstFaulting(Vector mask, sbyte* address) { throw new PlatformNotSupportedException(); } + + /// + /// svuint64_t svldff1sb_u64(svbool_t pg, const int8_t *base) + /// LDFF1SB Zresult.D, Pg/Z, [Xarray, Xindex] + /// LDFF1SB Zresult.D, Pg/Z, [Xbase, XZR] + /// + public static unsafe Vector LoadVectorSByteSignExtendFirstFaulting(Vector mask, sbyte* address) { throw new PlatformNotSupportedException(); } + + /// Max : Maximum /// diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs index ed64f367321ef..789109e855dcc 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs @@ -5070,6 +5070,47 @@ internal Arm64() { } /// LDNF1B Zresult.D, Pg/Z, [Xbase, #0, MUL VL] /// public static unsafe Vector LoadVectorByteNonFaultingZeroExtendToUInt64(byte* address) => LoadVectorByteNonFaultingZeroExtendToUInt64(address); + /// + /// svint16_t svldff1ub_s16(svbool_t pg, const uint8_t *base) + /// LDFF1B Zresult.H, Pg/Z, [Xarray, Xindex] + /// LDFF1B Zresult.H, Pg/Z, [Xbase, XZR] + /// + public static unsafe Vector LoadVectorByteZeroExtendFirstFaulting(Vector mask, byte* address) => LoadVectorByteZeroExtendFirstFaulting(mask, address); + + /// + /// svint32_t svldff1ub_s32(svbool_t pg, const uint8_t *base) + /// LDFF1B Zresult.S, Pg/Z, [Xarray, Xindex] + /// LDFF1B Zresult.S, Pg/Z, [Xbase, XZR] + /// + public static unsafe Vector LoadVectorByteZeroExtendFirstFaulting(Vector mask, byte* address) => LoadVectorByteZeroExtendFirstFaulting(mask, address); + + /// + /// svint64_t svldff1ub_s64(svbool_t pg, const uint8_t *base) + /// LDFF1B Zresult.D, Pg/Z, [Xarray, Xindex] + /// LDFF1B Zresult.D, Pg/Z, [Xbase, XZR] + /// + public static unsafe Vector LoadVectorByteZeroExtendFirstFaulting(Vector mask, byte* address) => LoadVectorByteZeroExtendFirstFaulting(mask, address); + + /// + /// svuint16_t svldff1ub_u16(svbool_t pg, const uint8_t *base) + /// LDFF1B Zresult.H, Pg/Z, [Xarray, Xindex] + /// LDFF1B Zresult.H, Pg/Z, [Xbase, XZR] + /// + public static unsafe Vector LoadVectorByteZeroExtendFirstFaulting(Vector mask, byte* address) => LoadVectorByteZeroExtendFirstFaulting(mask, address); + + /// + /// svuint32_t svldff1ub_u32(svbool_t pg, const uint8_t *base) + /// LDFF1B Zresult.S, Pg/Z, [Xarray, Xindex] + /// LDFF1B Zresult.S, Pg/Z, [Xbase, XZR] + /// + public static unsafe Vector LoadVectorByteZeroExtendFirstFaulting(Vector mask, byte* address) => LoadVectorByteZeroExtendFirstFaulting(mask, address); + + /// + /// svuint64_t svldff1ub_u64(svbool_t pg, const uint8_t *base) + /// LDFF1B Zresult.D, Pg/Z, [Xarray, Xindex] + /// LDFF1B Zresult.D, Pg/Z, [Xbase, XZR] + /// + public static unsafe Vector LoadVectorByteZeroExtendFirstFaulting(Vector mask, byte* address) => LoadVectorByteZeroExtendFirstFaulting(mask, address); /// /// svint32_t svldnf1uh_s32(svbool_t pg, const uint16_t *base) @@ -5095,6 +5136,34 @@ internal Arm64() { } /// public static unsafe Vector LoadVectorUInt16NonFaultingZeroExtendToUInt64(ushort* address) => LoadVectorUInt16NonFaultingZeroExtendToUInt64(address); + /// + /// svint32_t svldff1uh_s32(svbool_t pg, const uint16_t *base) + /// LDFF1H Zresult.S, Pg/Z, [Xarray, Xindex, LSL #1] + /// LDFF1H Zresult.S, Pg/Z, [Xbase, XZR, LSL #1] + /// + public static unsafe Vector LoadVectorUInt16ZeroExtendFirstFaulting(Vector mask, ushort* address) => LoadVectorUInt16ZeroExtendFirstFaulting(mask, address); + + /// + /// svint64_t svldff1uh_s64(svbool_t pg, const uint16_t *base) + /// LDFF1H Zresult.D, Pg/Z, [Xarray, Xindex, LSL #1] + /// LDFF1H Zresult.D, Pg/Z, [Xbase, XZR, LSL #1] + /// + public static unsafe Vector LoadVectorUInt16ZeroExtendFirstFaulting(Vector mask, ushort* address) => LoadVectorUInt16ZeroExtendFirstFaulting(mask, address); + + /// + /// svuint32_t svldff1uh_u32(svbool_t pg, const uint16_t *base) + /// LDFF1H Zresult.S, Pg/Z, [Xarray, Xindex, LSL #1] + /// LDFF1H Zresult.S, Pg/Z, [Xbase, XZR, LSL #1] + /// + public static unsafe Vector LoadVectorUInt16ZeroExtendFirstFaulting(Vector mask, ushort* address) => LoadVectorUInt16ZeroExtendFirstFaulting(mask, address); + + /// + /// svuint64_t svldff1uh_u64(svbool_t pg, const uint16_t *base) + /// LDFF1H Zresult.D, Pg/Z, [Xarray, Xindex, LSL #1] + /// LDFF1H Zresult.D, Pg/Z, [Xbase, XZR, LSL #1] + /// + public static unsafe Vector LoadVectorUInt16ZeroExtendFirstFaulting(Vector mask, ushort* address) => LoadVectorUInt16ZeroExtendFirstFaulting(mask, address); + /// /// svint64_t svldnf1uw_s64(svbool_t pg, const uint32_t *base) /// LDNF1W Zresult.D, Pg/Z, [Xbase, #0, MUL VL] @@ -5107,6 +5176,20 @@ internal Arm64() { } /// public static unsafe Vector LoadVectorUInt32NonFaultingZeroExtendToUInt64(uint* address) => LoadVectorUInt32NonFaultingZeroExtendToUInt64(address); + /// + /// svint64_t svldff1uw_s64(svbool_t pg, const uint32_t *base) + /// LDFF1W Zresult.D, Pg/Z, [Xarray, Xindex, LSL #2] + /// LDFF1W Zresult.D, Pg/Z, [Xbase, XZR, LSL #2] + /// + public static unsafe Vector LoadVectorUInt32ZeroExtendFirstFaulting(Vector mask, uint* address) => LoadVectorUInt32ZeroExtendFirstFaulting(mask, address); + + /// + /// svuint64_t svldff1uw_u64(svbool_t pg, const uint32_t *base) + /// LDFF1W Zresult.D, Pg/Z, [Xarray, Xindex, LSL #2] + /// LDFF1W Zresult.D, Pg/Z, [Xbase, XZR, LSL #2] + /// + public static unsafe Vector LoadVectorUInt32ZeroExtendFirstFaulting(Vector mask, uint* address) => LoadVectorUInt32ZeroExtendFirstFaulting(mask, address); + /// /// svuint8x2_t svld2[_u8](svbool_t pg, const uint8_t *base) /// LD2B {Zresult0.B, Zresult1.B}, Pg/Z, [Xbase, #0, MUL VL] @@ -5322,6 +5405,38 @@ internal Arm64() { } /// public static unsafe Vector LoadVectorInt16NonFaultingSignExtendToUInt64(short* address) => LoadVectorInt16NonFaultingSignExtendToUInt64(address); + + /// Load 16-bit data and sign-extend, first-faulting + + /// + /// svint32_t svldff1sh_s32(svbool_t pg, const int16_t *base) + /// LDFF1SH Zresult.S, Pg/Z, [Xarray, Xindex, LSL #1] + /// LDFF1SH Zresult.S, Pg/Z, [Xbase, XZR, LSL #1] + /// + public static unsafe Vector LoadVectorInt16SignExtendFirstFaulting(Vector mask, short* address) => LoadVectorInt16SignExtendFirstFaulting(mask, address); + + /// + /// svint64_t svldff1sh_s64(svbool_t pg, const int16_t *base) + /// LDFF1SH Zresult.D, Pg/Z, [Xarray, Xindex, LSL #1] + /// LDFF1SH Zresult.D, Pg/Z, [Xbase, XZR, LSL #1] + /// + public static unsafe Vector LoadVectorInt16SignExtendFirstFaulting(Vector mask, short* address) => LoadVectorInt16SignExtendFirstFaulting(mask, address); + + /// + /// svuint32_t svldff1sh_u32(svbool_t pg, const int16_t *base) + /// LDFF1SH Zresult.S, Pg/Z, [Xarray, Xindex, LSL #1] + /// LDFF1SH Zresult.S, Pg/Z, [Xbase, XZR, LSL #1] + /// + public static unsafe Vector LoadVectorInt16SignExtendFirstFaulting(Vector mask, short* address) => LoadVectorInt16SignExtendFirstFaulting(mask, address); + + /// + /// svuint64_t svldff1sh_u64(svbool_t pg, const int16_t *base) + /// LDFF1SH Zresult.D, Pg/Z, [Xarray, Xindex, LSL #1] + /// LDFF1SH Zresult.D, Pg/Z, [Xbase, XZR, LSL #1] + /// + public static unsafe Vector LoadVectorInt16SignExtendFirstFaulting(Vector mask, short* address) => LoadVectorInt16SignExtendFirstFaulting(mask, address); + + /// Load 32-bit data and sign-extend, non-faulting /// @@ -5338,6 +5453,24 @@ internal Arm64() { } /// public static unsafe Vector LoadVectorInt32NonFaultingSignExtendToUInt64(int* address) => LoadVectorInt32NonFaultingSignExtendToUInt64(address); + + /// Load 32-bit data and sign-extend, first-faulting + + /// + /// svint64_t svldff1sw_s64(svbool_t pg, const int32_t *base) + /// LDFF1SW Zresult.D, Pg/Z, [Xarray, Xindex, LSL #2] + /// LDFF1SW Zresult.D, Pg/Z, [Xbase, XZR, LSL #2] + /// + public static unsafe Vector LoadVectorInt32SignExtendFirstFaulting(Vector mask, int* address) => LoadVectorInt32SignExtendFirstFaulting(mask, address); + + /// + /// svuint64_t svldff1sw_u64(svbool_t pg, const int32_t *base) + /// LDFF1SW Zresult.D, Pg/Z, [Xarray, Xindex, LSL #2] + /// LDFF1SW Zresult.D, Pg/Z, [Xbase, XZR, LSL #2] + /// + public static unsafe Vector LoadVectorInt32SignExtendFirstFaulting(Vector mask, int* address) => LoadVectorInt32SignExtendFirstFaulting(mask, address); + + /// Load 8-bit data and sign-extend, non-faulting /// @@ -5390,6 +5523,52 @@ internal Arm64() { } /// public static unsafe Vector LoadVectorSByteNonFaultingSignExtendToUInt64(sbyte* address) => LoadVectorSByteNonFaultingSignExtendToUInt64(address); + + /// Load 8-bit data and sign-extend, first-faulting + + /// + /// svint16_t svldff1sb_s16(svbool_t pg, const int8_t *base) + /// LDFF1SB Zresult.H, Pg/Z, [Xarray, Xindex] + /// LDFF1SB Zresult.H, Pg/Z, [Xbase, XZR] + /// + public static unsafe Vector LoadVectorSByteSignExtendFirstFaulting(Vector mask, sbyte* address) => LoadVectorSByteSignExtendFirstFaulting(mask, address); + + /// + /// svint32_t svldff1sb_s32(svbool_t pg, const int8_t *base) + /// LDFF1SB Zresult.S, Pg/Z, [Xarray, Xindex] + /// LDFF1SB Zresult.S, Pg/Z, [Xbase, XZR] + /// + public static unsafe Vector LoadVectorSByteSignExtendFirstFaulting(Vector mask, sbyte* address) => LoadVectorSByteSignExtendFirstFaulting(mask, address); + + /// + /// svint64_t svldff1sb_s64(svbool_t pg, const int8_t *base) + /// LDFF1SB Zresult.D, Pg/Z, [Xarray, Xindex] + /// LDFF1SB Zresult.D, Pg/Z, [Xbase, XZR] + /// + public static unsafe Vector LoadVectorSByteSignExtendFirstFaulting(Vector mask, sbyte* address) => LoadVectorSByteSignExtendFirstFaulting(mask, address); + + /// + /// svuint16_t svldff1sb_u16(svbool_t pg, const int8_t *base) + /// LDFF1SB Zresult.H, Pg/Z, [Xarray, Xindex] + /// LDFF1SB Zresult.H, Pg/Z, [Xbase, XZR] + /// + public static unsafe Vector LoadVectorSByteSignExtendFirstFaulting(Vector mask, sbyte* address) => LoadVectorSByteSignExtendFirstFaulting(mask, address); + + /// + /// svuint32_t svldff1sb_u32(svbool_t pg, const int8_t *base) + /// LDFF1SB Zresult.S, Pg/Z, [Xarray, Xindex] + /// LDFF1SB Zresult.S, Pg/Z, [Xbase, XZR] + /// + public static unsafe Vector LoadVectorSByteSignExtendFirstFaulting(Vector mask, sbyte* address) => LoadVectorSByteSignExtendFirstFaulting(mask, address); + + /// + /// svuint64_t svldff1sb_u64(svbool_t pg, const int8_t *base) + /// LDFF1SB Zresult.D, Pg/Z, [Xarray, Xindex] + /// LDFF1SB Zresult.D, Pg/Z, [Xbase, XZR] + /// + public static unsafe Vector LoadVectorSByteSignExtendFirstFaulting(Vector mask, sbyte* address) => LoadVectorSByteSignExtendFirstFaulting(mask, address); + + /// Max : Maximum /// diff --git a/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs b/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs index 71263bd49943d..c8894af3c9270 100644 --- a/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs +++ b/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs @@ -5096,12 +5096,24 @@ internal Arm64() { } public static unsafe System.Numerics.Vector LoadVectorByteNonFaultingZeroExtendToUInt16(byte* address) { throw null; } public static unsafe System.Numerics.Vector LoadVectorByteNonFaultingZeroExtendToUInt32(byte* address) { throw null; } public static unsafe System.Numerics.Vector LoadVectorByteNonFaultingZeroExtendToUInt64(byte* address) { throw null; } + public static unsafe System.Numerics.Vector LoadVectorByteZeroExtendFirstFaulting(System.Numerics.Vector mask, byte* address) { throw null; } + public static unsafe System.Numerics.Vector LoadVectorByteZeroExtendFirstFaulting(System.Numerics.Vector mask, byte* address) { throw null; } + public static unsafe System.Numerics.Vector LoadVectorByteZeroExtendFirstFaulting(System.Numerics.Vector mask, byte* address) { throw null; } + public static unsafe System.Numerics.Vector LoadVectorByteZeroExtendFirstFaulting(System.Numerics.Vector mask, byte* address) { throw null; } + public static unsafe System.Numerics.Vector LoadVectorByteZeroExtendFirstFaulting(System.Numerics.Vector mask, byte* address) { throw null; } + public static unsafe System.Numerics.Vector LoadVectorByteZeroExtendFirstFaulting(System.Numerics.Vector mask, byte* address) { throw null; } public static unsafe System.Numerics.Vector LoadVectorUInt16NonFaultingZeroExtendToInt32(ushort* address) { throw null; } public static unsafe System.Numerics.Vector LoadVectorUInt16NonFaultingZeroExtendToInt64(ushort* address) { throw null; } public static unsafe System.Numerics.Vector LoadVectorUInt16NonFaultingZeroExtendToUInt32(ushort* address) { throw null; } public static unsafe System.Numerics.Vector LoadVectorUInt16NonFaultingZeroExtendToUInt64(ushort* address) { throw null; } + public static unsafe System.Numerics.Vector LoadVectorUInt16ZeroExtendFirstFaulting(System.Numerics.Vector mask, ushort* address) { throw null; } + public static unsafe System.Numerics.Vector LoadVectorUInt16ZeroExtendFirstFaulting(System.Numerics.Vector mask, ushort* address) { throw null; } + public static unsafe System.Numerics.Vector LoadVectorUInt16ZeroExtendFirstFaulting(System.Numerics.Vector mask, ushort* address) { throw null; } + public static unsafe System.Numerics.Vector LoadVectorUInt16ZeroExtendFirstFaulting(System.Numerics.Vector mask, ushort* address) { throw null; } public static unsafe System.Numerics.Vector LoadVectorUInt32NonFaultingZeroExtendToInt64(uint* address) { throw null; } public static unsafe System.Numerics.Vector LoadVectorUInt32NonFaultingZeroExtendToUInt64(uint* address) { throw null; } + public static unsafe System.Numerics.Vector LoadVectorUInt32ZeroExtendFirstFaulting(System.Numerics.Vector mask, uint* address) { throw null; } + public static unsafe System.Numerics.Vector LoadVectorUInt32ZeroExtendFirstFaulting(System.Numerics.Vector mask, uint* address) { throw null; } public static unsafe (System.Numerics.Vector, System.Numerics.Vector) Load2xVectorAndUnzip(System.Numerics.Vector mask, byte* address) { throw null; } public static unsafe (System.Numerics.Vector, System.Numerics.Vector) Load2xVectorAndUnzip(System.Numerics.Vector mask, double* address) { throw null; } @@ -5138,14 +5150,26 @@ internal Arm64() { } public static unsafe System.Numerics.Vector LoadVectorInt16NonFaultingSignExtendToInt64(short* address) { throw null; } public static unsafe System.Numerics.Vector LoadVectorInt16NonFaultingSignExtendToUInt32(short* address) { throw null; } public static unsafe System.Numerics.Vector LoadVectorInt16NonFaultingSignExtendToUInt64(short* address) { throw null; } + public static unsafe System.Numerics.Vector LoadVectorInt16SignExtendFirstFaulting(System.Numerics.Vector mask, short* address) { throw null; } + public static unsafe System.Numerics.Vector LoadVectorInt16SignExtendFirstFaulting(System.Numerics.Vector mask, short* address) { throw null; } + public static unsafe System.Numerics.Vector LoadVectorInt16SignExtendFirstFaulting(System.Numerics.Vector mask, short* address) { throw null; } + public static unsafe System.Numerics.Vector LoadVectorInt16SignExtendFirstFaulting(System.Numerics.Vector mask, short* address) { throw null; } public static unsafe System.Numerics.Vector LoadVectorInt32NonFaultingSignExtendToInt64(int* address) { throw null; } public static unsafe System.Numerics.Vector LoadVectorInt32NonFaultingSignExtendToUInt64(int* address) { throw null; } + public static unsafe System.Numerics.Vector LoadVectorInt32SignExtendFirstFaulting(System.Numerics.Vector mask, int* address) { throw null; } + public static unsafe System.Numerics.Vector LoadVectorInt32SignExtendFirstFaulting(System.Numerics.Vector mask, int* address) { throw null; } public static unsafe System.Numerics.Vector LoadVectorSByteNonFaultingSignExtendToInt16(sbyte* address) { throw null; } public static unsafe System.Numerics.Vector LoadVectorSByteNonFaultingSignExtendToInt32(sbyte* address) { throw null; } public static unsafe System.Numerics.Vector LoadVectorSByteNonFaultingSignExtendToInt64(sbyte* address) { throw null; } public static unsafe System.Numerics.Vector LoadVectorSByteNonFaultingSignExtendToUInt16(sbyte* address) { throw null; } public static unsafe System.Numerics.Vector LoadVectorSByteNonFaultingSignExtendToUInt32(sbyte* address) { throw null; } public static unsafe System.Numerics.Vector LoadVectorSByteNonFaultingSignExtendToUInt64(sbyte* address) { throw null; } + public static unsafe System.Numerics.Vector LoadVectorSByteSignExtendFirstFaulting(System.Numerics.Vector mask, sbyte* address) { throw null; } + public static unsafe System.Numerics.Vector LoadVectorSByteSignExtendFirstFaulting(System.Numerics.Vector mask, sbyte* address) { throw null; } + public static unsafe System.Numerics.Vector LoadVectorSByteSignExtendFirstFaulting(System.Numerics.Vector mask, sbyte* address) { throw null; } + public static unsafe System.Numerics.Vector LoadVectorSByteSignExtendFirstFaulting(System.Numerics.Vector mask, sbyte* address) { throw null; } + public static unsafe System.Numerics.Vector LoadVectorSByteSignExtendFirstFaulting(System.Numerics.Vector mask, sbyte* address) { throw null; } + public static unsafe System.Numerics.Vector LoadVectorSByteSignExtendFirstFaulting(System.Numerics.Vector mask, sbyte* address) { throw null; } public static System.Numerics.Vector Max(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } public static System.Numerics.Vector Max(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } diff --git a/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs b/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs index 50b052a2fd5c6..b8d29c3a28479 100644 --- a/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs +++ b/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs @@ -3692,6 +3692,21 @@ ("SveLoadNonFaultingUnOpTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorNonFaulting_uint", ["Isa"] = "Sve", ["Method"] = "LoadVectorNonFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2BaseType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt32()", ["Cast"] = "(uint*)", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), ("SveLoadNonFaultingUnOpTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorNonFaulting_ulong", ["Isa"] = "Sve", ["Method"] = "LoadVectorNonFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2BaseType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt64()", ["Cast"] = "(ulong*)", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary {["TestName"] = "Sve_LoadVectorByteZeroExtendFirstFaulting_short", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorByteZeroExtendFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int16", ["Op2BaseType"] = "Byte", ["GetFfrType"] = "Int16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt16()", ["NextValueOp2"] = "TestLibrary.Generator.GetByte()"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary {["TestName"] = "Sve_LoadVectorByteZeroExtendFirstFaulting_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorByteZeroExtendFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2BaseType"] = "Byte", ["GetFfrType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetByte()"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary {["TestName"] = "Sve_LoadVectorByteZeroExtendFirstFaulting_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorByteZeroExtendFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2BaseType"] = "Byte", ["GetFfrType"] = "Int64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetByte()"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary {["TestName"] = "Sve_LoadVectorByteZeroExtendFirstFaulting_ushort", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorByteZeroExtendFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt16", ["Op2BaseType"] = "Byte", ["GetFfrType"] = "UInt16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt16()", ["NextValueOp2"] = "TestLibrary.Generator.GetByte()"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary {["TestName"] = "Sve_LoadVectorByteZeroExtendFirstFaulting_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorByteZeroExtendFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2BaseType"] = "Byte", ["GetFfrType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetByte()"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary {["TestName"] = "Sve_LoadVectorByteZeroExtendFirstFaulting_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorByteZeroExtendFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2BaseType"] = "Byte", ["GetFfrType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetByte()"}), + + ("SveLoadVectorFirstFaultingTest.template", new Dictionary {["TestName"] = "Sve_LoadVectorUInt16ZeroExtendFirstFaulting_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorUInt16ZeroExtendFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2BaseType"] = "UInt16", ["GetFfrType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt16()"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary {["TestName"] = "Sve_LoadVectorUInt16ZeroExtendFirstFaulting_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorUInt16ZeroExtendFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2BaseType"] = "UInt16", ["GetFfrType"] = "Int64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt16()"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary {["TestName"] = "Sve_LoadVectorUInt16ZeroExtendFirstFaulting_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorUInt16ZeroExtendFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2BaseType"] = "UInt16", ["GetFfrType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt16()"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary {["TestName"] = "Sve_LoadVectorUInt16ZeroExtendFirstFaulting_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorUInt16ZeroExtendFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2BaseType"] = "UInt16", ["GetFfrType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt16()"}), + + ("SveLoadVectorFirstFaultingTest.template", new Dictionary {["TestName"] = "Sve_LoadVectorUInt32ZeroExtendFirstFaulting_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorUInt32ZeroExtendFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2BaseType"] = "UInt32", ["GetFfrType"] = "Int64" , ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary {["TestName"] = "Sve_LoadVectorUInt32ZeroExtendFirstFaulting_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorUInt32ZeroExtendFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2BaseType"] = "UInt32", ["GetFfrType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_float", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Single", ["GetFfrType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSingle()", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()"}), ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_double", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Double", ["GetFfrType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetDouble()", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()"}), ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_sbyte", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "SByte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "SByte", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "SByte", ["GetFfrType"] = "SByte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSByte()", ["NextValueOp2"] = "TestLibrary.Generator.GetSByte()"}), @@ -3703,6 +3718,21 @@ ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt32", ["GetFfrType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()"}), ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt64", ["GetFfrType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary {["TestName"] = "Sve_LoadVectorInt16SignExtendFirstFaulting_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorInt16SignExtendFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2BaseType"] = "Int16", ["GetFfrType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt16()"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary {["TestName"] = "Sve_LoadVectorInt16SignExtendFirstFaulting_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorInt16SignExtendFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2BaseType"] = "Int16", ["GetFfrType"] = "Int64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt16()"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary {["TestName"] = "Sve_LoadVectorInt16SignExtendFirstFaulting_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorInt16SignExtendFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2BaseType"] = "Int16", ["GetFfrType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt16()"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary {["TestName"] = "Sve_LoadVectorInt16SignExtendFirstFaulting_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorInt16SignExtendFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2BaseType"] = "Int16", ["GetFfrType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt16()"}), + + ("SveLoadVectorFirstFaultingTest.template", new Dictionary {["TestName"] = "Sve_LoadVectorInt32SignExtendFirstFaulting_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorInt32SignExtendFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2BaseType"] = "Int32", ["GetFfrType"] = "Int64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt32()"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary {["TestName"] = "Sve_LoadVectorInt32SignExtendFirstFaulting_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorInt32SignExtendFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2BaseType"] = "Int32", ["GetFfrType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt32()"}), + + ("SveLoadVectorFirstFaultingTest.template", new Dictionary {["TestName"] = "Sve_LoadVectorSByteSignExtendFirstFaulting_short", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorSByteSignExtendFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int16", ["Op2BaseType"] = "SByte", ["GetFfrType"] = "Int16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt16()", ["NextValueOp2"] = "TestLibrary.Generator.GetSByte()"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary {["TestName"] = "Sve_LoadVectorSByteSignExtendFirstFaulting_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorSByteSignExtendFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2BaseType"] = "SByte", ["GetFfrType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetSByte()"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary {["TestName"] = "Sve_LoadVectorSByteSignExtendFirstFaulting_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorSByteSignExtendFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2BaseType"] = "SByte", ["GetFfrType"] = "Int64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetSByte()"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary {["TestName"] = "Sve_LoadVectorSByteSignExtendFirstFaulting_ushort", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorSByteSignExtendFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt16", ["Op2BaseType"] = "SByte", ["GetFfrType"] = "UInt16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt16()", ["NextValueOp2"] = "TestLibrary.Generator.GetSByte()"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary {["TestName"] = "Sve_LoadVectorSByteSignExtendFirstFaulting_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorSByteSignExtendFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2BaseType"] = "SByte", ["GetFfrType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetSByte()"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary {["TestName"] = "Sve_LoadVectorSByteSignExtendFirstFaulting_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorSByteSignExtendFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2BaseType"] = "SByte", ["GetFfrType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetSByte()"}), + ("SveLoadVectorMaskedTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorNonTemporal_float", ["Isa"] = "Sve", ["Method"] = "LoadVectorNonTemporal", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2BaseType"] = "Single", ["LargestVectorSize"] = "64", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), ("SveLoadVectorMaskedTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorNonTemporal_double", ["Isa"] = "Sve", ["Method"] = "LoadVectorNonTemporal", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2BaseType"] = "Double", ["LargestVectorSize"] = "64", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), ("SveLoadVectorMaskedTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorNonTemporal_sbyte", ["Isa"] = "Sve", ["Method"] = "LoadVectorNonTemporal", ["RetVectorType"] = "Vector", ["RetBaseType"] = "SByte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "SByte", ["Op2BaseType"] = "SByte", ["LargestVectorSize"] = "64", ["NextValueOp2"] = "TestLibrary.Generator.GetSByte()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs index a4a57bc9e89b0..79a608700bb01 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs @@ -8125,13 +8125,22 @@ private static T ConditionalSelectTrueResult(T maskResult, T result, T trueRe } - private static bool CheckLoadVectorBehaviorCore(T[] firstOp, T[] result, Func map) where T : INumberBase + private static TElem GetLoadVectorExpectedResultByIndex(int index, TMem[] firstOp, TElem[] result) + where TMem : INumberBase + where TElem : INumberBase + { + return (firstOp[index] == TMem.Zero) ? TElem.Zero : TElem.CreateTruncating(firstOp[index]); + } + + private static bool CheckLoadVectorBehaviorCore(TMem[] firstOp, TElem[] result, Func map) + where TMem : INumberBase + where TElem : INumberBase { for (var i = 0; i < firstOp.Length; i++) { - T loadResult = firstOp[i]; - loadResult = map(i, loadResult); - if (result[i] != loadResult) + TElem expectedResult = GetLoadVectorExpectedResultByIndex(i, firstOp, result); + expectedResult = map(i, expectedResult); + if (result[i] != expectedResult) { return false; } @@ -8139,12 +8148,16 @@ private static bool CheckLoadVectorBehaviorCore(T[] firstOp, T[] result, Func return true; } - public static bool CheckLoadVectorBehavior(T[] firstOp, T[] result) where T : INumberBase + public static bool CheckLoadVectorBehavior(TMem[] firstOp, TElem[] result) + where TMem : INumberBase, IConvertible + where TElem : INumberBase { return CheckLoadVectorBehaviorCore(firstOp, result, (_, loadResult) => loadResult); } - public static bool CheckLoadVectorBehavior(T[] maskOp, T[] firstOp, T[] result, T[] falseOp) where T : INumberBase + public static bool CheckLoadVectorBehavior(TElem[] maskOp, TMem[] firstOp, TElem[] result, TElem[] falseOp) + where TMem : INumberBase, IConvertible + where TElem : INumberBase { return CheckLoadVectorBehaviorCore(firstOp, result, (i, loadResult) => ConditionalSelectResult(maskOp[i], loadResult, falseOp[i])); } @@ -8270,6 +8283,12 @@ private static bool CheckFirstFaultingBehaviorCore(T[] result, Vector faul { return false; } + + if (result[i] != T.Zero) + { + return false; + } + hitFault = true; } else @@ -8285,9 +8304,11 @@ private static bool CheckFirstFaultingBehaviorCore(T[] result, Vector faul return true; } - public static bool CheckLoadVectorFirstFaultingBehavior(T[] firstOp, T[] result, Vector faultResult) where T : INumberBase + public static bool CheckLoadVectorFirstFaultingBehavior(TMem[] firstOp, TElem[] result, Vector faultResult) + where TMem : INumberBase, IConvertible + where TElem : INumberBase { - return CheckFirstFaultingBehaviorCore(result, faultResult, i => firstOp[i] == result[i]); + return CheckFirstFaultingBehaviorCore(result, faultResult, i => GetLoadVectorExpectedResultByIndex(i, firstOp, result) == result[i]); } public static bool CheckGatherVectorFirstFaultingBehavior(T[] mask, ExtendedElementT[] data, Index[] indices, T[] result, Vector faultResult) @@ -8305,7 +8326,7 @@ public static bool CheckGatherVectorBasesFirstFaultingBehavior GetGatherVectorBasesResultByIndex(i, mask, data) == result[i]); } - + private static byte ConditionalExtract(byte[] op1, byte op2, byte[] op3, bool after) { int last = LastActiveElement(op1); diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorFirstFaultingIndices.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorFirstFaultingIndices.template index c45dfc62d86d2..b53c96ab25cc5 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorFirstFaultingIndices.template +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorFirstFaultingIndices.template @@ -109,7 +109,7 @@ namespace JIT.HardwareIntrinsics.Arm int sizeOfinArray1 = inArray1.Length * Unsafe.SizeOf<{Op1BaseType}>(); int sizeOfinArray2 = inArray2.Length * Unsafe.SizeOf<{Op2BaseType}>(); int sizeOfinArray3 = inArray3.Length * Unsafe.SizeOf<{Op3BaseType}>(); - int sizeOfinBounded = new Random().Next(Unsafe.SizeOf<{Op2BaseType}>(), Vector<{Op2BaseType}>.Count * Unsafe.SizeOf<{Op2BaseType}>() - 1); + int sizeOfinBounded = new Random().Next(Unsafe.SizeOf<{Op2BaseType}>(), Vector<{RetBaseType}>.Count * Unsafe.SizeOf<{Op2BaseType}>() - 1); int sizeOfoutArray = outArray.Length * Unsafe.SizeOf<{RetBaseType}>(); if ((alignment != 64 && alignment != 16 && alignment != 8) || (alignment * 2) < sizeOfinArray1 || (alignment * 2) < sizeOfinArray3 || (alignment * 2) < sizeOfoutArray) @@ -309,7 +309,7 @@ namespace JIT.HardwareIntrinsics.Arm ref var op3Ref = ref op3; Unsafe.Write(_dataTable.outArrayPtr, result); - ValidateFirstFaultingResult(_dataTable.inArray1Ptr, ref op2Ref, _dataTable.inBounded.Span.Length, Unsafe.AsPointer(ref op3Ref), _dataTable.outArrayPtr, faultResult.As<{GetFfrType}, {RetBaseType}>()); + ValidateFirstFaultingResult(_dataTable.inArray1Ptr, ref op2Ref, Op2ElementCount * Unsafe.SizeOf<{Op2BaseType}>(), Unsafe.AsPointer(ref op3Ref), _dataTable.outArrayPtr, faultResult.As<{GetFfrType}, {RetBaseType}>()); } public void RunBasicScenario_FalseMask() diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveLoadVectorFirstFaultingTest.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveLoadVectorFirstFaultingTest.template index 4e7cb213d63d1..19574da19065a 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveLoadVectorFirstFaultingTest.template +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveLoadVectorFirstFaultingTest.template @@ -76,7 +76,7 @@ namespace JIT.HardwareIntrinsics.Arm public DataTable({Op2BaseType}[] inArray1, {RetBaseType}[] outArray, int alignment) { int sizeOfinArray1 = inArray1.Length * Unsafe.SizeOf<{Op2BaseType}>(); - int sizeOfinBounded = new Random().Next(Unsafe.SizeOf<{Op2BaseType}>(), Vector<{Op2BaseType}>.Count * Unsafe.SizeOf<{Op2BaseType}>() - 1); + int sizeOfinBounded = new Random().Next(Unsafe.SizeOf<{Op2BaseType}>(), Vector<{RetBaseType}>.Count * Unsafe.SizeOf<{Op2BaseType}>() - 1); int sizeOfoutArray = outArray.Length * Unsafe.SizeOf<{RetBaseType}>(); if ((alignment != 64 && alignment != 16 && alignment != 8) || (alignment * 2) < sizeOfinArray1 || (alignment * 2) < sizeOfoutArray) { @@ -116,14 +116,14 @@ namespace JIT.HardwareIntrinsics.Arm private struct TestStruct { - public {Op1VectorType}<{RetBaseType}> _fld1; + public {RetVectorType}<{RetBaseType}> _fld1; public static TestStruct Create() { var testStruct = new TestStruct(); - for (var i = 0; i < Op2ElementCount; i++) { _data[i] = {NextValueOp2}; } - Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1VectorType}<{RetBaseType}>, byte>(ref testStruct._fld1), ref Unsafe.As<{Op1BaseType}, byte>(ref _data[0]), (uint)Unsafe.SizeOf<{Op1VectorType}<{RetBaseType}>>()); + for (var i = 0; i < RetElementCount; i++) { _data[i] = {NextValueOp2}; } + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1VectorType}<{RetBaseType}>, byte>(ref testStruct._fld1), ref Unsafe.As<{Op2BaseType}, byte>(ref _data[0]), (uint)Unsafe.SizeOf<{Op1VectorType}<{RetBaseType}>>()); return testStruct; } @@ -137,19 +137,18 @@ namespace JIT.HardwareIntrinsics.Arm Unsafe.Write(testClass._dataTable.outArrayPtr, result); testClass.ValidateResult(testClass._dataTable.inArray1Ptr, testClass._dataTable.outArrayPtr); } - } + } private static readonly int LargestVectorSize = {LargestVectorSize}; - private static readonly int Op2ElementCount = Unsafe.SizeOf<{RetVectorType}<{Op2BaseType}>>() / sizeof({Op2BaseType}); private static readonly int RetElementCount = Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>() / sizeof({RetBaseType}); - private static {Op1BaseType}[] _data = new {Op1BaseType}[Op2ElementCount]; + private static {Op2BaseType}[] _data = new {Op2BaseType}[RetElementCount]; private static {RetBaseType}[] _maskData = new {RetBaseType}[RetElementCount]; private static {RetBaseType}[] _falseData = new {RetBaseType}[RetElementCount]; - private {Op1VectorType}<{RetBaseType}> _fld1; + private {RetVectorType}<{RetBaseType}> _fld1; private {Op1VectorType}<{Op1BaseType}> _mask; private {Op1VectorType}<{Op1BaseType}> _falseFld; @@ -160,7 +159,7 @@ namespace JIT.HardwareIntrinsics.Arm { Succeeded = true; - for (var i = 0; i < Op2ElementCount; i++) { _data[i] = {NextValueOp2}; } + for (var i = 0; i < RetElementCount; i++) { _data[i] = {NextValueOp2}; } _dataTable = new DataTable(_data, new {RetBaseType}[RetElementCount], LargestVectorSize); for (var i = 0; i < RetElementCount; i++) { _maskData[i] = ({RetBaseType})({NextValueOp1}); } @@ -196,7 +195,7 @@ namespace JIT.HardwareIntrinsics.Arm {Op1VectorType}<{Op1BaseType}> loadMask = Sve.CreateTrueMask{RetBaseType}(SveMaskPattern.All); ref var op2Ref = ref (_dataTable.inBounded.Span.GetPinnableReference()); - + Sve.SetFfr(Sve.CreateTrueMaskByte(SveMaskPattern.All)); var result = {Isa}.{Method}( loadMask, @@ -205,7 +204,7 @@ namespace JIT.HardwareIntrinsics.Arm var faultResult = Sve.GetFfr{GetFfrType}(); Unsafe.Write(_dataTable.outArrayPtr, result); - ValidateFirstFaultingResult(ref op2Ref, _dataTable.inBounded.Span.Length, _dataTable.outArrayPtr, faultResult.As<{GetFfrType}, {RetBaseType}>()); + ValidateFirstFaultingResult(ref op2Ref, Vector<{RetBaseType}>.Count * Unsafe.SizeOf<{Op2BaseType}>(), _dataTable.outArrayPtr, faultResult.As<{GetFfrType}, {RetBaseType}>()); } public void RunReflectionScenario_Load() @@ -260,22 +259,22 @@ namespace JIT.HardwareIntrinsics.Arm public void ConditionalSelect_ZeroOp() { TestLibrary.TestFramework.BeginScenario(nameof(ConditionalSelect_ZeroOp)); - ConditionalSelectScenario(_mask, ref _fld1, {Op1VectorType}<{RetBaseType}>.Zero); - + ConditionalSelectScenario(_mask, _dataTable.inArray1Ptr, {Op1VectorType}<{RetBaseType}>.Zero); + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_zero"); - ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}>.Zero, ref _fld1, {Op1VectorType}<{Op1BaseType}>.Zero); - + ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}>.Zero, _dataTable.inArray1Ptr, {Op1VectorType}<{Op1BaseType}>.Zero); + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_all"); - ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}>.AllBitsSet, ref _fld1, {Op1VectorType}<{Op1BaseType}>.Zero); - } + ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}>.AllBitsSet, _dataTable.inArray1Ptr, {Op1VectorType}<{Op1BaseType}>.Zero); + } [method: MethodImpl(MethodImplOptions.AggressiveInlining)] - private void ConditionalSelectScenario({RetVectorType}<{RetBaseType}> mask, ref {RetVectorType}<{Op2BaseType}> op1, {Op1VectorType}<{Op1BaseType}> falseOp) + private void ConditionalSelectScenario({RetVectorType}<{RetBaseType}> mask, void* op1, {Op1VectorType}<{Op1BaseType}> falseOp) { - var result = Sve.ConditionalSelect(mask, {Isa}.{Method}(Sve.CreateTrueMask{RetBaseType}(SveMaskPattern.All), ({Op2BaseType}*)Unsafe.AsPointer(ref op1)), falseOp); + var result = Sve.ConditionalSelect(mask, {Isa}.{Method}(Sve.CreateTrueMask{RetBaseType}(SveMaskPattern.All), ({Op2BaseType}*)op1), falseOp); Unsafe.Write(_dataTable.outArrayPtr, result); - ValidateConditionalSelectResult(mask, op1, falseOp, _dataTable.outArrayPtr); + ValidateConditionalSelectResult(mask, ({Op2BaseType}*)op1, falseOp, _dataTable.outArrayPtr); } public void RunUnsupportedScenario() @@ -296,7 +295,7 @@ namespace JIT.HardwareIntrinsics.Arm private void ValidateResult(void* firstOp, void* result, [CallerMemberName] string method = "") { - {Op2BaseType}[] inArray = new {Op2BaseType}[Op2ElementCount]; + {Op2BaseType}[] inArray = new {Op2BaseType}[RetElementCount]; {RetBaseType}[] outArray = new {RetBaseType}[RetElementCount]; Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op2BaseType}, byte>(ref inArray[0]), ref Unsafe.AsRef(firstOp), (uint)Unsafe.SizeOf<{RetVectorType}<{Op2BaseType}>>()); @@ -320,22 +319,22 @@ namespace JIT.HardwareIntrinsics.Arm } } - private void ValidateConditionalSelectResult({RetVectorType}<{RetBaseType}> maskOp, {Op1VectorType}<{Op1BaseType}> op1, {RetVectorType}<{RetBaseType}> falseOp, void* result, [CallerMemberName] string method = "") + private void ValidateConditionalSelectResult({RetVectorType}<{RetBaseType}> maskOp, {Op2BaseType}* op1, {RetVectorType}<{RetBaseType}> falseOp, void* result, [CallerMemberName] string method = "") { {RetBaseType}[] maskArray = new {RetBaseType}[RetElementCount]; - {Op1BaseType}[] op1Array = new {Op1BaseType}[RetElementCount]; + {Op2BaseType}[] op1Array = new {Op2BaseType}[RetElementCount]; {RetBaseType}[] falseValArray = new {RetBaseType}[RetElementCount]; {RetBaseType}[] resultArray = new {RetBaseType}[RetElementCount]; Unsafe.WriteUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref maskArray[0]), maskOp); - Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref op1Array[0]), op1); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op2BaseType}, byte>(ref op1Array[0]), ref Unsafe.AsRef(op1), (uint)(op1Array.Length * Unsafe.SizeOf<{Op2BaseType}>())); Unsafe.WriteUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref falseValArray[0]), falseOp); Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref resultArray[0]), ref Unsafe.AsRef(result), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); ValidateConditionalSelectResult(maskArray, op1Array, falseValArray, resultArray, method); } - private void ValidateConditionalSelectResult({RetBaseType}[] maskOp, {Op1BaseType}[] firstOp, {RetBaseType}[] falseOp, {RetBaseType}[] result, [CallerMemberName] string method = "") + private void ValidateConditionalSelectResult({RetBaseType}[] maskOp, {Op2BaseType}[] firstOp, {RetBaseType}[] falseOp, {RetBaseType}[] result, [CallerMemberName] string method = "") { var succeeded = Helpers.CheckLoadVectorBehavior(maskOp, firstOp, result, falseOp); From 0f06c243d88306d1ea1df00ed8fbc9973395a3b2 Mon Sep 17 00:00:00 2001 From: Mikhail Ablakatov Date: Tue, 16 Jul 2024 15:14:43 +0000 Subject: [PATCH 19/83] Remove HW_Flag_SpecialSideEffectMask for LoadVector*FirstFaulting --- src/coreclr/jit/hwintrinsiclistarm64sve.h | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/coreclr/jit/hwintrinsiclistarm64sve.h b/src/coreclr/jit/hwintrinsiclistarm64sve.h index 22910a7133048..09801c0564a16 100644 --- a/src/coreclr/jit/hwintrinsiclistarm64sve.h +++ b/src/coreclr/jit/hwintrinsiclistarm64sve.h @@ -144,26 +144,26 @@ HARDWARE_INTRINSIC(Sve, LoadVectorByteNonFaultingZeroExtendToInt64, HARDWARE_INTRINSIC(Sve, LoadVectorByteNonFaultingZeroExtendToUInt16, -1, 1, false, {INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorByteNonFaultingZeroExtendToUInt32, -1, 1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorByteNonFaultingZeroExtendToUInt64, -1, 1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1b, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendFirstFaulting, -1, 2, true, {INS_invalid, INS_invalid, INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1b, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendFirstFaulting, -1, 2, true, {INS_invalid, INS_invalid, INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1b, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialCodeGen) HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendToInt16, -1, 2, false, {INS_invalid, INS_invalid, INS_sve_ld1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendToInt32, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendToInt64, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1b, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendToUInt16, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendToUInt32, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendToUInt64, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1b, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorFirstFaulting, -1, 2, true, {INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1h, INS_sve_ldff1h, INS_sve_ldff1w, INS_sve_ldff1w, INS_sve_ldff1d, INS_sve_ldff1d, INS_sve_ldff1w, INS_sve_ldff1d}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorFirstFaulting, -1, 2, true, {INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1h, INS_sve_ldff1h, INS_sve_ldff1w, INS_sve_ldff1w, INS_sve_ldff1d, INS_sve_ldff1d, INS_sve_ldff1w, INS_sve_ldff1d}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialCodeGen) HARDWARE_INTRINSIC(Sve, LoadVectorInt16NonFaultingSignExtendToInt32, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorInt16NonFaultingSignExtendToInt64, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sh, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorInt16NonFaultingSignExtendToUInt32, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sh, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorInt16NonFaultingSignExtendToUInt64, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sh, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorInt16SignExtendFirstFaulting, -1, 2, true, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1sh, INS_sve_ldff1sh, INS_sve_ldff1sh, INS_sve_ldff1sh, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorInt16SignExtendFirstFaulting, -1, 2, true, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1sh, INS_sve_ldff1sh, INS_sve_ldff1sh, INS_sve_ldff1sh, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialCodeGen) HARDWARE_INTRINSIC(Sve, LoadVectorInt16SignExtendToInt32, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorInt16SignExtendToInt64, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sh, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorInt16SignExtendToUInt32, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sh, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorInt16SignExtendToUInt64, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sh, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorInt32NonFaultingSignExtendToInt64, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sw, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorInt32NonFaultingSignExtendToUInt64, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sw, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorInt32SignExtendFirstFaulting, -1, 2, true, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1sw, INS_sve_ldff1sw, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorInt32SignExtendFirstFaulting, -1, 2, true, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1sw, INS_sve_ldff1sw, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialCodeGen) HARDWARE_INTRINSIC(Sve, LoadVectorInt32SignExtendToInt64, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sw, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorInt32SignExtendToUInt64, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sw, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorNonFaulting, -1, 1, true, {INS_sve_ldnf1b, INS_sve_ldnf1b, INS_sve_ldnf1h, INS_sve_ldnf1h, INS_sve_ldnf1w, INS_sve_ldnf1w, INS_sve_ldnf1d, INS_sve_ldnf1d, INS_sve_ldnf1w, INS_sve_ldnf1d}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) @@ -174,7 +174,7 @@ HARDWARE_INTRINSIC(Sve, LoadVectorSByteNonFaultingSignExtendToInt64, HARDWARE_INTRINSIC(Sve, LoadVectorSByteNonFaultingSignExtendToUInt16, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorSByteNonFaultingSignExtendToUInt32, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorSByteNonFaultingSignExtendToUInt64, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sb, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorSByteSignExtendFirstFaulting, -1, 2, true, {INS_invalid, INS_invalid, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorSByteSignExtendFirstFaulting, -1, 2, true, {INS_invalid, INS_invalid, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialCodeGen) HARDWARE_INTRINSIC(Sve, LoadVectorSByteSignExtendToInt16, -1, 2, false, {INS_invalid, INS_invalid, INS_sve_ld1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorSByteSignExtendToInt32, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorSByteSignExtendToInt64, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sb, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) @@ -185,14 +185,14 @@ HARDWARE_INTRINSIC(Sve, LoadVectorUInt16NonFaultingZeroExtendToInt32, HARDWARE_INTRINSIC(Sve, LoadVectorUInt16NonFaultingZeroExtendToInt64, -1, 1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1h, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorUInt16NonFaultingZeroExtendToUInt32, -1, 1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1h, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorUInt16NonFaultingZeroExtendToUInt64, -1, 1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1h, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorUInt16ZeroExtendFirstFaulting, -1, 2, true, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1h, INS_sve_ldff1h, INS_sve_ldff1h, INS_sve_ldff1h, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorUInt16ZeroExtendFirstFaulting, -1, 2, true, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1h, INS_sve_ldff1h, INS_sve_ldff1h, INS_sve_ldff1h, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialCodeGen) HARDWARE_INTRINSIC(Sve, LoadVectorUInt16ZeroExtendToInt32, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1h, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorUInt16ZeroExtendToInt64, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1h, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorUInt16ZeroExtendToUInt32, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1h, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorUInt16ZeroExtendToUInt64, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1h, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorUInt32NonFaultingZeroExtendToInt64, -1, 1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1w, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorUInt32NonFaultingZeroExtendToUInt64, -1, 1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1w, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorUInt32ZeroExtendFirstFaulting, -1, 2, true, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1w, INS_sve_ldff1w, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, LoadVectorUInt32ZeroExtendFirstFaulting, -1, 2, true, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1w, INS_sve_ldff1w, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialCodeGen) HARDWARE_INTRINSIC(Sve, LoadVectorUInt32ZeroExtendToInt64, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorUInt32ZeroExtendToUInt64, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, Max, -1, -1, false, {INS_sve_smax, INS_sve_umax, INS_sve_smax, INS_sve_umax, INS_sve_smax, INS_sve_umax, INS_sve_smax, INS_sve_umax, INS_sve_fmax, INS_sve_fmax}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) From 80e9c058574cc07f0d8a7b5baf45479e6bc719c0 Mon Sep 17 00:00:00 2001 From: Mikhail Ablakatov Date: Tue, 16 Jul 2024 15:21:02 +0000 Subject: [PATCH 20/83] cleanup: remove [Xarray, Xindex] instruction variant from the comments at the declarations as not used --- .../Arm/Sve.PlatformNotSupported.cs | 24 ------------------- .../src/System/Runtime/Intrinsics/Arm/Sve.cs | 24 ------------------- 2 files changed, 48 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs index 6ae295292d7fd..15e81030ecd3f 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs @@ -5146,42 +5146,36 @@ internal Arm64() { } /// /// svint16_t svldff1ub_s16(svbool_t pg, const uint8_t *base) - /// LDFF1B Zresult.H, Pg/Z, [Xarray, Xindex] /// LDFF1B Zresult.H, Pg/Z, [Xbase, XZR] /// public static unsafe Vector LoadVectorByteZeroExtendFirstFaulting(Vector mask, byte* address) { throw new PlatformNotSupportedException(); } /// /// svint32_t svldff1ub_s32(svbool_t pg, const uint8_t *base) - /// LDFF1B Zresult.S, Pg/Z, [Xarray, Xindex] /// LDFF1B Zresult.S, Pg/Z, [Xbase, XZR] /// public static unsafe Vector LoadVectorByteZeroExtendFirstFaulting(Vector mask, byte* address) { throw new PlatformNotSupportedException(); } /// /// svint64_t svldff1ub_s64(svbool_t pg, const uint8_t *base) - /// LDFF1B Zresult.D, Pg/Z, [Xarray, Xindex] /// LDFF1B Zresult.D, Pg/Z, [Xbase, XZR] /// public static unsafe Vector LoadVectorByteZeroExtendFirstFaulting(Vector mask, byte* address) { throw new PlatformNotSupportedException(); } /// /// svuint16_t svldff1ub_u16(svbool_t pg, const uint8_t *base) - /// LDFF1B Zresult.H, Pg/Z, [Xarray, Xindex] /// LDFF1B Zresult.H, Pg/Z, [Xbase, XZR] /// public static unsafe Vector LoadVectorByteZeroExtendFirstFaulting(Vector mask, byte* address) { throw new PlatformNotSupportedException(); } /// /// svuint32_t svldff1ub_u32(svbool_t pg, const uint8_t *base) - /// LDFF1B Zresult.S, Pg/Z, [Xarray, Xindex] /// LDFF1B Zresult.S, Pg/Z, [Xbase, XZR] /// public static unsafe Vector LoadVectorByteZeroExtendFirstFaulting(Vector mask, byte* address) { throw new PlatformNotSupportedException(); } /// /// svuint64_t svldff1ub_u64(svbool_t pg, const uint8_t *base) - /// LDFF1B Zresult.D, Pg/Z, [Xarray, Xindex] /// LDFF1B Zresult.D, Pg/Z, [Xbase, XZR] /// public static unsafe Vector LoadVectorByteZeroExtendFirstFaulting(Vector mask, byte* address) { throw new PlatformNotSupportedException(); } @@ -5214,28 +5208,24 @@ internal Arm64() { } /// /// svint32_t svldff1uh_s32(svbool_t pg, const uint16_t *base) - /// LDFF1H Zresult.S, Pg/Z, [Xarray, Xindex, LSL #1] /// LDFF1H Zresult.S, Pg/Z, [Xbase, XZR, LSL #1] /// public static unsafe Vector LoadVectorUInt16ZeroExtendFirstFaulting(Vector mask, ushort* address) { throw new PlatformNotSupportedException(); } /// /// svint64_t svldff1uh_s64(svbool_t pg, const uint16_t *base) - /// LDFF1H Zresult.D, Pg/Z, [Xarray, Xindex, LSL #1] /// LDFF1H Zresult.D, Pg/Z, [Xbase, XZR, LSL #1] /// public static unsafe Vector LoadVectorUInt16ZeroExtendFirstFaulting(Vector mask, ushort* address) { throw new PlatformNotSupportedException(); } /// /// svuint32_t svldff1uh_u32(svbool_t pg, const uint16_t *base) - /// LDFF1H Zresult.S, Pg/Z, [Xarray, Xindex, LSL #1] /// LDFF1H Zresult.S, Pg/Z, [Xbase, XZR, LSL #1] /// public static unsafe Vector LoadVectorUInt16ZeroExtendFirstFaulting(Vector mask, ushort* address) { throw new PlatformNotSupportedException(); } /// /// svuint64_t svldff1uh_u64(svbool_t pg, const uint16_t *base) - /// LDFF1H Zresult.D, Pg/Z, [Xarray, Xindex, LSL #1] /// LDFF1H Zresult.D, Pg/Z, [Xbase, XZR, LSL #1] /// public static unsafe Vector LoadVectorUInt16ZeroExtendFirstFaulting(Vector mask, ushort* address) { throw new PlatformNotSupportedException(); } @@ -5256,14 +5246,12 @@ internal Arm64() { } /// /// svint64_t svldff1uw_s64(svbool_t pg, const uint32_t *base) - /// LDFF1W Zresult.D, Pg/Z, [Xarray, Xindex, LSL #2] /// LDFF1W Zresult.D, Pg/Z, [Xbase, XZR, LSL #2] /// public static unsafe Vector LoadVectorUInt32ZeroExtendFirstFaulting(Vector mask, uint* address) { throw new PlatformNotSupportedException(); } /// /// svuint64_t svldff1uw_u64(svbool_t pg, const uint32_t *base) - /// LDFF1W Zresult.D, Pg/Z, [Xarray, Xindex, LSL #2] /// LDFF1W Zresult.D, Pg/Z, [Xbase, XZR, LSL #2] /// public static unsafe Vector LoadVectorUInt32ZeroExtendFirstFaulting(Vector mask, uint* address) { throw new PlatformNotSupportedException(); } @@ -5488,28 +5476,24 @@ internal Arm64() { } /// /// svint32_t svldff1sh_s32(svbool_t pg, const int16_t *base) - /// LDFF1SH Zresult.S, Pg/Z, [Xarray, Xindex, LSL #1] /// LDFF1SH Zresult.S, Pg/Z, [Xbase, XZR, LSL #1] /// public static unsafe Vector LoadVectorInt16SignExtendFirstFaulting(Vector mask, short* address) { throw new PlatformNotSupportedException(); } /// /// svint64_t svldff1sh_s64(svbool_t pg, const int16_t *base) - /// LDFF1SH Zresult.D, Pg/Z, [Xarray, Xindex, LSL #1] /// LDFF1SH Zresult.D, Pg/Z, [Xbase, XZR, LSL #1] /// public static unsafe Vector LoadVectorInt16SignExtendFirstFaulting(Vector mask, short* address) { throw new PlatformNotSupportedException(); } /// /// svuint32_t svldff1sh_u32(svbool_t pg, const int16_t *base) - /// LDFF1SH Zresult.S, Pg/Z, [Xarray, Xindex, LSL #1] /// LDFF1SH Zresult.S, Pg/Z, [Xbase, XZR, LSL #1] /// public static unsafe Vector LoadVectorInt16SignExtendFirstFaulting(Vector mask, short* address) { throw new PlatformNotSupportedException(); } /// /// svuint64_t svldff1sh_u64(svbool_t pg, const int16_t *base) - /// LDFF1SH Zresult.D, Pg/Z, [Xarray, Xindex, LSL #1] /// LDFF1SH Zresult.D, Pg/Z, [Xbase, XZR, LSL #1] /// public static unsafe Vector LoadVectorInt16SignExtendFirstFaulting(Vector mask, short* address) { throw new PlatformNotSupportedException(); } @@ -5536,14 +5520,12 @@ internal Arm64() { } /// /// svint64_t svldff1sw_s64(svbool_t pg, const int32_t *base) - /// LDFF1SW Zresult.D, Pg/Z, [Xarray, Xindex, LSL #2] /// LDFF1SW Zresult.D, Pg/Z, [Xbase, XZR, LSL #2] /// public static unsafe Vector LoadVectorInt32SignExtendFirstFaulting(Vector mask, int* address) { throw new PlatformNotSupportedException(); } /// /// svuint64_t svldff1sw_u64(svbool_t pg, const int32_t *base) - /// LDFF1SW Zresult.D, Pg/Z, [Xarray, Xindex, LSL #2] /// LDFF1SW Zresult.D, Pg/Z, [Xbase, XZR, LSL #2] /// public static unsafe Vector LoadVectorInt32SignExtendFirstFaulting(Vector mask, int* address) { throw new PlatformNotSupportedException(); } @@ -5606,42 +5588,36 @@ internal Arm64() { } /// /// svint16_t svldff1sb_s16(svbool_t pg, const int8_t *base) - /// LDFF1SB Zresult.H, Pg/Z, [Xarray, Xindex] /// LDFF1SB Zresult.H, Pg/Z, [Xbase, XZR] /// public static unsafe Vector LoadVectorSByteSignExtendFirstFaulting(Vector mask, sbyte* address) { throw new PlatformNotSupportedException(); } /// /// svint32_t svldff1sb_s32(svbool_t pg, const int8_t *base) - /// LDFF1SB Zresult.S, Pg/Z, [Xarray, Xindex] /// LDFF1SB Zresult.S, Pg/Z, [Xbase, XZR] /// public static unsafe Vector LoadVectorSByteSignExtendFirstFaulting(Vector mask, sbyte* address) { throw new PlatformNotSupportedException(); } /// /// svint64_t svldff1sb_s64(svbool_t pg, const int8_t *base) - /// LDFF1SB Zresult.D, Pg/Z, [Xarray, Xindex] /// LDFF1SB Zresult.D, Pg/Z, [Xbase, XZR] /// public static unsafe Vector LoadVectorSByteSignExtendFirstFaulting(Vector mask, sbyte* address) { throw new PlatformNotSupportedException(); } /// /// svuint16_t svldff1sb_u16(svbool_t pg, const int8_t *base) - /// LDFF1SB Zresult.H, Pg/Z, [Xarray, Xindex] /// LDFF1SB Zresult.H, Pg/Z, [Xbase, XZR] /// public static unsafe Vector LoadVectorSByteSignExtendFirstFaulting(Vector mask, sbyte* address) { throw new PlatformNotSupportedException(); } /// /// svuint32_t svldff1sb_u32(svbool_t pg, const int8_t *base) - /// LDFF1SB Zresult.S, Pg/Z, [Xarray, Xindex] /// LDFF1SB Zresult.S, Pg/Z, [Xbase, XZR] /// public static unsafe Vector LoadVectorSByteSignExtendFirstFaulting(Vector mask, sbyte* address) { throw new PlatformNotSupportedException(); } /// /// svuint64_t svldff1sb_u64(svbool_t pg, const int8_t *base) - /// LDFF1SB Zresult.D, Pg/Z, [Xarray, Xindex] /// LDFF1SB Zresult.D, Pg/Z, [Xbase, XZR] /// public static unsafe Vector LoadVectorSByteSignExtendFirstFaulting(Vector mask, sbyte* address) { throw new PlatformNotSupportedException(); } diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs index c144b69efaa78..80704372dfab8 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs @@ -5202,42 +5202,36 @@ internal Arm64() { } public static unsafe Vector LoadVectorByteNonFaultingZeroExtendToUInt64(byte* address) => LoadVectorByteNonFaultingZeroExtendToUInt64(address); /// /// svint16_t svldff1ub_s16(svbool_t pg, const uint8_t *base) - /// LDFF1B Zresult.H, Pg/Z, [Xarray, Xindex] /// LDFF1B Zresult.H, Pg/Z, [Xbase, XZR] /// public static unsafe Vector LoadVectorByteZeroExtendFirstFaulting(Vector mask, byte* address) => LoadVectorByteZeroExtendFirstFaulting(mask, address); /// /// svint32_t svldff1ub_s32(svbool_t pg, const uint8_t *base) - /// LDFF1B Zresult.S, Pg/Z, [Xarray, Xindex] /// LDFF1B Zresult.S, Pg/Z, [Xbase, XZR] /// public static unsafe Vector LoadVectorByteZeroExtendFirstFaulting(Vector mask, byte* address) => LoadVectorByteZeroExtendFirstFaulting(mask, address); /// /// svint64_t svldff1ub_s64(svbool_t pg, const uint8_t *base) - /// LDFF1B Zresult.D, Pg/Z, [Xarray, Xindex] /// LDFF1B Zresult.D, Pg/Z, [Xbase, XZR] /// public static unsafe Vector LoadVectorByteZeroExtendFirstFaulting(Vector mask, byte* address) => LoadVectorByteZeroExtendFirstFaulting(mask, address); /// /// svuint16_t svldff1ub_u16(svbool_t pg, const uint8_t *base) - /// LDFF1B Zresult.H, Pg/Z, [Xarray, Xindex] /// LDFF1B Zresult.H, Pg/Z, [Xbase, XZR] /// public static unsafe Vector LoadVectorByteZeroExtendFirstFaulting(Vector mask, byte* address) => LoadVectorByteZeroExtendFirstFaulting(mask, address); /// /// svuint32_t svldff1ub_u32(svbool_t pg, const uint8_t *base) - /// LDFF1B Zresult.S, Pg/Z, [Xarray, Xindex] /// LDFF1B Zresult.S, Pg/Z, [Xbase, XZR] /// public static unsafe Vector LoadVectorByteZeroExtendFirstFaulting(Vector mask, byte* address) => LoadVectorByteZeroExtendFirstFaulting(mask, address); /// /// svuint64_t svldff1ub_u64(svbool_t pg, const uint8_t *base) - /// LDFF1B Zresult.D, Pg/Z, [Xarray, Xindex] /// LDFF1B Zresult.D, Pg/Z, [Xbase, XZR] /// public static unsafe Vector LoadVectorByteZeroExtendFirstFaulting(Vector mask, byte* address) => LoadVectorByteZeroExtendFirstFaulting(mask, address); @@ -5268,28 +5262,24 @@ internal Arm64() { } /// /// svint32_t svldff1uh_s32(svbool_t pg, const uint16_t *base) - /// LDFF1H Zresult.S, Pg/Z, [Xarray, Xindex, LSL #1] /// LDFF1H Zresult.S, Pg/Z, [Xbase, XZR, LSL #1] /// public static unsafe Vector LoadVectorUInt16ZeroExtendFirstFaulting(Vector mask, ushort* address) => LoadVectorUInt16ZeroExtendFirstFaulting(mask, address); /// /// svint64_t svldff1uh_s64(svbool_t pg, const uint16_t *base) - /// LDFF1H Zresult.D, Pg/Z, [Xarray, Xindex, LSL #1] /// LDFF1H Zresult.D, Pg/Z, [Xbase, XZR, LSL #1] /// public static unsafe Vector LoadVectorUInt16ZeroExtendFirstFaulting(Vector mask, ushort* address) => LoadVectorUInt16ZeroExtendFirstFaulting(mask, address); /// /// svuint32_t svldff1uh_u32(svbool_t pg, const uint16_t *base) - /// LDFF1H Zresult.S, Pg/Z, [Xarray, Xindex, LSL #1] /// LDFF1H Zresult.S, Pg/Z, [Xbase, XZR, LSL #1] /// public static unsafe Vector LoadVectorUInt16ZeroExtendFirstFaulting(Vector mask, ushort* address) => LoadVectorUInt16ZeroExtendFirstFaulting(mask, address); /// /// svuint64_t svldff1uh_u64(svbool_t pg, const uint16_t *base) - /// LDFF1H Zresult.D, Pg/Z, [Xarray, Xindex, LSL #1] /// LDFF1H Zresult.D, Pg/Z, [Xbase, XZR, LSL #1] /// public static unsafe Vector LoadVectorUInt16ZeroExtendFirstFaulting(Vector mask, ushort* address) => LoadVectorUInt16ZeroExtendFirstFaulting(mask, address); @@ -5308,14 +5298,12 @@ internal Arm64() { } /// /// svint64_t svldff1uw_s64(svbool_t pg, const uint32_t *base) - /// LDFF1W Zresult.D, Pg/Z, [Xarray, Xindex, LSL #2] /// LDFF1W Zresult.D, Pg/Z, [Xbase, XZR, LSL #2] /// public static unsafe Vector LoadVectorUInt32ZeroExtendFirstFaulting(Vector mask, uint* address) => LoadVectorUInt32ZeroExtendFirstFaulting(mask, address); /// /// svuint64_t svldff1uw_u64(svbool_t pg, const uint32_t *base) - /// LDFF1W Zresult.D, Pg/Z, [Xarray, Xindex, LSL #2] /// LDFF1W Zresult.D, Pg/Z, [Xbase, XZR, LSL #2] /// public static unsafe Vector LoadVectorUInt32ZeroExtendFirstFaulting(Vector mask, uint* address) => LoadVectorUInt32ZeroExtendFirstFaulting(mask, address); @@ -5540,28 +5528,24 @@ internal Arm64() { } /// /// svint32_t svldff1sh_s32(svbool_t pg, const int16_t *base) - /// LDFF1SH Zresult.S, Pg/Z, [Xarray, Xindex, LSL #1] /// LDFF1SH Zresult.S, Pg/Z, [Xbase, XZR, LSL #1] /// public static unsafe Vector LoadVectorInt16SignExtendFirstFaulting(Vector mask, short* address) => LoadVectorInt16SignExtendFirstFaulting(mask, address); /// /// svint64_t svldff1sh_s64(svbool_t pg, const int16_t *base) - /// LDFF1SH Zresult.D, Pg/Z, [Xarray, Xindex, LSL #1] /// LDFF1SH Zresult.D, Pg/Z, [Xbase, XZR, LSL #1] /// public static unsafe Vector LoadVectorInt16SignExtendFirstFaulting(Vector mask, short* address) => LoadVectorInt16SignExtendFirstFaulting(mask, address); /// /// svuint32_t svldff1sh_u32(svbool_t pg, const int16_t *base) - /// LDFF1SH Zresult.S, Pg/Z, [Xarray, Xindex, LSL #1] /// LDFF1SH Zresult.S, Pg/Z, [Xbase, XZR, LSL #1] /// public static unsafe Vector LoadVectorInt16SignExtendFirstFaulting(Vector mask, short* address) => LoadVectorInt16SignExtendFirstFaulting(mask, address); /// /// svuint64_t svldff1sh_u64(svbool_t pg, const int16_t *base) - /// LDFF1SH Zresult.D, Pg/Z, [Xarray, Xindex, LSL #1] /// LDFF1SH Zresult.D, Pg/Z, [Xbase, XZR, LSL #1] /// public static unsafe Vector LoadVectorInt16SignExtendFirstFaulting(Vector mask, short* address) => LoadVectorInt16SignExtendFirstFaulting(mask, address); @@ -5588,14 +5572,12 @@ internal Arm64() { } /// /// svint64_t svldff1sw_s64(svbool_t pg, const int32_t *base) - /// LDFF1SW Zresult.D, Pg/Z, [Xarray, Xindex, LSL #2] /// LDFF1SW Zresult.D, Pg/Z, [Xbase, XZR, LSL #2] /// public static unsafe Vector LoadVectorInt32SignExtendFirstFaulting(Vector mask, int* address) => LoadVectorInt32SignExtendFirstFaulting(mask, address); /// /// svuint64_t svldff1sw_u64(svbool_t pg, const int32_t *base) - /// LDFF1SW Zresult.D, Pg/Z, [Xarray, Xindex, LSL #2] /// LDFF1SW Zresult.D, Pg/Z, [Xbase, XZR, LSL #2] /// public static unsafe Vector LoadVectorInt32SignExtendFirstFaulting(Vector mask, int* address) => LoadVectorInt32SignExtendFirstFaulting(mask, address); @@ -5658,42 +5640,36 @@ internal Arm64() { } /// /// svint16_t svldff1sb_s16(svbool_t pg, const int8_t *base) - /// LDFF1SB Zresult.H, Pg/Z, [Xarray, Xindex] /// LDFF1SB Zresult.H, Pg/Z, [Xbase, XZR] /// public static unsafe Vector LoadVectorSByteSignExtendFirstFaulting(Vector mask, sbyte* address) => LoadVectorSByteSignExtendFirstFaulting(mask, address); /// /// svint32_t svldff1sb_s32(svbool_t pg, const int8_t *base) - /// LDFF1SB Zresult.S, Pg/Z, [Xarray, Xindex] /// LDFF1SB Zresult.S, Pg/Z, [Xbase, XZR] /// public static unsafe Vector LoadVectorSByteSignExtendFirstFaulting(Vector mask, sbyte* address) => LoadVectorSByteSignExtendFirstFaulting(mask, address); /// /// svint64_t svldff1sb_s64(svbool_t pg, const int8_t *base) - /// LDFF1SB Zresult.D, Pg/Z, [Xarray, Xindex] /// LDFF1SB Zresult.D, Pg/Z, [Xbase, XZR] /// public static unsafe Vector LoadVectorSByteSignExtendFirstFaulting(Vector mask, sbyte* address) => LoadVectorSByteSignExtendFirstFaulting(mask, address); /// /// svuint16_t svldff1sb_u16(svbool_t pg, const int8_t *base) - /// LDFF1SB Zresult.H, Pg/Z, [Xarray, Xindex] /// LDFF1SB Zresult.H, Pg/Z, [Xbase, XZR] /// public static unsafe Vector LoadVectorSByteSignExtendFirstFaulting(Vector mask, sbyte* address) => LoadVectorSByteSignExtendFirstFaulting(mask, address); /// /// svuint32_t svldff1sb_u32(svbool_t pg, const int8_t *base) - /// LDFF1SB Zresult.S, Pg/Z, [Xarray, Xindex] /// LDFF1SB Zresult.S, Pg/Z, [Xbase, XZR] /// public static unsafe Vector LoadVectorSByteSignExtendFirstFaulting(Vector mask, sbyte* address) => LoadVectorSByteSignExtendFirstFaulting(mask, address); /// /// svuint64_t svldff1sb_u64(svbool_t pg, const int8_t *base) - /// LDFF1SB Zresult.D, Pg/Z, [Xarray, Xindex] /// LDFF1SB Zresult.D, Pg/Z, [Xbase, XZR] /// public static unsafe Vector LoadVectorSByteSignExtendFirstFaulting(Vector mask, sbyte* address) => LoadVectorSByteSignExtendFirstFaulting(mask, address); From 76abc0176416a67e0719c74b8d409a2772f86cd3 Mon Sep 17 00:00:00 2001 From: Mikhail Ablakatov Date: Tue, 16 Jul 2024 15:28:52 +0000 Subject: [PATCH 21/83] cleanup: arrange the tests alphabetically --- .../GenerateHWIntrinsicTests_Arm.cs | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs b/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs index fc25f7829286c..fd5f1fded6456 100644 --- a/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs +++ b/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs @@ -3714,31 +3714,23 @@ ("SveLoadNonFaultingUnOpTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorNonFaulting_uint", ["Isa"] = "Sve", ["Method"] = "LoadVectorNonFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2BaseType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt32()", ["Cast"] = "(uint*)", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), ("SveLoadNonFaultingUnOpTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorNonFaulting_ulong", ["Isa"] = "Sve", ["Method"] = "LoadVectorNonFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2BaseType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt64()", ["Cast"] = "(ulong*)", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadVectorFirstFaultingTest.template", new Dictionary {["TestName"] = "Sve_LoadVectorByteZeroExtendFirstFaulting_short", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorByteZeroExtendFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int16", ["Op2BaseType"] = "Byte", ["GetFfrType"] = "Int16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt16()", ["NextValueOp2"] = "TestLibrary.Generator.GetByte()"}), ("SveLoadVectorFirstFaultingTest.template", new Dictionary {["TestName"] = "Sve_LoadVectorByteZeroExtendFirstFaulting_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorByteZeroExtendFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2BaseType"] = "Byte", ["GetFfrType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetByte()"}), ("SveLoadVectorFirstFaultingTest.template", new Dictionary {["TestName"] = "Sve_LoadVectorByteZeroExtendFirstFaulting_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorByteZeroExtendFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2BaseType"] = "Byte", ["GetFfrType"] = "Int64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetByte()"}), - ("SveLoadVectorFirstFaultingTest.template", new Dictionary {["TestName"] = "Sve_LoadVectorByteZeroExtendFirstFaulting_ushort", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorByteZeroExtendFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt16", ["Op2BaseType"] = "Byte", ["GetFfrType"] = "UInt16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt16()", ["NextValueOp2"] = "TestLibrary.Generator.GetByte()"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary {["TestName"] = "Sve_LoadVectorByteZeroExtendFirstFaulting_short", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorByteZeroExtendFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int16", ["Op2BaseType"] = "Byte", ["GetFfrType"] = "Int16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt16()", ["NextValueOp2"] = "TestLibrary.Generator.GetByte()"}), ("SveLoadVectorFirstFaultingTest.template", new Dictionary {["TestName"] = "Sve_LoadVectorByteZeroExtendFirstFaulting_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorByteZeroExtendFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2BaseType"] = "Byte", ["GetFfrType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetByte()"}), ("SveLoadVectorFirstFaultingTest.template", new Dictionary {["TestName"] = "Sve_LoadVectorByteZeroExtendFirstFaulting_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorByteZeroExtendFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2BaseType"] = "Byte", ["GetFfrType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetByte()"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary {["TestName"] = "Sve_LoadVectorByteZeroExtendFirstFaulting_ushort", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorByteZeroExtendFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt16", ["Op2BaseType"] = "Byte", ["GetFfrType"] = "UInt16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt16()", ["NextValueOp2"] = "TestLibrary.Generator.GetByte()"}), - ("SveLoadVectorFirstFaultingTest.template", new Dictionary {["TestName"] = "Sve_LoadVectorUInt16ZeroExtendFirstFaulting_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorUInt16ZeroExtendFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2BaseType"] = "UInt16", ["GetFfrType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt16()"}), - ("SveLoadVectorFirstFaultingTest.template", new Dictionary {["TestName"] = "Sve_LoadVectorUInt16ZeroExtendFirstFaulting_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorUInt16ZeroExtendFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2BaseType"] = "UInt16", ["GetFfrType"] = "Int64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt16()"}), - ("SveLoadVectorFirstFaultingTest.template", new Dictionary {["TestName"] = "Sve_LoadVectorUInt16ZeroExtendFirstFaulting_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorUInt16ZeroExtendFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2BaseType"] = "UInt16", ["GetFfrType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt16()"}), - ("SveLoadVectorFirstFaultingTest.template", new Dictionary {["TestName"] = "Sve_LoadVectorUInt16ZeroExtendFirstFaulting_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorUInt16ZeroExtendFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2BaseType"] = "UInt16", ["GetFfrType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt16()"}), - - ("SveLoadVectorFirstFaultingTest.template", new Dictionary {["TestName"] = "Sve_LoadVectorUInt32ZeroExtendFirstFaulting_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorUInt32ZeroExtendFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2BaseType"] = "UInt32", ["GetFfrType"] = "Int64" , ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()"}), - ("SveLoadVectorFirstFaultingTest.template", new Dictionary {["TestName"] = "Sve_LoadVectorUInt32ZeroExtendFirstFaulting_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorUInt32ZeroExtendFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2BaseType"] = "UInt32", ["GetFfrType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()"}), - - ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_float", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Single", ["GetFfrType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSingle()", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_byte", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Byte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Byte", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Byte", ["GetFfrType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetByte()", ["NextValueOp2"] = "TestLibrary.Generator.GetByte()"}), ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_double", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Double", ["GetFfrType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetDouble()", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()"}), - ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_sbyte", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "SByte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "SByte", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "SByte", ["GetFfrType"] = "SByte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSByte()", ["NextValueOp2"] = "TestLibrary.Generator.GetSByte()"}), - ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_short", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int16", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int16", ["GetFfrType"] = "Int16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt16()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt16()"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_float", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Single", ["GetFfrType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSingle()", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()"}), ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int32", ["GetFfrType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt32()"}), ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int64", ["GetFfrType"] = "Int64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt64()"}), - ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_byte", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Byte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Byte", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Byte", ["GetFfrType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetByte()", ["NextValueOp2"] = "TestLibrary.Generator.GetByte()"}), - ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_ushort", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt16", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt16", ["GetFfrType"] = "UInt16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt16()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt16()"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_sbyte", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "SByte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "SByte", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "SByte", ["GetFfrType"] = "SByte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSByte()", ["NextValueOp2"] = "TestLibrary.Generator.GetSByte()"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_short", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int16", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int16", ["GetFfrType"] = "Int16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt16()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt16()"}), ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt32", ["GetFfrType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()"}), ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt64", ["GetFfrType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_ushort", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt16", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt16", ["GetFfrType"] = "UInt16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt16()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt16()"}), ("SveLoadVectorFirstFaultingTest.template", new Dictionary {["TestName"] = "Sve_LoadVectorInt16SignExtendFirstFaulting_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorInt16SignExtendFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2BaseType"] = "Int16", ["GetFfrType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt16()"}), ("SveLoadVectorFirstFaultingTest.template", new Dictionary {["TestName"] = "Sve_LoadVectorInt16SignExtendFirstFaulting_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorInt16SignExtendFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2BaseType"] = "Int16", ["GetFfrType"] = "Int64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt16()"}), @@ -3748,12 +3740,20 @@ ("SveLoadVectorFirstFaultingTest.template", new Dictionary {["TestName"] = "Sve_LoadVectorInt32SignExtendFirstFaulting_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorInt32SignExtendFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2BaseType"] = "Int32", ["GetFfrType"] = "Int64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt32()"}), ("SveLoadVectorFirstFaultingTest.template", new Dictionary {["TestName"] = "Sve_LoadVectorInt32SignExtendFirstFaulting_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorInt32SignExtendFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2BaseType"] = "Int32", ["GetFfrType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt32()"}), - ("SveLoadVectorFirstFaultingTest.template", new Dictionary {["TestName"] = "Sve_LoadVectorSByteSignExtendFirstFaulting_short", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorSByteSignExtendFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int16", ["Op2BaseType"] = "SByte", ["GetFfrType"] = "Int16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt16()", ["NextValueOp2"] = "TestLibrary.Generator.GetSByte()"}), ("SveLoadVectorFirstFaultingTest.template", new Dictionary {["TestName"] = "Sve_LoadVectorSByteSignExtendFirstFaulting_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorSByteSignExtendFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2BaseType"] = "SByte", ["GetFfrType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetSByte()"}), ("SveLoadVectorFirstFaultingTest.template", new Dictionary {["TestName"] = "Sve_LoadVectorSByteSignExtendFirstFaulting_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorSByteSignExtendFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2BaseType"] = "SByte", ["GetFfrType"] = "Int64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetSByte()"}), - ("SveLoadVectorFirstFaultingTest.template", new Dictionary {["TestName"] = "Sve_LoadVectorSByteSignExtendFirstFaulting_ushort", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorSByteSignExtendFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt16", ["Op2BaseType"] = "SByte", ["GetFfrType"] = "UInt16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt16()", ["NextValueOp2"] = "TestLibrary.Generator.GetSByte()"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary {["TestName"] = "Sve_LoadVectorSByteSignExtendFirstFaulting_short", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorSByteSignExtendFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int16", ["Op2BaseType"] = "SByte", ["GetFfrType"] = "Int16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt16()", ["NextValueOp2"] = "TestLibrary.Generator.GetSByte()"}), ("SveLoadVectorFirstFaultingTest.template", new Dictionary {["TestName"] = "Sve_LoadVectorSByteSignExtendFirstFaulting_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorSByteSignExtendFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2BaseType"] = "SByte", ["GetFfrType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetSByte()"}), ("SveLoadVectorFirstFaultingTest.template", new Dictionary {["TestName"] = "Sve_LoadVectorSByteSignExtendFirstFaulting_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorSByteSignExtendFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2BaseType"] = "SByte", ["GetFfrType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetSByte()"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary {["TestName"] = "Sve_LoadVectorSByteSignExtendFirstFaulting_ushort", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorSByteSignExtendFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt16", ["Op2BaseType"] = "SByte", ["GetFfrType"] = "UInt16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt16()", ["NextValueOp2"] = "TestLibrary.Generator.GetSByte()"}), + + ("SveLoadVectorFirstFaultingTest.template", new Dictionary {["TestName"] = "Sve_LoadVectorUInt16ZeroExtendFirstFaulting_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorUInt16ZeroExtendFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2BaseType"] = "UInt16", ["GetFfrType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt16()"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary {["TestName"] = "Sve_LoadVectorUInt16ZeroExtendFirstFaulting_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorUInt16ZeroExtendFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2BaseType"] = "UInt16", ["GetFfrType"] = "Int64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt16()"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary {["TestName"] = "Sve_LoadVectorUInt16ZeroExtendFirstFaulting_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorUInt16ZeroExtendFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2BaseType"] = "UInt16", ["GetFfrType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt16()"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary {["TestName"] = "Sve_LoadVectorUInt16ZeroExtendFirstFaulting_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorUInt16ZeroExtendFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2BaseType"] = "UInt16", ["GetFfrType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt16()"}), + + ("SveLoadVectorFirstFaultingTest.template", new Dictionary {["TestName"] = "Sve_LoadVectorUInt32ZeroExtendFirstFaulting_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorUInt32ZeroExtendFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2BaseType"] = "UInt32", ["GetFfrType"] = "Int64" , ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary {["TestName"] = "Sve_LoadVectorUInt32ZeroExtendFirstFaulting_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorUInt32ZeroExtendFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2BaseType"] = "UInt32", ["GetFfrType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()"}), ("SveLoadVectorMaskedTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorNonTemporal_float", ["Isa"] = "Sve", ["Method"] = "LoadVectorNonTemporal", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2BaseType"] = "Single", ["LargestVectorSize"] = "64", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), ("SveLoadVectorMaskedTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorNonTemporal_double", ["Isa"] = "Sve", ["Method"] = "LoadVectorNonTemporal", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2BaseType"] = "Double", ["LargestVectorSize"] = "64", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), From 7d7d8e0bb2cf2fc2d9eff12ab1658d6368741999 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 16 Jul 2024 15:15:44 -0700 Subject: [PATCH 22/83] Add ci legs to test SVE using AltJit --- .../templates/runtimes/run-test-job.yml | 12 ++++++- .../templates/runtimes/test-variables.yml | 2 +- eng/pipelines/coreclr/jitstress-isas-sve.yml | 36 +++++++++++++++++++ src/tests/Common/testenvironment.proj | 11 ++++++ 4 files changed, 59 insertions(+), 2 deletions(-) create mode 100644 eng/pipelines/coreclr/jitstress-isas-sve.yml diff --git a/eng/pipelines/common/templates/runtimes/run-test-job.yml b/eng/pipelines/common/templates/runtimes/run-test-job.yml index c604e0c48306f..e774875be9ff9 100644 --- a/eng/pipelines/common/templates/runtimes/run-test-job.yml +++ b/eng/pipelines/common/templates/runtimes/run-test-job.yml @@ -190,7 +190,7 @@ jobs: timeoutInMinutes: 390 ${{ if in(parameters.testGroup, 'gcstress-extra', 'r2r-extra', 'clrinterpreter', 'pgo', 'pgostress', 'jit-experimental') }}: timeoutInMinutes: 510 - ${{ if in(parameters.testGroup, 'jitstress-isas-x86', 'jitstress-isas-avx512') }}: + ${{ if in(parameters.testGroup, 'jitstress-isas-x86', 'jitstress-isas-avx512', 'jitstress-isas-sve') }}: timeoutInMinutes: 960 steps: @@ -428,6 +428,16 @@ jobs: - jitstress_isas_x86_evex - jitstress_isas_x86_noavx512 - jitstressregs0x2000 + ${{ if in(parameters.testGroup, 'jitstress-isas-sve') }}: + scenarios: + - default_isas_arm64_sve + - jitstress_isas_1_arm64_sve + - jitstress_isas_2_arm64_sve + - jitstressregs_isas_1_arm64_sve + - jitstressregs_isas_2_arm64_sve + - jitstressregs_isas_3_arm64_sve + - jitstressregs_isas_4_arm64_sve + - jitstressregs_isas_8_arm64_sve ${{ if in(parameters.testGroup, 'jitstressregs-x86') }}: scenarios: - jitstressregs1_x86_noavx diff --git a/eng/pipelines/common/templates/runtimes/test-variables.yml b/eng/pipelines/common/templates/runtimes/test-variables.yml index 285f13caeb082..d0125c52c054f 100644 --- a/eng/pipelines/common/templates/runtimes/test-variables.yml +++ b/eng/pipelines/common/templates/runtimes/test-variables.yml @@ -50,7 +50,7 @@ variables: # gc reliability may take up to 2 hours to shutdown. Some scenarios have very long iteration times. - name: timeoutPerTestInMinutes value: 240 - - ${{ if in(parameters.testGroup, 'jitstress', 'jitstress-random', 'jitstress-isas-arm', 'jitstress-isas-x86', 'jitstress-isas-avx512', 'jitstressregs-x86', 'jitstressregs', 'jitstress2-jitstressregs', 'jitelthookenabled' ) }}: + - ${{ if in(parameters.testGroup, 'jitstress', 'jitstress-random', 'jitstress-isas-arm', 'jitstress-isas-x86', 'jitstress-isas-avx512', 'jitstress-isas-sve', 'jitstressregs-x86', 'jitstressregs', 'jitstress2-jitstressregs', 'jitelthookenabled' ) }}: - name: timeoutPerTestCollectionInMinutes value: 120 - name: timeoutPerTestInMinutes diff --git a/eng/pipelines/coreclr/jitstress-isas-sve.yml b/eng/pipelines/coreclr/jitstress-isas-sve.yml new file mode 100644 index 0000000000000..ec27ba55c3ed0 --- /dev/null +++ b/eng/pipelines/coreclr/jitstress-isas-sve.yml @@ -0,0 +1,36 @@ +# This pipeline only runs on GitHub PRs, not on merges. +trigger: none + +# Only run on specific changes to the JIT directory that are likely to affect Sve. +pr: + branches: + include: + - main + paths: + include: + - src/coreclr/jit/hwintrinsiccodegenarm64.cpp + - src/coreclr/jit/hwintrinsiclistarm64sve.h + - src/coreclr/jit/hwintrinsicarm64.cpp + - src/coreclr/jit/instrsarm64sve.h + - src/coreclr/jit/emitarm64sve.cpp + - src/coreclr/jit/emitfmtsarm64sve.h + - src/coreclr/jit/lsraarm64.cpp + +schedules: +- cron: "30 19 * * 6" + displayName: Sat at 11:30 AM (UTC-8:00) + branches: + include: + - main + always: true + +variables: + - template: /eng/pipelines/common/variables.yml + +extends: + template: /eng/pipelines/coreclr/templates/jit-outerloop-pipeline.yml + parameters: + platforms: + # just run on windows for now, because abi is universal for other platforms + - windows_x64 + testGroup: jitstress-isas-sve diff --git a/src/tests/Common/testenvironment.proj b/src/tests/Common/testenvironment.proj index eae6075bc0d3b..59c6a5271cf7e 100644 --- a/src/tests/Common/testenvironment.proj +++ b/src/tests/Common/testenvironment.proj @@ -172,6 +172,17 @@ + + + + + + + + + + + From 1737e6f8c1189c880d1e0b4c9541a95e6af16604 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 16 Jul 2024 15:54:36 -0700 Subject: [PATCH 23/83] fix a check for HWIntrinsic --- src/coreclr/jit/gentree.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index f7da7729d2375..0121af708708f 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -30439,7 +30439,7 @@ GenTree* Compiler::gtFoldExprHWIntrinsic(GenTreeHWIntrinsic* tree) #if defined(TARGET_XARCH) tryHandle = op->OperIsHWIntrinsic(); #elif defined(TARGET_ARM64) - if (op->OperIsHWIntrinsic() && op->OperIsHWIntrinsic(NI_Sve_CreateTrueMaskAll)) + if (op->OperIsHWIntrinsic() && op->OperIsHWIntrinsic(NI_Sve_CreateTrueMaskAll) && op2->OperIsHWIntrinsic()) { op = op2; tryHandle = true; From ca23305256818b6067017ecfc908e60d1a8bb5cf Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 16 Jul 2024 16:03:09 -0700 Subject: [PATCH 24/83] add gentree.cpp for now --- eng/pipelines/coreclr/jitstress-isas-sve.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eng/pipelines/coreclr/jitstress-isas-sve.yml b/eng/pipelines/coreclr/jitstress-isas-sve.yml index ec27ba55c3ed0..ab2ee5ed5b75e 100644 --- a/eng/pipelines/coreclr/jitstress-isas-sve.yml +++ b/eng/pipelines/coreclr/jitstress-isas-sve.yml @@ -15,6 +15,7 @@ pr: - src/coreclr/jit/emitarm64sve.cpp - src/coreclr/jit/emitfmtsarm64sve.h - src/coreclr/jit/lsraarm64.cpp + - src/coreclr/jit/gentree.cpp schedules: - cron: "30 19 * * 6" From 0ef018f4e0d6cd798fc427ee319206e3de1067de Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 16 Jul 2024 16:07:51 -0700 Subject: [PATCH 25/83] Revert "add gentree.cpp for now" This reverts commit ca23305256818b6067017ecfc908e60d1a8bb5cf. --- eng/pipelines/coreclr/jitstress-isas-sve.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/eng/pipelines/coreclr/jitstress-isas-sve.yml b/eng/pipelines/coreclr/jitstress-isas-sve.yml index ab2ee5ed5b75e..ec27ba55c3ed0 100644 --- a/eng/pipelines/coreclr/jitstress-isas-sve.yml +++ b/eng/pipelines/coreclr/jitstress-isas-sve.yml @@ -15,7 +15,6 @@ pr: - src/coreclr/jit/emitarm64sve.cpp - src/coreclr/jit/emitfmtsarm64sve.h - src/coreclr/jit/lsraarm64.cpp - - src/coreclr/jit/gentree.cpp schedules: - cron: "30 19 * * 6" From d781fdc9e74bcb6a0b6c667108022c67463ff2d0 Mon Sep 17 00:00:00 2001 From: TIHan Date: Tue, 16 Jul 2024 16:11:35 -0700 Subject: [PATCH 26/83] Mark GetFfr methods as side-effectful --- src/coreclr/jit/fgdiagnostic.cpp | 8 ++++++++ src/coreclr/jit/gentree.cpp | 16 ++++++++++++++++ src/coreclr/jit/hwintrinsiclistarm64sve.h | 16 ++++++++-------- 3 files changed, 32 insertions(+), 8 deletions(-) diff --git a/src/coreclr/jit/fgdiagnostic.cpp b/src/coreclr/jit/fgdiagnostic.cpp index 322aae30ec500..067890f3d9b84 100644 --- a/src/coreclr/jit/fgdiagnostic.cpp +++ b/src/coreclr/jit/fgdiagnostic.cpp @@ -3468,6 +3468,14 @@ void Compiler::fgDebugCheckFlags(GenTree* tree, BasicBlock* block) case NI_Sve_GatherPrefetch64Bit: case NI_Sve_GatherPrefetch8Bit: case NI_Sve_SetFfr: + case NI_Sve_GetFfrByte: + case NI_Sve_GetFfrInt16: + case NI_Sve_GetFfrInt32: + case NI_Sve_GetFfrInt64: + case NI_Sve_GetFfrSByte: + case NI_Sve_GetFfrUInt16: + case NI_Sve_GetFfrUInt32: + case NI_Sve_GetFfrUInt64: { assert(tree->OperRequiresCallFlag(this)); expectedFlags |= GTF_GLOB_REF; diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 9aedea5052d6c..a188f8f85c78d 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -27129,6 +27129,14 @@ bool GenTreeHWIntrinsic::OperRequiresCallFlag() const case NI_Sve_GatherPrefetch64Bit: case NI_Sve_GatherPrefetch8Bit: case NI_Sve_SetFfr: + case NI_Sve_GetFfrByte: + case NI_Sve_GetFfrInt16: + case NI_Sve_GetFfrInt32: + case NI_Sve_GetFfrInt64: + case NI_Sve_GetFfrSByte: + case NI_Sve_GetFfrUInt16: + case NI_Sve_GetFfrUInt32: + case NI_Sve_GetFfrUInt64: { return true; } @@ -27320,6 +27328,14 @@ void GenTreeHWIntrinsic::Initialize(NamedIntrinsic intrinsicId) case NI_Sve_GatherPrefetch64Bit: case NI_Sve_GatherPrefetch8Bit: case NI_Sve_SetFfr: + case NI_Sve_GetFfrByte: + case NI_Sve_GetFfrInt16: + case NI_Sve_GetFfrInt32: + case NI_Sve_GetFfrInt64: + case NI_Sve_GetFfrSByte: + case NI_Sve_GetFfrUInt16: + case NI_Sve_GetFfrUInt32: + case NI_Sve_GetFfrUInt64: { // Mark as a call and global reference, much as is done for GT_KEEPALIVE gtFlags |= (GTF_CALL | GTF_GLOB_REF); diff --git a/src/coreclr/jit/hwintrinsiclistarm64sve.h b/src/coreclr/jit/hwintrinsiclistarm64sve.h index b7420689beb45..8019844558838 100644 --- a/src/coreclr/jit/hwintrinsiclistarm64sve.h +++ b/src/coreclr/jit/hwintrinsiclistarm64sve.h @@ -118,14 +118,14 @@ HARDWARE_INTRINSIC(Sve, GatherVectorUInt32WithByteOffsetsZeroExtend, HARDWARE_INTRINSIC(Sve, GatherVectorUInt32ZeroExtend, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_sve_ld1w, INS_sve_ld1w, INS_sve_ld1w, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, GatherVectorWithByteOffsets, -1, 3, true, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_sve_ld1w, INS_sve_ld1d, INS_sve_ld1d, INS_sve_ld1w, INS_sve_ld1d}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, GetActiveElementCount, -1, 2, true, {INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ExplicitMaskedOperation) -HARDWARE_INTRINSIC(Sve, GetFfrByte, -1, 0, false, {INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, GetFfrInt16, -1, 0, false, {INS_invalid, INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, GetFfrInt32, -1, 0, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, GetFfrInt64, -1, 0, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, GetFfrSByte, -1, 0, false, {INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, GetFfrUInt16, -1, 0, false, {INS_invalid, INS_invalid, INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, GetFfrUInt32, -1, 0, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, GetFfrUInt64, -1, 0, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, GetFfrByte, -1, 0, false, {INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, GetFfrInt16, -1, 0, false, {INS_invalid, INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, GetFfrInt32, -1, 0, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, GetFfrInt64, -1, 0, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, GetFfrSByte, -1, 0, false, {INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, GetFfrUInt16, -1, 0, false, {INS_invalid, INS_invalid, INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, GetFfrUInt32, -1, 0, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, GetFfrUInt64, -1, 0, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffectMask) HARDWARE_INTRINSIC(Sve, InsertIntoShiftedVector, -1, 2, true, {INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics) HARDWARE_INTRINSIC(Sve, LeadingSignCount, -1, -1, false, {INS_sve_cls, INS_invalid, INS_sve_cls, INS_invalid, INS_sve_cls, INS_invalid, INS_sve_cls, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LeadingZeroCount, -1, -1, false, {INS_sve_clz, INS_sve_clz, INS_sve_clz, INS_sve_clz, INS_sve_clz, INS_sve_clz, INS_sve_clz, INS_sve_clz, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) From c8649009077b10cf996450a64f0aff91747cdea3 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 16 Jul 2024 19:01:27 -0700 Subject: [PATCH 27/83] review feedback --- .../common/templates/runtimes/run-test-job.yml | 16 ++++++++-------- src/coreclr/jit/gentree.cpp | 4 ++-- src/tests/Common/testenvironment.proj | 16 ++++++++-------- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/eng/pipelines/common/templates/runtimes/run-test-job.yml b/eng/pipelines/common/templates/runtimes/run-test-job.yml index e774875be9ff9..b810d7a428e16 100644 --- a/eng/pipelines/common/templates/runtimes/run-test-job.yml +++ b/eng/pipelines/common/templates/runtimes/run-test-job.yml @@ -430,14 +430,14 @@ jobs: - jitstressregs0x2000 ${{ if in(parameters.testGroup, 'jitstress-isas-sve') }}: scenarios: - - default_isas_arm64_sve - - jitstress_isas_1_arm64_sve - - jitstress_isas_2_arm64_sve - - jitstressregs_isas_1_arm64_sve - - jitstressregs_isas_2_arm64_sve - - jitstressregs_isas_3_arm64_sve - - jitstressregs_isas_4_arm64_sve - - jitstressregs_isas_8_arm64_sve + - default_arm64_sve + - jitstress1_arm64_sve + - jitstress2_arm64_sve + - jitstressregs1_arm64_sve + - jitstressregs2_arm64_sve + - jitstressregs3_arm64_sve + - jitstressregs4_arm64_sve + - jitstressregs8_arm64_sve ${{ if in(parameters.testGroup, 'jitstressregs-x86') }}: scenarios: - jitstressregs1_x86_noavx diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 0121af708708f..ef54b89d9c319 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -30439,10 +30439,10 @@ GenTree* Compiler::gtFoldExprHWIntrinsic(GenTreeHWIntrinsic* tree) #if defined(TARGET_XARCH) tryHandle = op->OperIsHWIntrinsic(); #elif defined(TARGET_ARM64) - if (op->OperIsHWIntrinsic() && op->OperIsHWIntrinsic(NI_Sve_CreateTrueMaskAll) && op2->OperIsHWIntrinsic()) + if (op->OperIsHWIntrinsic(NI_Sve_CreateTrueMaskAll)) { op = op2; - tryHandle = true; + tryHandle = op->OperIsHWIntrinsic(); } #endif // TARGET_ARM64 diff --git a/src/tests/Common/testenvironment.proj b/src/tests/Common/testenvironment.proj index 59c6a5271cf7e..e1a4d074605e6 100644 --- a/src/tests/Common/testenvironment.proj +++ b/src/tests/Common/testenvironment.proj @@ -174,14 +174,14 @@ - - - - - - - - + + + + + + + + From f7e2a2cc96639fc6a2a0d611c0b25d311e444e18 Mon Sep 17 00:00:00 2001 From: Mikhail Ablakatov Date: Wed, 17 Jul 2024 21:09:11 +0000 Subject: [PATCH 28/83] Addressing review comments. --- src/coreclr/jit/lsraarm64.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/coreclr/jit/lsraarm64.cpp b/src/coreclr/jit/lsraarm64.cpp index 266cb47537d50..d16ba2f17c4b5 100644 --- a/src/coreclr/jit/lsraarm64.cpp +++ b/src/coreclr/jit/lsraarm64.cpp @@ -2027,7 +2027,13 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou { switch (intrin.id) { + case NI_Sve_LoadVectorByteZeroExtendFirstFaulting: + case NI_Sve_LoadVectorInt16SignExtendFirstFaulting: + case NI_Sve_LoadVectorInt32SignExtendFirstFaulting: case NI_Sve_LoadVectorNonTemporal: + case NI_Sve_LoadVectorSByteSignExtendFirstFaulting: + case NI_Sve_LoadVectorUInt16ZeroExtendFirstFaulting: + case NI_Sve_LoadVectorUInt32ZeroExtendFirstFaulting: case NI_Sve_LoadVector128AndReplicateToVector: case NI_Sve_StoreAndZip: assert(intrinsicTree->OperIsMemoryLoadOrStore()); From a73fe353b283d8deaf2c6b5070b22e0c5c34c387 Mon Sep 17 00:00:00 2001 From: TIHan Date: Fri, 19 Jul 2024 16:38:04 -0700 Subject: [PATCH 29/83] Verifying expected fault result. Test weaks. --- .../GenerateHWIntrinsicTests_Arm.cs | 18 +-- .../HardwareIntrinsics/Arm/Shared/Helpers.cs | 122 +++++++++++++++++- ...eGatherVectorFirstFaultingIndices.template | 45 +++++-- ...herVectorFirstFaultingVectorBases.template | 82 ++++++++++++ .../SveLoadVectorFirstFaultingTest.template | 8 +- 5 files changed, 248 insertions(+), 27 deletions(-) diff --git a/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs b/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs index 50b052a2fd5c6..0ba683a3e691a 100644 --- a/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs +++ b/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs @@ -3616,20 +3616,20 @@ ("SveGatherVectorIndices.template", new Dictionary {["TestName"] = "Sve_GatherVectorUInt32ZeroExtend_Indices_ulong_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorUInt32ZeroExtend", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2BaseType"] = "UInt32", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt64", ["ExtendedElementType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt64()"}), ("SveGatherVectorIndices.template", new Dictionary {["TestName"] = "Sve_GatherVectorUInt32ZeroExtend_Indices_uint_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorUInt32ZeroExtend", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2BaseType"] = "UInt32", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt32", ["ExtendedElementType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt32()"}), - ("SveGatherVectorFirstFaultingVectorBases.template", new Dictionary {["TestName"] = "Sve_GatherVectorFirstFaulting_Bases_double_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt64", ["ExtendedElementType"] = "Double", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskDouble()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()", ["NextValueBase"] = "TestLibrary.Generator.GetDouble()"}), - ("SveGatherVectorFirstFaultingVectorBases.template", new Dictionary {["TestName"] = "Sve_GatherVectorFirstFaulting_Bases_long_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt64", ["ExtendedElementType"] = "Int64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()", ["NextValueBase"] = "TestLibrary.Generator.GetInt64()"}), - ("SveGatherVectorFirstFaultingVectorBases.template", new Dictionary {["TestName"] = "Sve_GatherVectorFirstFaulting_Bases_ulong_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt64", ["ExtendedElementType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()", ["NextValueBase"] = "TestLibrary.Generator.GetUInt64()"}), + ("SveGatherVectorFirstFaultingVectorBases.template", new Dictionary {["TestName"] = "Sve_GatherVectorFirstFaulting_Bases_double_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt64", ["ExtendedElementType"] = "Double", ["GetFfrType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskDouble()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()", ["NextValueBase"] = "TestLibrary.Generator.GetDouble()"}), + ("SveGatherVectorFirstFaultingVectorBases.template", new Dictionary {["TestName"] = "Sve_GatherVectorFirstFaulting_Bases_long_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt64", ["ExtendedElementType"] = "Int64", ["GetFfrType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()", ["NextValueBase"] = "TestLibrary.Generator.GetInt64()"}), + ("SveGatherVectorFirstFaultingVectorBases.template", new Dictionary {["TestName"] = "Sve_GatherVectorFirstFaulting_Bases_ulong_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt64", ["ExtendedElementType"] = "UInt64", ["GetFfrType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()", ["NextValueBase"] = "TestLibrary.Generator.GetUInt64()"}), - ("SveGatherVectorFirstFaultingIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_float_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2BaseType"] = "Single", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int32", ["ExtendedElementType"] = "Single", ["GetFfrType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskSingle()", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt32()"}), + ("SveGatherVectorFirstFaultingIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_float_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2BaseType"] = "Single", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int32", ["ExtendedElementType"] = "Single", ["GetFfrType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskSingle()", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt32()"}), ("SveGatherVectorFirstFaultingIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_int_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2BaseType"] = "Int32", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int32", ["ExtendedElementType"] = "Int32", ["GetFfrType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt32()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt32()"}), ("SveGatherVectorFirstFaultingIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_uint_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2BaseType"] = "UInt32", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int32", ["ExtendedElementType"] = "UInt32", ["GetFfrType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt32()"}), - ("SveGatherVectorFirstFaultingIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_float_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2BaseType"] = "Single", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt32", ["ExtendedElementType"] = "Single", ["GetFfrType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskSingle()", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt32()"}), + ("SveGatherVectorFirstFaultingIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_float_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2BaseType"] = "Single", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt32", ["ExtendedElementType"] = "Single", ["GetFfrType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskSingle()", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt32()"}), ("SveGatherVectorFirstFaultingIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_int_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2BaseType"] = "Int32", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt32", ["ExtendedElementType"] = "Int32", ["GetFfrType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt32()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt32()"}), ("SveGatherVectorFirstFaultingIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_uint_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2BaseType"] = "UInt32", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt32", ["ExtendedElementType"] = "UInt32", ["GetFfrType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt32()"}), - ("SveGatherVectorFirstFaultingIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_double_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2BaseType"] = "Double", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int64", ["ExtendedElementType"] = "Double", ["GetFfrType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskDouble()", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt64()"}), + ("SveGatherVectorFirstFaultingIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_double_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2BaseType"] = "Double", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int64", ["ExtendedElementType"] = "Double", ["GetFfrType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskDouble()", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt64()"}), ("SveGatherVectorFirstFaultingIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_long_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2BaseType"] = "Int64", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int64", ["ExtendedElementType"] = "Int64", ["GetFfrType"] = "Int64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt64()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt64()"}), ("SveGatherVectorFirstFaultingIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_ulong_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2BaseType"] = "UInt64", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int64", ["ExtendedElementType"] = "UInt64", ["GetFfrType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt64()"}), - ("SveGatherVectorFirstFaultingIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_double_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2BaseType"] = "Double", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt64", ["ExtendedElementType"] = "Double", ["GetFfrType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskDouble()", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt64()"}), + ("SveGatherVectorFirstFaultingIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_double_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2BaseType"] = "Double", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt64", ["ExtendedElementType"] = "Double", ["GetFfrType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskDouble()", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt64()"}), ("SveGatherVectorFirstFaultingIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_long_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2BaseType"] = "Int64", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt64", ["ExtendedElementType"] = "Int64", ["GetFfrType"] = "Int64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt64()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt64()"}), ("SveGatherVectorFirstFaultingIndices.template", new Dictionary { ["TestName"] = "Sve_GatherVectorFirstFaulting_Indices_ulong_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "GatherVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2BaseType"] = "UInt64", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt64", ["ExtendedElementType"] = "UInt64", ["GetFfrType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt64()"}), @@ -3692,8 +3692,8 @@ ("SveLoadNonFaultingUnOpTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorNonFaulting_uint", ["Isa"] = "Sve", ["Method"] = "LoadVectorNonFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2BaseType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt32()", ["Cast"] = "(uint*)", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), ("SveLoadNonFaultingUnOpTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorNonFaulting_ulong", ["Isa"] = "Sve", ["Method"] = "LoadVectorNonFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2BaseType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt64()", ["Cast"] = "(ulong*)", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_float", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Single", ["GetFfrType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSingle()", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()"}), - ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_double", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Double", ["GetFfrType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetDouble()", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_float", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Single", ["GetFfrType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSingle()", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()"}), + ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_double", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Double", ["GetFfrType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetDouble()", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()"}), ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_sbyte", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "SByte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "SByte", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "SByte", ["GetFfrType"] = "SByte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSByte()", ["NextValueOp2"] = "TestLibrary.Generator.GetSByte()"}), ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_short", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int16", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int16", ["GetFfrType"] = "Int16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt16()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt16()"}), ("SveLoadVectorFirstFaultingTest.template", new Dictionary { ["TestName"] = "Sve_LoadVectorFirstFaulting_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "LoadVectorFirstFaulting", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int32", ["GetFfrType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt32()"}), diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs index a4a57bc9e89b0..0f547a235b38a 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs @@ -8248,7 +8248,9 @@ public static bool CheckGatherVectorBasesConditionalSelectTrueBehavior(mask, data, result, (i, gatherResult) => ConditionalSelectTrueResult(cndSelMask[i], gatherResult, cndSelTrue[i])); } - private static bool CheckFirstFaultingBehaviorCore(T[] result, Vector faultResult, Func checkIter) where T : INumberBase + private static bool CheckFirstFaultingBehaviorCore(T[] result, Vector faultResult, Func checkIter) + where T : INumberBase + where TFault : INumberBase { bool hitFault = false; @@ -8256,14 +8258,14 @@ private static bool CheckFirstFaultingBehaviorCore(T[] result, Vector faul { if (hitFault) { - if (faultResult[i] != T.Zero) + if (faultResult[i] != TFault.Zero) { return false; } } else { - if (faultResult[i] == T.Zero) + if (faultResult[i] == TFault.Zero) { // There has to be a valid value for the first element, so check it. if (i == 0) @@ -8285,24 +8287,132 @@ private static bool CheckFirstFaultingBehaviorCore(T[] result, Vector faul return true; } - public static bool CheckLoadVectorFirstFaultingBehavior(T[] firstOp, T[] result, Vector faultResult) where T : INumberBase + private static bool CheckFaultResultHasAtLeastOneZero(Vector faultResult) where T : INumberBase + { + for (var i = 0; i < Vector.Count; i++) + { + if (faultResult[i] == T.Zero) + { + return true; + } + } + return false; + } + + public static bool CheckLoadVectorFirstFaultingBehavior(T[] firstOp, T[] result, Vector faultResult) + where T : INumberBase + where TFault : INumberBase { + // Checking first faulting behavior requires at least one zero to ensure we are testing the behavior. + if (!CheckFaultResultHasAtLeastOneZero(faultResult)) + { + TestLibrary.TestFramework.LogInformation("Fault result requires at least one zero."); + return false; + } + + var validElementCount = firstOp.Length; + var expectedFaultResult = + InitVector(i => + { + if (i < validElementCount) + { + return TFault.One; + } + return TFault.Zero; + }); + if (expectedFaultResult != faultResult) + { + TestLibrary.TestFramework.LogInformation($"Expected fault result: {expectedFaultResult}\nActual fault result: {faultResult}"); + return false; + } + return CheckFirstFaultingBehaviorCore(result, faultResult, i => firstOp[i] == result[i]); } - public static bool CheckGatherVectorFirstFaultingBehavior(T[] mask, ExtendedElementT[] data, Index[] indices, T[] result, Vector faultResult) + public static bool CheckGatherVectorFirstFaultingBehavior(T[] mask, ExtendedElementT[] data, Index[] indices, T[] result, Vector faultResult) where T : INumberBase where ExtendedElementT : INumberBase where Index : IBinaryInteger + where TFault : INumberBase { + // Checking first faulting behavior requires at least one zero to ensure we are testing the behavior. + if (!CheckFaultResultHasAtLeastOneZero(faultResult)) + { + TestLibrary.TestFramework.LogInformation("Fault result requires at least one zero."); + return false; + } + + var hasFaulted = false; + var expectedFaultResult = + InitVector(i => + { + if (hasFaulted) + { + return TFault.Zero; + } + + if (mask[i] == T.Zero) + { + return TFault.One; + } + + var index = int.CreateChecked(indices[i]); + if (index < 0 || index >= data.Length) + { + hasFaulted = true; + return TFault.Zero; + } + return TFault.One; + }); + if (expectedFaultResult != faultResult) + { + TestLibrary.TestFramework.LogInformation($"Expected fault result: {expectedFaultResult}\nActual fault result: {faultResult}"); + return false; + } + return CheckFirstFaultingBehaviorCore(result, faultResult, i => GetGatherVectorResultByIndex(i, mask, data, indices) == result[i]); } - public static bool CheckGatherVectorBasesFirstFaultingBehavior(T[] mask, AddressT[] data, T[] result, Vector faultResult) + public static bool CheckGatherVectorBasesFirstFaultingBehavior(T[] mask, AddressT[] data, T[] result, Vector faultResult) where T : INumberBase where AddressT : unmanaged, INumberBase where ExtendedElementT : unmanaged, INumberBase + where TFault : INumberBase { + // Checking first faulting behavior requires at least one zero to ensure we are testing the behavior. + if (!CheckFaultResultHasAtLeastOneZero(faultResult)) + { + TestLibrary.TestFramework.LogInformation("Fault result requires at least one zero."); + return false; + } + + var hasFaulted = false; + var expectedFaultResult = + InitVector(i => + { + if (hasFaulted) + { + return TFault.Zero; + } + + if (mask[i] == T.Zero) + { + return TFault.One; + } + + if (data[i] == AddressT.Zero) + { + hasFaulted = true; + return TFault.Zero; + } + return TFault.One; + }); + if (expectedFaultResult != faultResult) + { + TestLibrary.TestFramework.LogInformation($"Expected fault result: {expectedFaultResult}\nActual fault result: {faultResult}"); + return false; + } + return CheckFirstFaultingBehaviorCore(result, faultResult, i => GetGatherVectorBasesResultByIndex(i, mask, data) == result[i]); } diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorFirstFaultingIndices.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorFirstFaultingIndices.template index c45dfc62d86d2..f06ff9f2e2e6d 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorFirstFaultingIndices.template +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorFirstFaultingIndices.template @@ -286,18 +286,44 @@ namespace JIT.HardwareIntrinsics.Arm ref var op2Ref = ref (_dataTable.inBounded.Span.GetPinnableReference()); var op3 = {LoadIsa}.Load{Op3VectorType}(loadMask3, ({Op3BaseType}*)(_dataTable.inArray3Ptr)); + // We know this is outside the bounds because 'inBounded' will never be the full size of a Vector. + var outsideBoundsIndex = ({Op3BaseType})(Vector<{Op2BaseType}>.Count - 1); + // When testing first-faulting behavior, we need to make sure we can get the first element. // So set the first active element of the index vector to 0. + var firstActiveElement = -1; for (var i = 0; i < Vector<{Op3BaseType}>.Count; i++) { // op1 is the mask for GatherVector. if (op1[i] != 0) { - op3 = op3.WithElement<{Op3BaseType}>(i, 0); - break; + if (firstActiveElement == -1) + { + op3 = op3.WithElement<{Op3BaseType}>(i, 0); + firstActiveElement = i; + } + else if (op3[i] < 0 || op3[i] > outsideBoundsIndex) + { + op3 = op3.WithElement<{Op3BaseType}>(i, outsideBoundsIndex); + } } } + // Force at least one element to cause a fault (required for testing). So set the last element to an index outside the bounds. + var lastIndex = Vector<{Op3BaseType}>.Count - 1; + + // Ensure we at least have one element that we can read. + if ((firstActiveElement == -1) || (firstActiveElement == lastIndex)) + { + op1 = op1.WithElement<{Op1BaseType}>(0, 1); + op3 = op3.WithElement<{Op3BaseType}>(0, 0); + } + + op1 = op1.WithElement<{Op1BaseType}>(lastIndex, 1); + + // Force an index outside the bounds. + op3 = op3.WithElement<{Op3BaseType}>(lastIndex, outsideBoundsIndex); + Sve.SetFfr(Sve.CreateTrueMaskByte(SveMaskPattern.All)); var result = {Isa}.{Method}( op1, @@ -306,10 +332,11 @@ namespace JIT.HardwareIntrinsics.Arm ); var faultResult = Sve.GetFfr{GetFfrType}(); + ref var op1Ref = ref op1; ref var op3Ref = ref op3; Unsafe.Write(_dataTable.outArrayPtr, result); - ValidateFirstFaultingResult(_dataTable.inArray1Ptr, ref op2Ref, _dataTable.inBounded.Span.Length, Unsafe.AsPointer(ref op3Ref), _dataTable.outArrayPtr, faultResult.As<{GetFfrType}, {RetBaseType}>()); + ValidateFirstFaultingResult(Unsafe.AsPointer(ref op1Ref), ref op2Ref, _dataTable.inBounded.Span.Length, Unsafe.AsPointer(ref op3Ref), _dataTable.outArrayPtr, faultResult); } public void RunBasicScenario_FalseMask() @@ -599,22 +626,22 @@ namespace JIT.HardwareIntrinsics.Arm } } - private void ValidateFirstFaultingResult(void* op1, ref byte op2, int op2Size, void* op3, void* result, Vector<{RetBaseType}> faultResult, [CallerMemberName] string method = "") + private void ValidateFirstFaultingResult(void* op1, ref byte op2, int op2Size, void* op3, void* result, Vector<{GetFfrType}> faultResult, [CallerMemberName] string method = "") { - {Op1BaseType}[] inArray1 = new {Op1BaseType}[Op1ElementCount]; + {Op1BaseType}[] inArray1 = new {Op1BaseType}[Vector<{Op1BaseType}>.Count]; {Op2BaseType}[] inArray2 = new {Op2BaseType}[op2Size / Unsafe.SizeOf<{Op2BaseType}>()]; - {Op3BaseType}[] inArray3 = new {Op3BaseType}[Op3ElementCount]; + {Op3BaseType}[] inArray3 = new {Op3BaseType}[Vector<{Op3BaseType}>.Count]; {RetBaseType}[] outArray = new {RetBaseType}[RetElementCount]; - Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref inArray1[0]), ref Unsafe.AsRef(op1), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref inArray1[0]), ref Unsafe.AsRef(op1), (uint)(inArray1.Length * Unsafe.SizeOf<{Op1BaseType}>())); Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op2BaseType}, byte>(ref inArray2[0]), ref op2, (uint)(inArray2.Length * Unsafe.SizeOf<{Op2BaseType}>())); - Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op3BaseType}, byte>(ref inArray3[0]), ref Unsafe.AsRef(op3), (uint)Unsafe.SizeOf<{Op3VectorType}<{Op3BaseType}>>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op3BaseType}, byte>(ref inArray3[0]), ref Unsafe.AsRef(op3), (uint)(inArray3.Length * Unsafe.SizeOf<{Op3BaseType}>())); Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref outArray[0]), ref Unsafe.AsRef(result), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); ValidateFirstFaultingResult(inArray1, inArray2, inArray3, outArray, faultResult, method); } - private void ValidateFirstFaultingResult({Op1BaseType}[] firstOp, {Op2BaseType}[] secondOp, {Op3BaseType}[] thirdOp, {RetBaseType}[] result, Vector<{RetBaseType}> faultResult, [CallerMemberName] string method = "") + private void ValidateFirstFaultingResult({Op1BaseType}[] firstOp, {Op2BaseType}[] secondOp, {Op3BaseType}[] thirdOp, {RetBaseType}[] result, Vector<{GetFfrType}> faultResult, [CallerMemberName] string method = "") { var succeeded = Helpers.CheckGatherVectorFirstFaultingBehavior(firstOp, secondOp, thirdOp, result, faultResult); diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorFirstFaultingVectorBases.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorFirstFaultingVectorBases.template index d4bc14542bef4..5ba9b48264f1f 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorFirstFaultingVectorBases.template +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorFirstFaultingVectorBases.template @@ -9,6 +9,7 @@ ******************************************************************************/ using System; +using System.Buffers; using System.Numerics; using System.Reflection; using System.Runtime.CompilerServices; @@ -34,6 +35,9 @@ namespace JIT.HardwareIntrinsics.Arm // Validates basic functionality works test.RunBasicScenario_Load(); + // Validates basic functionality of first-faulting behavior + test.RunBasicScenario_LoadFirstFaulting(); + // Validates fully masked out load works. test.RunBasicScenario_FalseMask(); @@ -90,11 +94,13 @@ namespace JIT.HardwareIntrinsics.Arm { private byte[] inArray1; private byte[] inArray2; + private byte[] inArray2Ffr; private byte[] baseArray; private byte[] outArray; private GCHandle inHandle1; private GCHandle inHandle2; + private GCHandle inHandle2Ffr; private GCHandle baseHandle; private GCHandle outHandle; @@ -114,11 +120,13 @@ namespace JIT.HardwareIntrinsics.Arm this.inArray1 = new byte[alignment * 2]; this.inArray2 = new byte[alignment * 2]; + this.inArray2Ffr = new byte[alignment * 2]; this.baseArray = new byte[alignment + sizeOfBaseArray]; this.outArray = new byte[alignment * 2]; this.inHandle1 = GCHandle.Alloc(this.inArray1, GCHandleType.Pinned); this.inHandle2 = GCHandle.Alloc(this.inArray2, GCHandleType.Pinned); + this.inHandle2Ffr = GCHandle.Alloc(this.inArray2Ffr, GCHandleType.Pinned); this.baseHandle = GCHandle.Alloc(this.baseArray, GCHandleType.Pinned); this.outHandle = GCHandle.Alloc(this.outArray, GCHandleType.Pinned); @@ -127,13 +135,34 @@ namespace JIT.HardwareIntrinsics.Arm Unsafe.CopyBlockUnaligned(ref Unsafe.AsRef(inArray1Ptr), ref Unsafe.As<{Op1BaseType}, byte>(ref inArray1[0]), (uint)sizeOfinArray1); Unsafe.CopyBlockUnaligned(ref Unsafe.AsRef(baseArrayPtr), ref Unsafe.As<{RetBaseType}, byte>(ref baseArray[0]), (uint)sizeOfBaseArray); + var inArray2Ffr = new {Op2BaseType}[inArray2.Length]; + inArray2.CopyTo(inArray2Ffr, 0); + // Add the base pointer to the offsets within inArray2 to create complete pointers. for (var i = 0; i < inArray2.Length; i++) { inArray2[i] += ({Op2BaseType})baseArrayPtr; } Unsafe.CopyBlockUnaligned(ref Unsafe.AsRef(inArray2Ptr), ref Unsafe.As<{Op2BaseType}, byte>(ref inArray2[0]), (uint)sizeOfinArray2); + + // Add the base pointer to the offsets within inArray2Ffr to create complete pointers. + var random = new Random(); + for (var i = 0; i < inArray2Ffr.Length; i++) + { + // In order to test the first-faulting behavior, we need to put in null pointers after the first active element (element 0). + // We will always cause a fault on the last element for testing. + if (((i != 0) && (random.Next() % 2) == 0) || (i == (inArray2Ffr.Length - 1))) + { + inArray2Ffr[i] = default; // nullptr + } + else + { + inArray2Ffr[i] += ({Op2BaseType})baseArrayPtr; + } + } + Unsafe.CopyBlockUnaligned(ref Unsafe.AsRef(inArray2FfrPtr), ref Unsafe.As<{Op2BaseType}, byte>(ref inArray2Ffr[0]), (uint)sizeOfinArray2); } public void* inArray1Ptr => Align((byte*)(inHandle1.AddrOfPinnedObject().ToPointer()), alignment); public void* inArray2Ptr => Align((byte*)(inHandle2.AddrOfPinnedObject().ToPointer()), alignment); + public void* inArray2FfrPtr => Align((byte*)(inHandle2Ffr.AddrOfPinnedObject().ToPointer()), alignment); public void* baseArrayPtr => Align((byte*)(baseHandle.AddrOfPinnedObject().ToPointer()), alignment); public void* outArrayPtr => Align((byte*)(outHandle.AddrOfPinnedObject().ToPointer()), alignment); @@ -141,6 +170,7 @@ namespace JIT.HardwareIntrinsics.Arm { inHandle1.Free(); inHandle2.Free(); + inHandle2Ffr.Free(); baseHandle.Free(); outHandle.Free(); } @@ -265,6 +295,28 @@ namespace JIT.HardwareIntrinsics.Arm ValidateResult(_dataTable.inArray1Ptr, _dataTable.inArray2Ptr, _dataTable.outArrayPtr); } + public void RunBasicScenario_LoadFirstFaulting() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunBasicScenario_LoadFirstFaulting)); + + {Op1VectorType}<{Op1BaseType}> loadMask1 = Sve.CreateTrueMask{Op1BaseType}(SveMaskPattern.All); + {Op2VectorType}<{Op2BaseType}> loadMask2 = Sve.CreateTrueMask{Op2BaseType}(SveMaskPattern.All); + + var op1 = {LoadIsa}.Load{Op1VectorType}(loadMask1, ({Op1BaseType}*)(_dataTable.inArray1Ptr)); + var op2 = {LoadIsa}.Load{Op2VectorType}(loadMask2, ({Op2BaseType}*)(_dataTable.inArray2FfrPtr)); + + // Force op1 (mask) to have the first and last element to be active. + op1 = op1.WithElement<{Op1BaseType}>(0, 1).WithElement<{Op1BaseType}>(Vector<{Op1BaseType}>.Count - 1, 1); + + Sve.SetFfr(Sve.CreateTrueMaskByte(SveMaskPattern.All)); + var result = {Isa}.{Method}(op1, op2); + var faultResult = Sve.GetFfr{GetFfrType}(); + + ref var op1Ref = ref op1; + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateFirstFaultingResult(Unsafe.AsPointer(ref op1Ref), _dataTable.inArray2FfrPtr, _dataTable.outArrayPtr, faultResult); + } public void RunBasicScenario_FalseMask() { @@ -596,5 +648,35 @@ namespace JIT.HardwareIntrinsics.Arm Succeeded = false; } } + + private void ValidateFirstFaultingResult(void* op1, void* op2, void* result, Vector<{GetFfrType}> faultResult, [CallerMemberName] string method = "") + { + {Op1BaseType}[] inArray1 = new {Op1BaseType}[Op1ElementCount]; + {Op2BaseType}[] inArray2 = new {Op2BaseType}[Op2ElementCount]; + {RetBaseType}[] outArray = new {RetBaseType}[RetElementCount]; + + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref inArray1[0]), ref Unsafe.AsRef(op1), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op2BaseType}, byte>(ref inArray2[0]), ref Unsafe.AsRef(op2), (uint)Unsafe.SizeOf<{Op2VectorType}<{Op2BaseType}>>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref outArray[0]), ref Unsafe.AsRef(result), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); + + ValidateFirstFaultingResult(inArray1, inArray2, outArray, faultResult, method); + } + + private void ValidateFirstFaultingResult({Op1BaseType}[] firstOp, {Op2BaseType}[] secondOp, {RetBaseType}[] result, Vector<{GetFfrType}> faultResult, [CallerMemberName] string method = "") + { + var succeeded = Helpers.CheckGatherVectorBasesFirstFaultingBehavior<{Op1BaseType}, {Op2BaseType}, {ExtendedElementType}, {GetFfrType}>(firstOp, secondOp, result, faultResult); + + if (!succeeded) + { + TestLibrary.TestFramework.LogInformation($"{nameof({Isa})}.{nameof({Isa}.{Method})}<{RetBaseType}>({Op1BaseType}, {Op2BaseType}): {method} failed:"); + TestLibrary.TestFramework.LogInformation($" firstOp: ({string.Join(", ", firstOp)})"); + TestLibrary.TestFramework.LogInformation($" secondOp: ({string.Join(", ", secondOp)})"); + TestLibrary.TestFramework.LogInformation($" result: ({string.Join(", ", result)})"); + TestLibrary.TestFramework.LogInformation($" faultResult: ({faultResult})"); + TestLibrary.TestFramework.LogInformation(string.Empty); + + Succeeded = false; + } + } } } \ No newline at end of file diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveLoadVectorFirstFaultingTest.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveLoadVectorFirstFaultingTest.template index 4e7cb213d63d1..c0e489d138950 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveLoadVectorFirstFaultingTest.template +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveLoadVectorFirstFaultingTest.template @@ -31,6 +31,8 @@ namespace JIT.HardwareIntrinsics.Arm { // Validates basic functionality works test.RunBasicScenario_Load(); + + // Validates basic functionality of first-faulting behavior test.RunBasicScenario_LoadFirstFaulting(); // Validates calling via reflection works @@ -205,7 +207,7 @@ namespace JIT.HardwareIntrinsics.Arm var faultResult = Sve.GetFfr{GetFfrType}(); Unsafe.Write(_dataTable.outArrayPtr, result); - ValidateFirstFaultingResult(ref op2Ref, _dataTable.inBounded.Span.Length, _dataTable.outArrayPtr, faultResult.As<{GetFfrType}, {RetBaseType}>()); + ValidateFirstFaultingResult(ref op2Ref, _dataTable.inBounded.Span.Length, _dataTable.outArrayPtr, faultResult); } public void RunReflectionScenario_Load() @@ -352,7 +354,7 @@ namespace JIT.HardwareIntrinsics.Arm } } - private void ValidateFirstFaultingResult(ref byte firstOp, int size, void* result, Vector<{RetBaseType}> faultResult, [CallerMemberName] string method = "") + private void ValidateFirstFaultingResult(ref byte firstOp, int size, void* result, Vector<{GetFfrType}> faultResult, [CallerMemberName] string method = "") { {Op2BaseType}[] inArray = new {Op2BaseType}[size / Unsafe.SizeOf<{Op2BaseType}>()]; {RetBaseType}[] outArray = new {RetBaseType}[RetElementCount]; @@ -363,7 +365,7 @@ namespace JIT.HardwareIntrinsics.Arm ValidateFirstFaultingResult(inArray, outArray, faultResult, method); } - private void ValidateFirstFaultingResult({Op2BaseType}[] firstOp, {RetBaseType}[] result, Vector<{RetBaseType}> faultResult, [CallerMemberName] string method = "") + private void ValidateFirstFaultingResult({Op2BaseType}[] firstOp, {RetBaseType}[] result, Vector<{GetFfrType}> faultResult, [CallerMemberName] string method = "") { var succeeded = Helpers.CheckLoadVectorFirstFaultingBehavior(firstOp, result, faultResult); From ad5ec2e23cbdf7de0d30d8d3dc47263dbb09432f Mon Sep 17 00:00:00 2001 From: TIHan Date: Fri, 19 Jul 2024 17:18:07 -0700 Subject: [PATCH 30/83] Fix build --- .../System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs b/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs index 819021d8fd793..235abf9affcf7 100644 --- a/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs +++ b/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs @@ -5531,7 +5531,7 @@ internal Arm64() { } public static unsafe void SetFfr(System.Numerics.Vector value) { throw null; } public static unsafe void SetFfr(System.Numerics.Vector value) { throw null; } public static unsafe void SetFfr(System.Numerics.Vector value) { throw null; } - public static unsafe void SetFfr(System.Numerics.Vector value) { throw null; + public static unsafe void SetFfr(System.Numerics.Vector value) { throw null; } public static System.Numerics.Vector ShiftLeftLogical(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } public static System.Numerics.Vector ShiftLeftLogical(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } From 463934655f1152e9c8b028e234266a8936e49fa2 Mon Sep 17 00:00:00 2001 From: Mikhail Ablakatov Date: Mon, 22 Jul 2024 13:38:15 +0000 Subject: [PATCH 31/83] fixup: revert to using the span's length for ValidateFirstFaultingResult --- .../Arm/Shared/SveGatherVectorFirstFaultingIndices.template | 2 +- .../Arm/Shared/SveLoadVectorFirstFaultingTest.template | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorFirstFaultingIndices.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorFirstFaultingIndices.template index bfd72bc390927..381da3c5a5133 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorFirstFaultingIndices.template +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorFirstFaultingIndices.template @@ -336,7 +336,7 @@ namespace JIT.HardwareIntrinsics.Arm ref var op3Ref = ref op3; Unsafe.Write(_dataTable.outArrayPtr, result); - ValidateFirstFaultingResult(Unsafe.AsPointer(ref op1Ref), ref op2Ref, Op2ElementCount * Unsafe.SizeOf<{Op2BaseType}>(), Unsafe.AsPointer(ref op3Ref), _dataTable.outArrayPtr, faultResult); + ValidateFirstFaultingResult(Unsafe.AsPointer(ref op1Ref), ref op2Ref, _dataTable.inBounded.Span.Length, Unsafe.AsPointer(ref op3Ref), _dataTable.outArrayPtr, faultResult); } public void RunBasicScenario_FalseMask() diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveLoadVectorFirstFaultingTest.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveLoadVectorFirstFaultingTest.template index d9269e0118ced..02eaff5bb4fad 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveLoadVectorFirstFaultingTest.template +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveLoadVectorFirstFaultingTest.template @@ -206,7 +206,7 @@ namespace JIT.HardwareIntrinsics.Arm var faultResult = Sve.GetFfr{GetFfrType}(); Unsafe.Write(_dataTable.outArrayPtr, result); - ValidateFirstFaultingResult(ref op2Ref, Vector<{RetBaseType}>.Count * Unsafe.SizeOf<{Op2BaseType}>(), _dataTable.outArrayPtr, faultResult); + ValidateFirstFaultingResult(ref op2Ref, _dataTable.inBounded.Span.Length, _dataTable.outArrayPtr, faultResult); } public void RunReflectionScenario_Load() From 0f88d8ec0bc2e7f0fabfe565f67fe3e622ce58b0 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 19 Jul 2024 07:18:14 -0700 Subject: [PATCH 32/83] Add tracking of FFR register somewhat workable code cleanup Remove FFR Add all the GetFfr* wip Work with MskCns() model Use physReg approach Remove commented prototypes working Remove bunch of unnecessary code Remove SpecialImport from GetFFR/SetFFR/LoadFirstFaulting some more code cleanup some fixup --- src/coreclr/jit/codegenarm64.cpp | 14 +++++ src/coreclr/jit/compiler.h | 4 ++ src/coreclr/jit/gentree.cpp | 10 +++ src/coreclr/jit/hwintrinsic.cpp | 29 +++------ src/coreclr/jit/hwintrinsiccodegenarm64.cpp | 55 +++++++++++++++++ src/coreclr/jit/hwintrinsiclistarm64sve.h | 10 +++ src/coreclr/jit/lclvars.cpp | 3 + src/coreclr/jit/lowerarmarch.cpp | 67 +++++++++++++++++++++ src/coreclr/jit/lsraarm64.cpp | 9 ++- src/coreclr/jit/registerarm64.h | 2 +- src/coreclr/jit/simd.cpp | 23 +++++++ 11 files changed, 205 insertions(+), 21 deletions(-) diff --git a/src/coreclr/jit/codegenarm64.cpp b/src/coreclr/jit/codegenarm64.cpp index 2f6ebaf56fc84..bb161856a3ca1 100644 --- a/src/coreclr/jit/codegenarm64.cpp +++ b/src/coreclr/jit/codegenarm64.cpp @@ -2954,6 +2954,20 @@ void CodeGen::genCodeForStoreLclVar(GenTreeLclVar* lclNode) } #endif // FEATURE_SIMD +#ifdef FEATURE_MASKED_HW_INTRINSICS + if ((compiler->lvaFfrRegister == lclNode->GetLclNum()) && (targetReg == REG_NA)) + { + assert(targetType == TYP_MASK); + // We are about to store the FFR on stack. So first read the FFR register and + // store it in op1Reg. After that, op1Reg will be stored on the stack. + assert(varDsc->lvImplicitlyReferenced == 1); + + regNumber op1Reg = data->GetRegNum(); + assert(op1Reg != REG_NA); + GetEmitter()->emitIns_R(INS_sve_rdffr, EA_SCALABLE, op1Reg); + } +#endif + genConsumeRegs(data); regNumber dataReg = REG_NA; diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 7013184028a87..60a478e16d5fa 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -4344,6 +4344,10 @@ class Compiler #endif // defined(FEATURE_SIMD) unsigned lvaGSSecurityCookie; // LclVar number +#ifdef TARGET_ARM64 + unsigned lvaFfrRegister; // LclVar number + unsigned getFFRegisterVarNum(); +#endif bool lvaTempsHaveLargerOffsetThanVars(); // Returns "true" iff local variable "lclNum" is in SSA form. diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 759dde9b584fb..c7c1cd549d9f8 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -7448,7 +7448,11 @@ GenTreeIntCon* Compiler::gtNewFalse() // return a new node representing the value in a physical register GenTree* Compiler::gtNewPhysRegNode(regNumber reg, var_types type) { +#ifdef TARGET_ARM64 + assert(genIsValidIntReg(reg) || (reg == REG_SPBASE) || (reg == REG_FFR)); +#else assert(genIsValidIntReg(reg) || (reg == REG_SPBASE)); +#endif GenTree* result = new (this, GT_PHYSREG) GenTreePhysReg(reg, type); return result; } @@ -11554,6 +11558,12 @@ void Compiler::gtGetLclVarNameInfo(unsigned lclNum, const char** ilKindOut, cons { ilName = "GsCookie"; } +#ifdef TARGET_ARM64 + else if (lclNum == lvaFfrRegister) + { + ilName = "FFReg"; + } +#endif else if (lclNum == lvaRetAddrVar) { ilName = "ReturnAddress"; diff --git a/src/coreclr/jit/hwintrinsic.cpp b/src/coreclr/jit/hwintrinsic.cpp index 63455226ed2fb..e977b649dc5a2 100644 --- a/src/coreclr/jit/hwintrinsic.cpp +++ b/src/coreclr/jit/hwintrinsic.cpp @@ -2297,10 +2297,15 @@ GenTree* Compiler::impHWIntrinsic(NamedIntrinsic intrinsic, switch (intrinsic) { - case NI_Sve_CreateBreakAfterMask: case NI_Sve_CreateBreakAfterPropagateMask: - case NI_Sve_CreateBreakBeforeMask: case NI_Sve_CreateBreakBeforePropagateMask: + { + // HWInstrinsic requires a mask for op3 + convertToMaskIfNeeded(retNode->AsHWIntrinsic()->Op(3)); + FALLTHROUGH; + } + case NI_Sve_CreateBreakAfterMask: + case NI_Sve_CreateBreakBeforeMask: case NI_Sve_CreateMaskForFirstActiveElement: case NI_Sve_CreateMaskForNextActiveElement: case NI_Sve_GetActiveElementCount: @@ -2310,30 +2315,16 @@ GenTree* Compiler::impHWIntrinsic(NamedIntrinsic intrinsic, { // HWInstrinsic requires a mask for op2 convertToMaskIfNeeded(retNode->AsHWIntrinsic()->Op(2)); - break; + FALLTHROUGH; } - default: - break; - } - - switch (intrinsic) - { - case NI_Sve_CreateBreakAfterPropagateMask: - case NI_Sve_CreateBreakBeforePropagateMask: { - // HWInstrinsic requires a mask for op3 - convertToMaskIfNeeded(retNode->AsHWIntrinsic()->Op(3)); + // HWInstrinsic requires a mask for op1 + convertToMaskIfNeeded(retNode->AsHWIntrinsic()->Op(1)); break; } - - default: - break; } - // HWInstrinsic requires a mask for op1 - convertToMaskIfNeeded(retNode->AsHWIntrinsic()->Op(1)); - if (HWIntrinsicInfo::IsMultiReg(intrinsic)) { assert(HWIntrinsicInfo::IsExplicitMaskedOperation(retNode->AsHWIntrinsic()->GetHWIntrinsicId())); diff --git a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp index a93fa678156ca..4141839958c80 100644 --- a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp @@ -303,6 +303,7 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) emitAttr emitSize; insOpts opt; + bool unspilledFfr = false; if (HWIntrinsicInfo::SIMDScalar(intrin.id)) { @@ -318,6 +319,29 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) { emitSize = EA_SCALABLE; opt = emitter::optGetSveInsOpt(emitTypeSize(intrin.baseType)); + + bool readsFfr = false; + switch (intrin.id) + { + case NI_Sve_GetFfrByte: + case NI_Sve_GetFfrInt16: + case NI_Sve_GetFfrInt32: + case NI_Sve_GetFfrInt64: + case NI_Sve_GetFfrSByte: + case NI_Sve_GetFfrUInt16: + case NI_Sve_GetFfrUInt32: + case NI_Sve_GetFfrUInt64: + case NI_Sve_LoadVectorFirstFaulting: + readsFfr = true; + break; + default: + break; + } + + if (readsFfr && (intrin.op1 != nullptr) && ((intrin.op1->gtFlags & GTF_SPILLED) != 0)) + { + unspilledFfr = true; + } } else if (intrin.category == HW_Category_Special) { @@ -2366,6 +2390,37 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) break; } + case NI_Sve_LoadVectorFirstFaulting: + { + insScalableOpts sopt = (opt == INS_OPTS_SCALABLE_B) ? INS_SCALABLE_OPTS_NONE : INS_SCALABLE_OPTS_LSL_N; + GetEmitter()->emitIns_R_R_R_R(ins, emitSize, targetReg, op1Reg, op2Reg, REG_ZR, opt, sopt); + break; + } + + case NI_Sve_GetFfrByte: + case NI_Sve_GetFfrInt16: + case NI_Sve_GetFfrInt32: + case NI_Sve_GetFfrInt64: + case NI_Sve_GetFfrSByte: + case NI_Sve_GetFfrUInt16: + case NI_Sve_GetFfrUInt32: + case NI_Sve_GetFfrUInt64: + { + if (unspilledFfr) + { + // We have unspilled the FFR in op1Reg. Restore it back in FFR register. + GetEmitter()->emitIns_R(INS_sve_wrffr, emitSize, op1Reg, opt); + } + + GetEmitter()->emitIns_R(ins, emitSize, targetReg, INS_OPTS_SCALABLE_B); + break; + } + case NI_Sve_SetFfr: + { + assert(targetReg == REG_NA); + GetEmitter()->emitIns_R(ins, emitSize, op1Reg, opt); + break; + } case NI_Sve_ConditionalExtractAfterLastActiveElementScalar: case NI_Sve_ConditionalExtractLastActiveElementScalar: { diff --git a/src/coreclr/jit/hwintrinsiclistarm64sve.h b/src/coreclr/jit/hwintrinsiclistarm64sve.h index 46dd188785d19..9d9d8521da688 100644 --- a/src/coreclr/jit/hwintrinsiclistarm64sve.h +++ b/src/coreclr/jit/hwintrinsiclistarm64sve.h @@ -122,6 +122,14 @@ HARDWARE_INTRINSIC(Sve, GatherVectorUInt32WithByteOffsetsZeroExtend, HARDWARE_INTRINSIC(Sve, GatherVectorUInt32ZeroExtend, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_sve_ld1w, INS_sve_ld1w, INS_sve_ld1w, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, GatherVectorWithByteOffsets, -1, 3, true, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_sve_ld1w, INS_sve_ld1d, INS_sve_ld1d, INS_sve_ld1w, INS_sve_ld1d}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, GetActiveElementCount, -1, 2, true, {INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp, INS_sve_cntp}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ExplicitMaskedOperation) +HARDWARE_INTRINSIC(Sve, GetFfrByte, -1, -1, false, {INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, GetFfrInt16, -1, -1, false, {INS_invalid, INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, GetFfrInt32, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, GetFfrInt64, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, GetFfrSByte, -1, -1, false, {INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, GetFfrUInt16, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, GetFfrUInt32, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffectMask) +HARDWARE_INTRINSIC(Sve, GetFfrUInt64, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_rdffr, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffectMask) HARDWARE_INTRINSIC(Sve, InsertIntoShiftedVector, -1, 2, true, {INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr, INS_sve_insr}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics) HARDWARE_INTRINSIC(Sve, LeadingSignCount, -1, -1, false, {INS_sve_cls, INS_invalid, INS_sve_cls, INS_invalid, INS_sve_cls, INS_invalid, INS_sve_cls, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LeadingZeroCount, -1, -1, false, {INS_sve_clz, INS_sve_clz, INS_sve_clz, INS_sve_clz, INS_sve_clz, INS_sve_clz, INS_sve_clz, INS_sve_clz, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) @@ -142,6 +150,7 @@ HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendToInt64, HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendToUInt16, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendToUInt32, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendToUInt64, -1, 2, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1b, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, LoadVectorFirstFaulting, -1, -1, false, {INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1h, INS_sve_ldff1h, INS_sve_ldff1w, INS_sve_ldff1w, INS_sve_ldff1d, INS_sve_ldff1d, INS_sve_ldff1w, INS_sve_ldff1d}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialSideEffectMask) HARDWARE_INTRINSIC(Sve, LoadVectorInt16NonFaultingSignExtendToInt32, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorInt16NonFaultingSignExtendToInt64, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sh, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorInt16NonFaultingSignExtendToUInt32, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sh, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) @@ -237,6 +246,7 @@ HARDWARE_INTRINSIC(Sve, Scatter32BitNarrowing, HARDWARE_INTRINSIC(Sve, Scatter32BitWithByteOffsetsNarrowing, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_st1w, INS_sve_st1w, INS_invalid, INS_invalid}, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, Scatter8BitNarrowing, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_st1b, INS_sve_st1b, INS_sve_st1b, INS_sve_st1b, INS_invalid, INS_invalid}, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, Scatter8BitWithByteOffsetsNarrowing, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_st1b, INS_sve_st1b, INS_sve_st1b, INS_sve_st1b, INS_invalid, INS_invalid}, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, SetFfr, -1, 1, true, {INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_sve_wrffr, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialSideEffectMask|HW_Flag_SpecialCodeGen) HARDWARE_INTRINSIC(Sve, ShiftLeftLogical, -1, -1, false, {INS_sve_lsl, INS_sve_lsl, INS_sve_lsl, INS_sve_lsl, INS_sve_lsl, INS_sve_lsl, INS_sve_lsl, INS_sve_lsl, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) HARDWARE_INTRINSIC(Sve, ShiftRightArithmetic, -1, -1, false, {INS_sve_asr, INS_invalid, INS_sve_asr, INS_invalid, INS_sve_asr, INS_invalid, INS_sve_asr, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) HARDWARE_INTRINSIC(Sve, ShiftRightArithmeticForDivide, -1, -1, false, {INS_sve_asrd, INS_invalid, INS_sve_asrd, INS_invalid, INS_sve_asrd, INS_invalid, INS_sve_asrd, INS_invalid, INS_invalid, INS_invalid}, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand) diff --git a/src/coreclr/jit/lclvars.cpp b/src/coreclr/jit/lclvars.cpp index 62d3769879b20..bbef4c47bbbd5 100644 --- a/src/coreclr/jit/lclvars.cpp +++ b/src/coreclr/jit/lclvars.cpp @@ -63,6 +63,9 @@ void Compiler::lvaInit() #endif // JIT32_GCENCODER lvaNewObjArrayArgs = BAD_VAR_NUM; lvaGSSecurityCookie = BAD_VAR_NUM; +#ifdef TARGET_ARM64 + lvaFfrRegister = BAD_VAR_NUM; +#endif #ifdef TARGET_X86 lvaVarargsBaseOfStkArgs = BAD_VAR_NUM; #endif // TARGET_X86 diff --git a/src/coreclr/jit/lowerarmarch.cpp b/src/coreclr/jit/lowerarmarch.cpp index cd5032921804d..1092fdd9ee316 100644 --- a/src/coreclr/jit/lowerarmarch.cpp +++ b/src/coreclr/jit/lowerarmarch.cpp @@ -1518,6 +1518,73 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node) break; case NI_Sve_ConditionalSelect: return LowerHWIntrinsicCndSel(node); + case NI_Sve_SetFfr: + { + // Create physReg FFR definition to store FFR register. + unsigned lclNum = comp->getFFRegisterVarNum(); + GenTree* ffrReg = comp->gtNewPhysRegNode(REG_FFR, TYP_MASK); + GenTree* storeLclVar = comp->gtNewStoreLclVarNode(lclNum, ffrReg); + BlockRange().InsertAfter(node, ffrReg, storeLclVar); + + break; + } + case NI_Sve_GetFfrByte: + case NI_Sve_GetFfrInt16: + case NI_Sve_GetFfrInt32: + case NI_Sve_GetFfrInt64: + case NI_Sve_GetFfrSByte: + case NI_Sve_GetFfrUInt16: + case NI_Sve_GetFfrUInt32: + case NI_Sve_GetFfrUInt64: + { + unsigned lclNum = comp->lvaFfrRegister; + if (lclNum != BAD_VAR_NUM) + { + // Consume the FFR register value from local variable to simulate "use" of FFR, + // only if we saw its definition earlier. + GenTree* lclVar = comp->gtNewLclvNode(lclNum, TYP_MASK); + BlockRange().InsertBefore(node, lclVar); + LowerNode(lclVar); + + node->ResetHWIntrinsicId(intrinsicId, lclVar); + } + break; + } + case NI_Sve_LoadVectorFirstFaulting: + { + LIR::Use use; + bool foundUse = BlockRange().TryGetUse(node, &use); + + unsigned lclNum = comp->lvaFfrRegister; + if (lclNum != BAD_VAR_NUM) + { + // Consume the FFR register value from local variable to simulate "use" of FFR, + // only if we saw its definition earlier. + GenTree* lclVar = comp->gtNewLclvNode(lclNum, TYP_MASK); + BlockRange().InsertBefore(node, lclVar); + LowerNode(lclVar); + + node->ResetHWIntrinsicId(intrinsicId, comp, node->Op(1), node->Op(2), lclVar); + } + + if (foundUse) + { + unsigned tmpNum = comp->lvaGrabTemp(true DEBUGARG("Return value result/FFR")); + LclVarDsc* tmpVarDsc = comp->lvaGetDesc(tmpNum); + tmpVarDsc->lvType = node->TypeGet(); + GenTree* storeLclVar; + use.ReplaceWithLclVar(comp, tmpNum, &storeLclVar); + } + + // Create physReg FFR definition to store FFR register. + lclNum = comp->getFFRegisterVarNum(); + GenTree* ffrReg = comp->gtNewPhysRegNode(REG_FFR, TYP_MASK); + GenTree* storeFfrLclVar = comp->gtNewStoreLclVarNode(lclNum, ffrReg); + + BlockRange().InsertAfter(node, ffrReg, storeFfrLclVar); + + break; + } default: break; } diff --git a/src/coreclr/jit/lsraarm64.cpp b/src/coreclr/jit/lsraarm64.cpp index 5d6cf7f1945e4..ad28f82973540 100644 --- a/src/coreclr/jit/lsraarm64.cpp +++ b/src/coreclr/jit/lsraarm64.cpp @@ -607,9 +607,16 @@ int LinearScan::BuildNode(GenTree* tree) switch (tree->OperGet()) { default: + { srcCount = BuildSimple(tree); break; - + } + case GT_PHYSREG: + { + srcCount = 0; + BuildDef(tree, getSingleTypeRegMask(tree->AsPhysReg()->gtSrcReg, TYP_MASK)); + break; + } case GT_LCL_VAR: // We make a final determination about whether a GT_LCL_VAR is a candidate or contained // after liveness. In either case we don't build any uses or defs. Otherwise, this is a diff --git a/src/coreclr/jit/registerarm64.h b/src/coreclr/jit/registerarm64.h index d296ab9497858..d099493535f43 100644 --- a/src/coreclr/jit/registerarm64.h +++ b/src/coreclr/jit/registerarm64.h @@ -115,7 +115,7 @@ REGDEF(P12, 12+PBASE, PMASK(12), "p12", "na") REGDEF(P13, 13+PBASE, PMASK(13), "p13", "na") REGDEF(P14, 14+PBASE, PMASK(14), "p14", "na") REGDEF(P15, 15+PBASE, PMASK(15), "p15", "na") - +REGALIAS(FFR, P14) // The registers with values 80 (NBASE) and above are not real register numbers #define NBASE 80 diff --git a/src/coreclr/jit/simd.cpp b/src/coreclr/jit/simd.cpp index 1157bf9e5bfc9..94563113562e2 100644 --- a/src/coreclr/jit/simd.cpp +++ b/src/coreclr/jit/simd.cpp @@ -130,6 +130,29 @@ unsigned Compiler::getSIMDInitTempVarNum(var_types simdType) return lvaSIMDInitTempVarNum; } +#ifdef TARGET_ARM64 +//------------------------------------------------------------------------ +// Get, and allocate if necessary, the SIMD temp used for various operations. +// The temp is allocated as the maximum sized type of all operations required. +// +// Arguments: +// simdType - Required SIMD type +// +// Returns: +// The temp number +// +unsigned Compiler::getFFRegisterVarNum() +{ + if (lvaFfrRegister == BAD_VAR_NUM) + { + lvaFfrRegister = lvaGrabTempWithImplicitUse(false DEBUGARG("Save the FFR value.")); + lvaTable[lvaFfrRegister].lvType = TYP_MASK; + lvaTable[lvaFfrRegister].lvUsedInSIMDIntrinsic = true; + } + return lvaFfrRegister; +} +#endif + //---------------------------------------------------------------------------------- // Return the base type and size of SIMD vector type given its type handle. // From 10cf342a2232f6f0d642efa6540c618dbccecbf3 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 23 Jul 2024 12:20:18 -0700 Subject: [PATCH 33/83] Change condition for PhysReg --- src/coreclr/jit/lsraarm64.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/coreclr/jit/lsraarm64.cpp b/src/coreclr/jit/lsraarm64.cpp index ad28f82973540..2352fa5ec0239 100644 --- a/src/coreclr/jit/lsraarm64.cpp +++ b/src/coreclr/jit/lsraarm64.cpp @@ -614,7 +614,15 @@ int LinearScan::BuildNode(GenTree* tree) case GT_PHYSREG: { srcCount = 0; - BuildDef(tree, getSingleTypeRegMask(tree->AsPhysReg()->gtSrcReg, TYP_MASK)); + if (varTypeIsMask(tree)) + { + assert(tree->AsPhysReg()->gtSrcReg == REG_FFR); + BuildDef(tree, getSingleTypeRegMask(tree->AsPhysReg()->gtSrcReg, TYP_MASK)); + } + else + { + BuildSimple(tree); + } break; } case GT_LCL_VAR: From e7507bb1372e35f0f30d73c251e41dd9da5d0b77 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 23 Jul 2024 12:20:44 -0700 Subject: [PATCH 34/83] jit format --- src/coreclr/jit/codegenarm64.cpp | 2 +- src/coreclr/jit/lclvars.cpp | 2 +- src/coreclr/jit/lowerarmarch.cpp | 6 +++--- src/coreclr/jit/simd.cpp | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/coreclr/jit/codegenarm64.cpp b/src/coreclr/jit/codegenarm64.cpp index bb161856a3ca1..eebb60272a587 100644 --- a/src/coreclr/jit/codegenarm64.cpp +++ b/src/coreclr/jit/codegenarm64.cpp @@ -2961,7 +2961,7 @@ void CodeGen::genCodeForStoreLclVar(GenTreeLclVar* lclNode) // We are about to store the FFR on stack. So first read the FFR register and // store it in op1Reg. After that, op1Reg will be stored on the stack. assert(varDsc->lvImplicitlyReferenced == 1); - + regNumber op1Reg = data->GetRegNum(); assert(op1Reg != REG_NA); GetEmitter()->emitIns_R(INS_sve_rdffr, EA_SCALABLE, op1Reg); diff --git a/src/coreclr/jit/lclvars.cpp b/src/coreclr/jit/lclvars.cpp index bbef4c47bbbd5..4dc8fec909def 100644 --- a/src/coreclr/jit/lclvars.cpp +++ b/src/coreclr/jit/lclvars.cpp @@ -64,7 +64,7 @@ void Compiler::lvaInit() lvaNewObjArrayArgs = BAD_VAR_NUM; lvaGSSecurityCookie = BAD_VAR_NUM; #ifdef TARGET_ARM64 - lvaFfrRegister = BAD_VAR_NUM; + lvaFfrRegister = BAD_VAR_NUM; #endif #ifdef TARGET_X86 lvaVarargsBaseOfStkArgs = BAD_VAR_NUM; diff --git a/src/coreclr/jit/lowerarmarch.cpp b/src/coreclr/jit/lowerarmarch.cpp index 1092fdd9ee316..a7751df89fbfb 100644 --- a/src/coreclr/jit/lowerarmarch.cpp +++ b/src/coreclr/jit/lowerarmarch.cpp @@ -1521,8 +1521,8 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node) case NI_Sve_SetFfr: { // Create physReg FFR definition to store FFR register. - unsigned lclNum = comp->getFFRegisterVarNum(); - GenTree* ffrReg = comp->gtNewPhysRegNode(REG_FFR, TYP_MASK); + unsigned lclNum = comp->getFFRegisterVarNum(); + GenTree* ffrReg = comp->gtNewPhysRegNode(REG_FFR, TYP_MASK); GenTree* storeLclVar = comp->gtNewStoreLclVarNode(lclNum, ffrReg); BlockRange().InsertAfter(node, ffrReg, storeLclVar); @@ -1565,7 +1565,7 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node) LowerNode(lclVar); node->ResetHWIntrinsicId(intrinsicId, comp, node->Op(1), node->Op(2), lclVar); - } + } if (foundUse) { diff --git a/src/coreclr/jit/simd.cpp b/src/coreclr/jit/simd.cpp index 94563113562e2..74c16fafff987 100644 --- a/src/coreclr/jit/simd.cpp +++ b/src/coreclr/jit/simd.cpp @@ -145,10 +145,10 @@ unsigned Compiler::getFFRegisterVarNum() { if (lvaFfrRegister == BAD_VAR_NUM) { - lvaFfrRegister = lvaGrabTempWithImplicitUse(false DEBUGARG("Save the FFR value.")); + lvaFfrRegister = lvaGrabTempWithImplicitUse(false DEBUGARG("Save the FFR value.")); lvaTable[lvaFfrRegister].lvType = TYP_MASK; lvaTable[lvaFfrRegister].lvUsedInSIMDIntrinsic = true; - } + } return lvaFfrRegister; } #endif From aef79cd4e44a0afb385be8bd8780048e21410c2b Mon Sep 17 00:00:00 2001 From: Swapnil Gaikwad Date: Tue, 23 Jul 2024 13:30:31 +0100 Subject: [PATCH 35/83] Fix PoisonPage configuration while creating BoundedMemory --- .../System/Buffers/BoundedMemory.Unix.cs | 76 +++++++++++++++---- 1 file changed, 62 insertions(+), 14 deletions(-) diff --git a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs index c8197b0055092..a4d1e3a89ecc2 100644 --- a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs +++ b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs @@ -9,10 +9,7 @@ public static unsafe partial class BoundedMemory { private static UnixImplementation AllocateWithoutDataPopulationUnix(int elementCount, PoisonPagePlacement placement) where T : unmanaged { - // On non-Windows platforms, we don't yet have support for changing the permissions of individual pages. - // We'll instead use AllocHGlobal / FreeHGlobal to carve out a r+w section of unmanaged memory. - - return new UnixImplementation(elementCount); + return new UnixImplementation(elementCount, placement); } private sealed class UnixImplementation : BoundedMemory where T : unmanaged @@ -21,9 +18,9 @@ private sealed class UnixImplementation : BoundedMemory where T : unmanage private readonly int _elementCount; private readonly BoundedMemoryManager _memoryManager; - public UnixImplementation(int elementCount) + public UnixImplementation(int elementCount, PoisonPagePlacement placement) { - _handle = AllocHGlobalHandle.Allocate(checked(elementCount * (nint)sizeof(T))); + _handle = AllocHGlobalHandle.Allocate(checked(elementCount * (nint)sizeof(T)), placement); _elementCount = elementCount; _memoryManager = new BoundedMemoryManager(this); } @@ -118,29 +115,80 @@ public override void Unpin() private sealed class AllocHGlobalHandle : SafeHandle { + private IntPtr buffer; + private Int32 allocationSize; + // Called by P/Invoke when returning SafeHandles - private AllocHGlobalHandle() + private AllocHGlobalHandle(IntPtr buffer, Int32 allocationSize) : base(IntPtr.Zero, ownsHandle: true) { + this.buffer = buffer; + this.allocationSize = allocationSize; } - internal static AllocHGlobalHandle Allocate(nint byteLength) + internal static AllocHGlobalHandle Allocate(nint byteLength, PoisonPagePlacement placement) { - AllocHGlobalHandle retVal = new AllocHGlobalHandle(); - retVal.SetHandle(Marshal.AllocHGlobal(byteLength)); // this is for unit testing; don't bother setting up a CER on Full Framework + + // Allocate number of pages to incorporate required (byteLength bytes of) memory and an additional page to create a poison page. + Int32 pageSize = Environment.SystemPageSize; + Int32 allocationSize = (Int32)((byteLength % pageSize) + 2) * pageSize; + IntPtr buffer = memalign(pageSize, allocationSize); + + if (buffer == IntPtr.Zero) + { + throw new InvalidOperationException("Memory allocation failed."); + } + + // Depending on the PoisonPagePlacement requirement (before/after) initialise the baseAddress and poisonPageAddress to point to the location + // in the buffer. Here the baseAddress points to the first valid allocation and poisonPageAddress points to the first invalid location. + // For PoisonPagePlacement.Before the first page is made inaccessible using mprotect and baseAddress points to the start of the second page. + // The allocation and protection is at the granularity of a page. Thus, `PoisonPagePlacement.Before` configuration has an additional accessible + // memory at the end of the page (bytes equivalent to `pageSize - (byteLength % pageSize)`). + // For `PoisonPagePlacement.After`, we adjust the baseAddress so that inaccessible memory is at the `byteLength` offset from the baseAddress. + IntPtr baseAddress = buffer + pageSize; + IntPtr poisonPageAddress = buffer; + if (placement == PoisonPagePlacement.After) + { + baseAddress = buffer + (allocationSize - pageSize - byteLength); + poisonPageAddress = buffer + (allocationSize - pageSize); + } + + // Protect the page before/after based on the poison page placement. + if (mprotect(poisonPageAddress, (ulong)pageSize, PROT_NONE) == -1) + { + throw new InvalidOperationException("Failed to mark page as a poison page using mprotect."); + } + + AllocHGlobalHandle retVal = new AllocHGlobalHandle(buffer, allocationSize); + retVal.SetHandle(baseAddress); // this base address would be used as the start of Span that is used during unit testing. return retVal; } - // Do not provide a finalizer - SafeHandle's critical finalizer will - // call ReleaseHandle for you. - public override bool IsInvalid => (handle == IntPtr.Zero); protected override bool ReleaseHandle() { - Marshal.FreeHGlobal(handle); + // Reset the protection on the allocated memory. + if (mprotect(buffer, (ulong)allocationSize, PROT_READ | PROT_WRITE) == -1) + { + throw new InvalidOperationException("Failed to reset memory protection using mprotect."); + } + free(buffer); return true; } + + const int PROT_NONE = 0x0; + const int PROT_READ = 0x1; + const int PROT_WRITE = 0x2; + + [DllImport("libc", SetLastError = true)] + static extern IntPtr memalign(int alignment, int size); + + [DllImport("libc", SetLastError = true)] + static extern int mprotect(IntPtr addr, ulong len, int prot); + + [DllImport("libc", SetLastError = true)] + static extern void free(IntPtr ptr); } } } From 690e7adf504476d331d9b8c244d0db5f51d89668 Mon Sep 17 00:00:00 2001 From: Swapnil Gaikwad Date: Tue, 23 Jul 2024 16:32:25 +0100 Subject: [PATCH 36/83] Use mmap() instead of memalign() for memory allocation --- .../System/Buffers/BoundedMemory.Unix.cs | 30 ++++++++----------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs index a4d1e3a89ecc2..565e6ff993462 100644 --- a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs +++ b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs @@ -116,10 +116,10 @@ public override void Unpin() private sealed class AllocHGlobalHandle : SafeHandle { private IntPtr buffer; - private Int32 allocationSize; + private ulong allocationSize; // Called by P/Invoke when returning SafeHandles - private AllocHGlobalHandle(IntPtr buffer, Int32 allocationSize) + private AllocHGlobalHandle(IntPtr buffer, ulong allocationSize) : base(IntPtr.Zero, ownsHandle: true) { this.buffer = buffer; @@ -130,9 +130,9 @@ internal static AllocHGlobalHandle Allocate(nint byteLength, PoisonPagePlacement { // Allocate number of pages to incorporate required (byteLength bytes of) memory and an additional page to create a poison page. - Int32 pageSize = Environment.SystemPageSize; - Int32 allocationSize = (Int32)((byteLength % pageSize) + 2) * pageSize; - IntPtr buffer = memalign(pageSize, allocationSize); + int pageSize = Environment.SystemPageSize; + int allocationSize = (int)(((byteLength / pageSize) + ((byteLength % pageSize) == 0 ? 0 : 1) + 1) * pageSize); + IntPtr buffer = mmap(IntPtr.Zero, (ulong)allocationSize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); if (buffer == IntPtr.Zero) { @@ -154,12 +154,12 @@ internal static AllocHGlobalHandle Allocate(nint byteLength, PoisonPagePlacement } // Protect the page before/after based on the poison page placement. - if (mprotect(poisonPageAddress, (ulong)pageSize, PROT_NONE) == -1) + if (mprotect(poisonPageAddress, (ulong) pageSize, PROT_NONE) == -1) { throw new InvalidOperationException("Failed to mark page as a poison page using mprotect."); } - AllocHGlobalHandle retVal = new AllocHGlobalHandle(buffer, allocationSize); + AllocHGlobalHandle retVal = new AllocHGlobalHandle(buffer, (ulong)allocationSize); retVal.SetHandle(baseAddress); // this base address would be used as the start of Span that is used during unit testing. return retVal; } @@ -168,27 +168,23 @@ internal static AllocHGlobalHandle Allocate(nint byteLength, PoisonPagePlacement protected override bool ReleaseHandle() { - // Reset the protection on the allocated memory. - if (mprotect(buffer, (ulong)allocationSize, PROT_READ | PROT_WRITE) == -1) - { - throw new InvalidOperationException("Failed to reset memory protection using mprotect."); - } - free(buffer); + munmap(buffer, allocationSize); return true; } - + const int MAP_PRIVATE = 0x2; + const int MAP_ANONYMOUS = 0x20; const int PROT_NONE = 0x0; const int PROT_READ = 0x1; const int PROT_WRITE = 0x2; [DllImport("libc", SetLastError = true)] - static extern IntPtr memalign(int alignment, int size); + static extern IntPtr mmap(IntPtr addrress, ulong length, int prot, int flags, int fd, int offset); [DllImport("libc", SetLastError = true)] - static extern int mprotect(IntPtr addr, ulong len, int prot); + static extern IntPtr munmap(IntPtr addrress, ulong length); [DllImport("libc", SetLastError = true)] - static extern void free(IntPtr ptr); + static extern int mprotect(IntPtr addr, ulong len, int prot); } } } From b23fac73f7413508a5410e4687f932abfb33834f Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 23 Jul 2024 14:26:02 -0700 Subject: [PATCH 37/83] review feedback --- src/coreclr/jit/codegenarm64.cpp | 14 -------------- src/coreclr/jit/codegenarmarch.cpp | 14 ++++++++++++-- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/src/coreclr/jit/codegenarm64.cpp b/src/coreclr/jit/codegenarm64.cpp index eebb60272a587..2f6ebaf56fc84 100644 --- a/src/coreclr/jit/codegenarm64.cpp +++ b/src/coreclr/jit/codegenarm64.cpp @@ -2954,20 +2954,6 @@ void CodeGen::genCodeForStoreLclVar(GenTreeLclVar* lclNode) } #endif // FEATURE_SIMD -#ifdef FEATURE_MASKED_HW_INTRINSICS - if ((compiler->lvaFfrRegister == lclNode->GetLclNum()) && (targetReg == REG_NA)) - { - assert(targetType == TYP_MASK); - // We are about to store the FFR on stack. So first read the FFR register and - // store it in op1Reg. After that, op1Reg will be stored on the stack. - assert(varDsc->lvImplicitlyReferenced == 1); - - regNumber op1Reg = data->GetRegNum(); - assert(op1Reg != REG_NA); - GetEmitter()->emitIns_R(INS_sve_rdffr, EA_SCALABLE, op1Reg); - } -#endif - genConsumeRegs(data); regNumber dataReg = REG_NA; diff --git a/src/coreclr/jit/codegenarmarch.cpp b/src/coreclr/jit/codegenarmarch.cpp index 16cad5618112b..0f9104f171661 100644 --- a/src/coreclr/jit/codegenarmarch.cpp +++ b/src/coreclr/jit/codegenarmarch.cpp @@ -1508,8 +1508,18 @@ void CodeGen::genCodeForPhysReg(GenTreePhysReg* tree) var_types targetType = tree->TypeGet(); regNumber targetReg = tree->GetRegNum(); - inst_Mov(targetType, targetReg, tree->gtSrcReg, /* canSkip */ true); - genTransferRegGCState(targetReg, tree->gtSrcReg); +#ifdef TARGET_ARM64 + if (varTypeIsMask(targetType)) + { + assert(tree->gtSrcReg == REG_FFR); + GetEmitter()->emitIns_R(INS_sve_rdffr, EA_SCALABLE, REG_FFR); + } + else +#endif + { + inst_Mov(targetType, targetReg, tree->gtSrcReg, /* canSkip */ true); + genTransferRegGCState(targetReg, tree->gtSrcReg); + } genProduceReg(tree); } From 0c8b68869d8e878536211352deebb60f4e965fff Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 23 Jul 2024 14:41:39 -0700 Subject: [PATCH 38/83] unspill for LoadVectorFirstFaulting as well --- src/coreclr/jit/hwintrinsiccodegenarm64.cpp | 30 ++++++++++++++++----- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp index 4141839958c80..7cb147c4ecfde 100644 --- a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp @@ -320,7 +320,6 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) emitSize = EA_SCALABLE; opt = emitter::optGetSveInsOpt(emitTypeSize(intrin.baseType)); - bool readsFfr = false; switch (intrin.id) { case NI_Sve_GetFfrByte: @@ -331,17 +330,28 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) case NI_Sve_GetFfrUInt16: case NI_Sve_GetFfrUInt32: case NI_Sve_GetFfrUInt64: + { + if ((intrin.op1 != nullptr) && ((intrin.op1->gtFlags & GTF_SPILLED) != 0)) + { + // If there was a op1 for this intrinsic, it means FFR is consumed here + // and we need to unspill. + unspilledFfr = true; + } + break; + } case NI_Sve_LoadVectorFirstFaulting: - readsFfr = true; + { + if ((intrin.op3 != nullptr) && ((intrin.op3->gtFlags & GTF_SPILLED) != 0)) + { + // If there was a op3 for this intrinsic, it means FFR is consumed here + // and we need to unspill. + unspilledFfr = true; + } break; + } default: break; } - - if (readsFfr && (intrin.op1 != nullptr) && ((intrin.op1->gtFlags & GTF_SPILLED) != 0)) - { - unspilledFfr = true; - } } else if (intrin.category == HW_Category_Special) { @@ -2392,6 +2402,12 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) case NI_Sve_LoadVectorFirstFaulting: { + if (unspilledFfr) + { + // We have unspilled the FFR in op1Reg. Restore it back in FFR register. + GetEmitter()->emitIns_R(INS_sve_wrffr, emitSize, op1Reg, opt); + } + insScalableOpts sopt = (opt == INS_OPTS_SCALABLE_B) ? INS_SCALABLE_OPTS_NONE : INS_SCALABLE_OPTS_LSL_N; GetEmitter()->emitIns_R_R_R_R(ins, emitSize, targetReg, op1Reg, op2Reg, REG_ZR, opt, sopt); break; From 35fe79141dda54dbb2232a0cf3daf291f8f90e43 Mon Sep 17 00:00:00 2001 From: Mikhail Ablakatov Date: Wed, 24 Jul 2024 17:08:20 +0000 Subject: [PATCH 39/83] fixup: revert a check of faulted elements --- src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs index ce222dad40bdd..9cbea6f844266 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs @@ -8678,12 +8678,6 @@ private static bool CheckFirstFaultingBehaviorCore(T[] result, Vector { return false; } - - if (result[i] != T.Zero) - { - return false; - } - hitFault = true; } else From 5bb0b3d0f4bbd7368c9dd7d4276dc87b70228471 Mon Sep 17 00:00:00 2001 From: Swapnil Gaikwad Date: Wed, 24 Jul 2024 18:09:06 +0100 Subject: [PATCH 40/83] Show error codes on failing failure --- .../System/Buffers/BoundedMemory.Unix.cs | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs index 565e6ff993462..26626b50cbe9f 100644 --- a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs +++ b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs @@ -136,12 +136,12 @@ internal static AllocHGlobalHandle Allocate(nint byteLength, PoisonPagePlacement if (buffer == IntPtr.Zero) { - throw new InvalidOperationException("Memory allocation failed."); + throw new InvalidOperationException($"Memory allocation failed with error {Marshal.GetLastPInvokeError()}."); } // Depending on the PoisonPagePlacement requirement (before/after) initialise the baseAddress and poisonPageAddress to point to the location // in the buffer. Here the baseAddress points to the first valid allocation and poisonPageAddress points to the first invalid location. - // For PoisonPagePlacement.Before the first page is made inaccessible using mprotect and baseAddress points to the start of the second page. + // For `PoisonPagePlacement.Before` the first page is made inaccessible using mprotect and baseAddress points to the start of the second page. // The allocation and protection is at the granularity of a page. Thus, `PoisonPagePlacement.Before` configuration has an additional accessible // memory at the end of the page (bytes equivalent to `pageSize - (byteLength % pageSize)`). // For `PoisonPagePlacement.After`, we adjust the baseAddress so that inaccessible memory is at the `byteLength` offset from the baseAddress. @@ -156,7 +156,7 @@ internal static AllocHGlobalHandle Allocate(nint byteLength, PoisonPagePlacement // Protect the page before/after based on the poison page placement. if (mprotect(poisonPageAddress, (ulong) pageSize, PROT_NONE) == -1) { - throw new InvalidOperationException("Failed to mark page as a poison page using mprotect."); + throw new InvalidOperationException($"Failed to mark page as a poison page using mprotect with error :{Marshal.GetLastPInvokeError()}."); } AllocHGlobalHandle retVal = new AllocHGlobalHandle(buffer, (ulong)allocationSize); @@ -168,9 +168,10 @@ internal static AllocHGlobalHandle Allocate(nint byteLength, PoisonPagePlacement protected override bool ReleaseHandle() { - munmap(buffer, allocationSize); - return true; + return munmap(buffer, allocationSize) == 0; } + + // Defined in const int MAP_PRIVATE = 0x2; const int MAP_ANONYMOUS = 0x20; const int PROT_NONE = 0x0; @@ -178,13 +179,13 @@ protected override bool ReleaseHandle() const int PROT_WRITE = 0x2; [DllImport("libc", SetLastError = true)] - static extern IntPtr mmap(IntPtr addrress, ulong length, int prot, int flags, int fd, int offset); + static extern IntPtr mmap(IntPtr address, ulong length, int prot, int flags, int fd, int offset); [DllImport("libc", SetLastError = true)] - static extern IntPtr munmap(IntPtr addrress, ulong length); + static extern IntPtr munmap(IntPtr address, ulong length); [DllImport("libc", SetLastError = true)] - static extern int mprotect(IntPtr addr, ulong len, int prot); + static extern int mprotect(IntPtr address, ulong length, int prot); } } } From 86715e5874e385d9a47378718abbe114af453873 Mon Sep 17 00:00:00 2001 From: TIHan Date: Thu, 25 Jul 2024 21:40:27 -0700 Subject: [PATCH 41/83] Feedback --- src/coreclr/jit/hwintrinsiccodegenarm64.cpp | 33 --------------------- 1 file changed, 33 deletions(-) diff --git a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp index 341c6c9ce43ab..7ed9b0e734d25 100644 --- a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp @@ -319,39 +319,6 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) { emitSize = EA_SCALABLE; opt = emitter::optGetSveInsOpt(emitTypeSize(intrin.baseType)); - - switch (intrin.id) - { - case NI_Sve_GetFfrByte: - case NI_Sve_GetFfrInt16: - case NI_Sve_GetFfrInt32: - case NI_Sve_GetFfrInt64: - case NI_Sve_GetFfrSByte: - case NI_Sve_GetFfrUInt16: - case NI_Sve_GetFfrUInt32: - case NI_Sve_GetFfrUInt64: - { - if ((intrin.op1 != nullptr) && ((intrin.op1->gtFlags & GTF_SPILLED) != 0)) - { - // If there was a op1 for this intrinsic, it means FFR is consumed here - // and we need to unspill. - unspilledFfr = true; - } - break; - } - case NI_Sve_LoadVectorFirstFaulting: - { - if ((intrin.op3 != nullptr) && ((intrin.op3->gtFlags & GTF_SPILLED) != 0)) - { - // If there was a op3 for this intrinsic, it means FFR is consumed here - // and we need to unspill. - unspilledFfr = true; - } - break; - } - default: - break; - } } else if (intrin.category == HW_Category_Special) { From 8b0f0007b842f01b2cc7859a34b9e64aabc3463e Mon Sep 17 00:00:00 2001 From: TIHan Date: Thu, 25 Jul 2024 21:41:18 -0700 Subject: [PATCH 42/83] Feedback --- src/coreclr/jit/hwintrinsiccodegenarm64.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp index 7ed9b0e734d25..8e296563e4f59 100644 --- a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp @@ -303,7 +303,6 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) emitAttr emitSize; insOpts opt; - bool unspilledFfr = false; if (HWIntrinsicInfo::SIMDScalar(intrin.id)) { From 044dbda55c08643118350ec170bf09ed1eea35ba Mon Sep 17 00:00:00 2001 From: TIHan Date: Thu, 25 Jul 2024 21:47:05 -0700 Subject: [PATCH 43/83] Feedback --- src/coreclr/jit/fgdiagnostic.cpp | 9 --------- src/coreclr/jit/gentree.cpp | 18 ------------------ 2 files changed, 27 deletions(-) diff --git a/src/coreclr/jit/fgdiagnostic.cpp b/src/coreclr/jit/fgdiagnostic.cpp index 3f82e9c10ca25..9f1a72afce63e 100644 --- a/src/coreclr/jit/fgdiagnostic.cpp +++ b/src/coreclr/jit/fgdiagnostic.cpp @@ -3436,15 +3436,6 @@ void Compiler::fgDebugCheckFlags(GenTree* tree, BasicBlock* block) case NI_Sve_GatherPrefetch32Bit: case NI_Sve_GatherPrefetch64Bit: case NI_Sve_GatherPrefetch8Bit: - case NI_Sve_SetFfr: - case NI_Sve_GetFfrByte: - case NI_Sve_GetFfrInt16: - case NI_Sve_GetFfrInt32: - case NI_Sve_GetFfrInt64: - case NI_Sve_GetFfrSByte: - case NI_Sve_GetFfrUInt16: - case NI_Sve_GetFfrUInt32: - case NI_Sve_GetFfrUInt64: { assert(tree->OperRequiresCallFlag(this)); expectedFlags |= GTF_GLOB_REF; diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 49557c3b936be..6654fc052b574 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -27290,15 +27290,6 @@ bool GenTreeHWIntrinsic::OperRequiresCallFlag() const case NI_Sve_GatherPrefetch32Bit: case NI_Sve_GatherPrefetch64Bit: case NI_Sve_GatherPrefetch8Bit: - case NI_Sve_SetFfr: - case NI_Sve_GetFfrByte: - case NI_Sve_GetFfrInt16: - case NI_Sve_GetFfrInt32: - case NI_Sve_GetFfrInt64: - case NI_Sve_GetFfrSByte: - case NI_Sve_GetFfrUInt16: - case NI_Sve_GetFfrUInt32: - case NI_Sve_GetFfrUInt64: { return true; } @@ -27489,15 +27480,6 @@ void GenTreeHWIntrinsic::Initialize(NamedIntrinsic intrinsicId) case NI_Sve_GatherPrefetch32Bit: case NI_Sve_GatherPrefetch64Bit: case NI_Sve_GatherPrefetch8Bit: - case NI_Sve_SetFfr: - case NI_Sve_GetFfrByte: - case NI_Sve_GetFfrInt16: - case NI_Sve_GetFfrInt32: - case NI_Sve_GetFfrInt64: - case NI_Sve_GetFfrSByte: - case NI_Sve_GetFfrUInt16: - case NI_Sve_GetFfrUInt32: - case NI_Sve_GetFfrUInt64: { // Mark as a call and global reference, much as is done for GT_KEEPALIVE gtFlags |= (GTF_CALL | GTF_GLOB_REF); From 0655d4b4bb915b981b6d3f28e664e14c932069cf Mon Sep 17 00:00:00 2001 From: TIHan Date: Thu, 25 Jul 2024 21:48:37 -0700 Subject: [PATCH 44/83] Feedback --- src/coreclr/jit/gentree.cpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 6654fc052b574..d9b6dbcae97f5 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -11742,12 +11742,6 @@ void Compiler::gtGetLclVarNameInfo(unsigned lclNum, const char** ilKindOut, cons { ilName = "GsCookie"; } -#ifdef TARGET_ARM64 - else if (lclNum == lvaFfrRegister) - { - ilName = "FFReg"; - } -#endif else if (lclNum == lvaRetAddrVar) { ilName = "ReturnAddress"; From 9d7f22fbd6f1f007f656a8f063bfe0ce6cd51753 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 26 Jul 2024 09:14:26 -0700 Subject: [PATCH 45/83] Handle FFR correctly --- src/coreclr/jit/gentree.cpp | 27 ++++----- src/coreclr/jit/hwintrinsiccodegenarm64.cpp | 64 ++++++++++++++++++++- src/coreclr/jit/lowerarmarch.cpp | 47 ++++++++++++++- 3 files changed, 121 insertions(+), 17 deletions(-) diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index d9b6dbcae97f5..64cb8f6b67bdc 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -26732,6 +26732,18 @@ bool GenTreeHWIntrinsic::OperIsMemoryLoad(GenTree** pAddr) const addr = Op(3); break; + case NI_Sve_GatherVector: + case NI_Sve_GatherVectorByteZeroExtend: + case NI_Sve_GatherVectorFirstFaulting: + case NI_Sve_GatherVectorInt16SignExtend: + case NI_Sve_GatherVectorInt16WithByteOffsetsSignExtend: + case NI_Sve_GatherVectorInt32SignExtend: + case NI_Sve_GatherVectorInt32WithByteOffsetsSignExtend: + case NI_Sve_GatherVectorSByteSignExtend: + case NI_Sve_GatherVectorUInt16WithByteOffsetsZeroExtend: + case NI_Sve_GatherVectorUInt16ZeroExtend: + case NI_Sve_GatherVectorUInt32WithByteOffsetsZeroExtend: + case NI_Sve_GatherVectorUInt32ZeroExtend: case NI_Sve_GatherVectorWithByteOffsets: case NI_Sve_LoadVector: case NI_Sve_LoadVectorNonTemporal: @@ -26767,21 +26779,6 @@ bool GenTreeHWIntrinsic::OperIsMemoryLoad(GenTree** pAddr) const addr = Op(2); break; - case NI_Sve_GatherVector: - case NI_Sve_GatherVectorByteZeroExtend: - case NI_Sve_GatherVectorFirstFaulting: - case NI_Sve_GatherVectorInt16SignExtend: - case NI_Sve_GatherVectorInt16WithByteOffsetsSignExtend: - case NI_Sve_GatherVectorInt32SignExtend: - case NI_Sve_GatherVectorInt32WithByteOffsetsSignExtend: - case NI_Sve_GatherVectorSByteSignExtend: - case NI_Sve_GatherVectorUInt16WithByteOffsetsZeroExtend: - case NI_Sve_GatherVectorUInt16ZeroExtend: - case NI_Sve_GatherVectorUInt32WithByteOffsetsZeroExtend: - case NI_Sve_GatherVectorUInt32ZeroExtend: - addr = Op(2); - break; - #endif // TARGET_ARM64 default: diff --git a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp index 8e296563e4f59..f6c84cb5809f4 100644 --- a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp @@ -2049,9 +2049,71 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) break; } + case NI_Sve_GatherVectorFirstFaulting: + { + if (node->GetAuxiliaryType() == TYP_UNKNOWN) + { + if (intrin.numOperands == 3) + { + // We have extra argument which means there is a "use" of FFR here. Restore it back in FFR + // register. + assert(op3Reg != REG_NA); + GetEmitter()->emitIns_R(INS_sve_wrffr, emitSize, op3Reg, opt); + } + } + else + { + // AuxilaryType is added only for numOperands == 3. If there is an extra argument, we need to + // "use" FFR here. Restore it back in FFR register. + + if (intrin.numOperands == 4) + { + // We have extra argument which means there is a "use" of FFR here. Restore it back in FFR + // register. + assert(op4Reg != REG_NA); + GetEmitter()->emitIns_R(INS_sve_wrffr, emitSize, op4Reg, opt); + } + } + + if (!varTypeIsSIMD(intrin.op2->gtType)) + { + // GatherVector...(Vector mask, T* address, Vector indices) + + assert(intrin.numOperands == 3); + emitAttr baseSize = emitActualTypeSize(intrin.baseType); + insScalableOpts sopt = INS_SCALABLE_OPTS_NONE; + + if (baseSize == EA_8BYTE) + { + // Index is multiplied. + sopt = (ins == INS_sve_ldff1b || ins == INS_sve_ldff1sb) ? INS_SCALABLE_OPTS_NONE + : INS_SCALABLE_OPTS_LSL_N; + } + else + { + // Index is sign or zero extended to 64bits, then multiplied. + assert(baseSize == EA_4BYTE); + opt = varTypeIsUnsigned(node->GetAuxiliaryType()) ? INS_OPTS_SCALABLE_S_UXTW + : INS_OPTS_SCALABLE_S_SXTW; + + sopt = (ins == INS_sve_ldff1b || ins == INS_sve_ldff1sb) ? INS_SCALABLE_OPTS_NONE + : INS_SCALABLE_OPTS_MOD_N; + } + + GetEmitter()->emitIns_R_R_R_R(ins, emitSize, targetReg, op1Reg, op2Reg, op3Reg, opt, sopt); + } + else + { + // GatherVector...(Vector mask, Vector addresses) + + assert(intrin.numOperands == 2); + GetEmitter()->emitIns_R_R_R_I(ins, emitSize, targetReg, op1Reg, op2Reg, 0, opt); + } + + break; + } case NI_Sve_GatherVector: case NI_Sve_GatherVectorByteZeroExtend: - case NI_Sve_GatherVectorFirstFaulting: case NI_Sve_GatherVectorInt16SignExtend: case NI_Sve_GatherVectorInt16WithByteOffsetsSignExtend: case NI_Sve_GatherVectorInt32SignExtend: diff --git a/src/coreclr/jit/lowerarmarch.cpp b/src/coreclr/jit/lowerarmarch.cpp index a227d8ac14148..ee026a7d23d7e 100644 --- a/src/coreclr/jit/lowerarmarch.cpp +++ b/src/coreclr/jit/lowerarmarch.cpp @@ -1775,6 +1775,49 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node) break; } + case NI_Sve_GatherVectorFirstFaulting: + { + LIR::Use use; + bool foundUse = BlockRange().TryGetUse(node, &use); + + if (m_ffrTrashed) + { + // Consume the FFR register value from local variable to simulate "use" of FFR, + // only if it was trashed. If it was not trashed, we do not have to reload the + // contents of the FFR register. + + GenTree* lclVar = comp->gtNewLclvNode(comp->lvaFfrRegister, TYP_MASK); + BlockRange().InsertBefore(node, lclVar); + LowerNode(lclVar); + + if (node->GetOperandCount() == 3) + { + assert(node->GetAuxiliaryType() != TYP_UNKNOWN); + node->ResetHWIntrinsicId(intrinsicId, comp, node->Op(1), node->Op(2), node->Op(3), lclVar); + } + else + { + assert(node->GetOperandCount() == 2); + node->ResetHWIntrinsicId(intrinsicId, comp, node->Op(1), node->Op(2), lclVar); + } + } + + if (foundUse) + { + unsigned tmpNum = comp->lvaGrabTemp(true DEBUGARG("Return value result/FFR")); + LclVarDsc* tmpVarDsc = comp->lvaGetDesc(tmpNum); + tmpVarDsc->lvType = node->TypeGet(); + GenTree* storeLclVar; + use.ReplaceWithLclVar(comp, tmpNum, &storeLclVar); + } + else + { + node->SetUnusedValue(); + } + + StoreFFRValue(node); + break; + } case NI_Sve_LoadVectorFirstFaulting: { LIR::Use use; @@ -4082,8 +4125,10 @@ void Lowering::StoreFFRValue(GenTreeHWIntrinsic* node) #ifdef DEBUG switch (node->GetHWIntrinsicId()) { - case NI_Sve_SetFfr: + case NI_Sve_GatherVectorFirstFaulting: case NI_Sve_LoadVectorFirstFaulting: + case NI_Sve_SetFfr: + break; default: assert(!"Unexpected HWIntrinsicId"); From 18f8f52c0015278a5e12890550aabe40cd1c7daa Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 26 Jul 2024 09:26:24 -0700 Subject: [PATCH 46/83] reuse some of the code --- src/coreclr/jit/hwintrinsiccodegenarm64.cpp | 63 +++++---------------- 1 file changed, 13 insertions(+), 50 deletions(-) diff --git a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp index f6c84cb5809f4..f6a490c3517d0 100644 --- a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp @@ -2075,42 +2075,7 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) } } - if (!varTypeIsSIMD(intrin.op2->gtType)) - { - // GatherVector...(Vector mask, T* address, Vector indices) - - assert(intrin.numOperands == 3); - emitAttr baseSize = emitActualTypeSize(intrin.baseType); - insScalableOpts sopt = INS_SCALABLE_OPTS_NONE; - - if (baseSize == EA_8BYTE) - { - // Index is multiplied. - sopt = (ins == INS_sve_ldff1b || ins == INS_sve_ldff1sb) ? INS_SCALABLE_OPTS_NONE - : INS_SCALABLE_OPTS_LSL_N; - } - else - { - // Index is sign or zero extended to 64bits, then multiplied. - assert(baseSize == EA_4BYTE); - opt = varTypeIsUnsigned(node->GetAuxiliaryType()) ? INS_OPTS_SCALABLE_S_UXTW - : INS_OPTS_SCALABLE_S_SXTW; - - sopt = (ins == INS_sve_ldff1b || ins == INS_sve_ldff1sb) ? INS_SCALABLE_OPTS_NONE - : INS_SCALABLE_OPTS_MOD_N; - } - - GetEmitter()->emitIns_R_R_R_R(ins, emitSize, targetReg, op1Reg, op2Reg, op3Reg, opt, sopt); - } - else - { - // GatherVector...(Vector mask, Vector addresses) - - assert(intrin.numOperands == 2); - GetEmitter()->emitIns_R_R_R_I(ins, emitSize, targetReg, op1Reg, op2Reg, 0, opt); - } - - break; + FALLTHROUGH; } case NI_Sve_GatherVector: case NI_Sve_GatherVectorByteZeroExtend: @@ -2128,25 +2093,24 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) { // GatherVector...(Vector mask, T* address, Vector indices) - assert(intrin.numOperands == 3); - emitAttr baseSize = emitActualTypeSize(intrin.baseType); - insScalableOpts sopt = INS_SCALABLE_OPTS_NONE; + emitAttr baseSize = emitActualTypeSize(intrin.baseType); + bool isLoadingBytes = ((ins == INS_sve_ld1b) || (ins == INS_sve_ld1sb) || (ins == INS_sve_ldff1b) || + (ins == INS_sve_ldff1sb)); + insScalableOpts sopt = INS_SCALABLE_OPTS_NONE; - if (baseSize == EA_8BYTE) - { - // Index is multiplied. - sopt = (ins == INS_sve_ld1b || ins == INS_sve_ld1sb) ? INS_SCALABLE_OPTS_NONE - : INS_SCALABLE_OPTS_LSL_N; - } - else + if (baseSize == EA_4BYTE) { // Index is sign or zero extended to 64bits, then multiplied. - assert(baseSize == EA_4BYTE); opt = varTypeIsUnsigned(node->GetAuxiliaryType()) ? INS_OPTS_SCALABLE_S_UXTW : INS_OPTS_SCALABLE_S_SXTW; - sopt = (ins == INS_sve_ld1b || ins == INS_sve_ld1sb) ? INS_SCALABLE_OPTS_NONE - : INS_SCALABLE_OPTS_MOD_N; + sopt = isLoadingBytes ? INS_SCALABLE_OPTS_NONE : INS_SCALABLE_OPTS_MOD_N; + } + else + { + // Index is multiplied. + assert(baseSize == EA_8BYTE); + sopt = isLoadingBytes ? INS_SCALABLE_OPTS_NONE : INS_SCALABLE_OPTS_LSL_N; } GetEmitter()->emitIns_R_R_R_R(ins, emitSize, targetReg, op1Reg, op2Reg, op3Reg, opt, sopt); @@ -2155,7 +2119,6 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) { // GatherVector...(Vector mask, Vector addresses) - assert(intrin.numOperands == 2); GetEmitter()->emitIns_R_R_R_I(ins, emitSize, targetReg, op1Reg, op2Reg, 0, opt); } From 0755372917d91ee83af7a9d5b5d908ce40fb898b Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 26 Jul 2024 12:13:09 -0700 Subject: [PATCH 47/83] Handle the special effect for SetFfr --- src/coreclr/jit/disasm.cpp | 4 ++-- src/coreclr/jit/fgdiagnostic.cpp | 9 +++++---- src/coreclr/jit/gentree.cpp | 10 ++++++---- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/coreclr/jit/disasm.cpp b/src/coreclr/jit/disasm.cpp index bff93c85150a6..c5c35c0bfd8c4 100644 --- a/src/coreclr/jit/disasm.cpp +++ b/src/coreclr/jit/disasm.cpp @@ -1619,8 +1619,8 @@ bool DisAssembler::InitCoredistoolsLibrary() s_disCoreDisToolsLibraryInitialized = true; // we only initialize once, even if it fails. s_disCoreDisToolsLibraryLoadSuccessful = false; // assume the worst - const WCHAR* coreDisToolsLibraryNameW = MAKEDLLNAME_W("coredistools"); - const CHAR* coreDisToolsLibraryNameA = MAKEDLLNAME_A("coredistools"); + const WCHAR* coreDisToolsLibraryNameW = MAKEDLLNAME_W("D:\\git\\jitutils\\obj\\win-x64\\Debug\\bin\\coredistools"); + const CHAR* coreDisToolsLibraryNameA = MAKEDLLNAME_A("D:\\git\\jitutils\\obj\\win-x64\\Debug\\bin\\coredistools"); HMODULE hCoreDisToolsLib = NULL; diff --git a/src/coreclr/jit/fgdiagnostic.cpp b/src/coreclr/jit/fgdiagnostic.cpp index 9f1a72afce63e..e014838915c03 100644 --- a/src/coreclr/jit/fgdiagnostic.cpp +++ b/src/coreclr/jit/fgdiagnostic.cpp @@ -3428,14 +3428,15 @@ void Compiler::fgDebugCheckFlags(GenTree* tree, BasicBlock* block) #if defined(TARGET_ARM64) case NI_ArmBase_Yield: - case NI_Sve_PrefetchBytes: - case NI_Sve_PrefetchInt16: - case NI_Sve_PrefetchInt32: - case NI_Sve_PrefetchInt64: case NI_Sve_GatherPrefetch16Bit: case NI_Sve_GatherPrefetch32Bit: case NI_Sve_GatherPrefetch64Bit: case NI_Sve_GatherPrefetch8Bit: + case NI_Sve_PrefetchBytes: + case NI_Sve_PrefetchInt16: + case NI_Sve_PrefetchInt32: + case NI_Sve_PrefetchInt64: + case NI_Sve_SetFfr: { assert(tree->OperRequiresCallFlag(this)); expectedFlags |= GTF_GLOB_REF; diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 64cb8f6b67bdc..0fde1f828a553 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -27281,6 +27281,7 @@ bool GenTreeHWIntrinsic::OperRequiresCallFlag() const case NI_Sve_GatherPrefetch32Bit: case NI_Sve_GatherPrefetch64Bit: case NI_Sve_GatherPrefetch8Bit: + case NI_Sve_SetFfr: { return true; } @@ -27463,14 +27464,15 @@ void GenTreeHWIntrinsic::Initialize(NamedIntrinsic intrinsicId) #if defined(TARGET_ARM64) case NI_ArmBase_Yield: - case NI_Sve_PrefetchBytes: - case NI_Sve_PrefetchInt16: - case NI_Sve_PrefetchInt32: - case NI_Sve_PrefetchInt64: case NI_Sve_GatherPrefetch16Bit: case NI_Sve_GatherPrefetch32Bit: case NI_Sve_GatherPrefetch64Bit: case NI_Sve_GatherPrefetch8Bit: + case NI_Sve_PrefetchBytes: + case NI_Sve_PrefetchInt16: + case NI_Sve_PrefetchInt32: + case NI_Sve_PrefetchInt64: + case NI_Sve_SetFfr: { // Mark as a call and global reference, much as is done for GT_KEEPALIVE gtFlags |= (GTF_CALL | GTF_GLOB_REF); From 567a44254cadfaff19d86c0e842096a050e1c661 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 26 Jul 2024 12:49:40 -0700 Subject: [PATCH 48/83] some fixes + test coverage --- src/coreclr/jit/lowerarmarch.cpp | 6 ++++-- .../JIT/HardwareIntrinsics/Arm/Shared/SveFfrTest.template | 8 ++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/coreclr/jit/lowerarmarch.cpp b/src/coreclr/jit/lowerarmarch.cpp index ee026a7d23d7e..be09ce70fcf4e 100644 --- a/src/coreclr/jit/lowerarmarch.cpp +++ b/src/coreclr/jit/lowerarmarch.cpp @@ -1786,7 +1786,8 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node) // only if it was trashed. If it was not trashed, we do not have to reload the // contents of the FFR register. - GenTree* lclVar = comp->gtNewLclvNode(comp->lvaFfrRegister, TYP_MASK); + unsigned lclNum = comp->getFFRegisterVarNum(); + GenTree* lclVar = comp->gtNewLclvNode(lclNum, TYP_MASK); BlockRange().InsertBefore(node, lclVar); LowerNode(lclVar); @@ -1829,7 +1830,8 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node) // only if it was trashed. If it was not trashed, we do not have to reload the // contents of the FFR register. - GenTree* lclVar = comp->gtNewLclvNode(comp->lvaFfrRegister, TYP_MASK); + unsigned lclNum = comp->getFFRegisterVarNum(); + GenTree* lclVar = comp->gtNewLclvNode(lclNum, TYP_MASK); BlockRange().InsertBefore(node, lclVar); LowerNode(lclVar); diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveFfrTest.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveFfrTest.template index dfb7e6a2b2629..b16742a718121 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveFfrTest.template +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveFfrTest.template @@ -174,6 +174,10 @@ namespace JIT.HardwareIntrinsics.Arm Sve.SetFfr( Unsafe.Read>(_dataTable.inArray1Ptr) ); + + // call to make sure FFR contents are preserved + Console.WriteLine(""); + var result = Sve.GetFfr{VectorBaseType}(); Unsafe.Write(_dataTable.outArrayPtr, result); @@ -189,6 +193,10 @@ namespace JIT.HardwareIntrinsics.Arm Sve.SetFfr( Sve.LoadVector(loadMask, ({VectorBaseType}*)(_dataTable.inArray1Ptr)) ); + + // call to make sure FFR contents are preserved + Console.WriteLine(""); + var result = Sve.GetFfr{VectorBaseType}(); Unsafe.Write(_dataTable.outArrayPtr, result); From 3ac987d58fb084046c2c982cd3c61dcbf9018940 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 26 Jul 2024 15:07:22 -0700 Subject: [PATCH 49/83] do not zero init lvaFfrRegister --- src/coreclr/jit/compiler.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index e4ab163452a16..99110bf769292 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -4243,7 +4243,8 @@ bool Compiler::fgVarIsNeverZeroInitializedInProlog(unsigned varNum) { LclVarDsc* varDsc = lvaGetDesc(varNum); bool result = varDsc->lvIsParam || lvaIsOSRLocal(varNum) || (varNum == lvaGSSecurityCookie) || - (varNum == lvaInlinedPInvokeFrameVar) || (varNum == lvaStubArgumentVar) || (varNum == lvaRetAddrVar); + (varNum == lvaInlinedPInvokeFrameVar) || (varNum == lvaStubArgumentVar) || + (varNum == lvaRetAddrVar) || (varNum == lvaFfrRegister); #if FEATURE_FIXED_OUT_ARGS result = result || (varNum == lvaOutgoingArgSpaceVar); From e8f7fcd8e94acb0e91041785786d1133965b6c5b Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 26 Jul 2024 15:13:37 -0700 Subject: [PATCH 50/83] reverted local change --- src/coreclr/jit/disasm.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/coreclr/jit/disasm.cpp b/src/coreclr/jit/disasm.cpp index c5c35c0bfd8c4..bff93c85150a6 100644 --- a/src/coreclr/jit/disasm.cpp +++ b/src/coreclr/jit/disasm.cpp @@ -1619,8 +1619,8 @@ bool DisAssembler::InitCoredistoolsLibrary() s_disCoreDisToolsLibraryInitialized = true; // we only initialize once, even if it fails. s_disCoreDisToolsLibraryLoadSuccessful = false; // assume the worst - const WCHAR* coreDisToolsLibraryNameW = MAKEDLLNAME_W("D:\\git\\jitutils\\obj\\win-x64\\Debug\\bin\\coredistools"); - const CHAR* coreDisToolsLibraryNameA = MAKEDLLNAME_A("D:\\git\\jitutils\\obj\\win-x64\\Debug\\bin\\coredistools"); + const WCHAR* coreDisToolsLibraryNameW = MAKEDLLNAME_W("coredistools"); + const CHAR* coreDisToolsLibraryNameA = MAKEDLLNAME_A("coredistools"); HMODULE hCoreDisToolsLib = NULL; From 77ec96cfae5c3360761b1dcd3ccf3c9c122adba1 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 26 Jul 2024 15:34:18 -0700 Subject: [PATCH 51/83] fix build break --- src/coreclr/jit/compiler.hpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index 99110bf769292..4f70a01a8a79e 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -4243,8 +4243,11 @@ bool Compiler::fgVarIsNeverZeroInitializedInProlog(unsigned varNum) { LclVarDsc* varDsc = lvaGetDesc(varNum); bool result = varDsc->lvIsParam || lvaIsOSRLocal(varNum) || (varNum == lvaGSSecurityCookie) || - (varNum == lvaInlinedPInvokeFrameVar) || (varNum == lvaStubArgumentVar) || - (varNum == lvaRetAddrVar) || (varNum == lvaFfrRegister); + (varNum == lvaInlinedPInvokeFrameVar) || (varNum == lvaStubArgumentVar) || (varNum == lvaRetAddrVar); + +#ifdef TARGET_ARM64 + result = result || (varNum == lvaFfrRegister); +#endif #if FEATURE_FIXED_OUT_ARGS result = result || (varNum == lvaOutgoingArgSpaceVar); From df0d4be1c917838001c02242b676e9ceb676ad08 Mon Sep 17 00:00:00 2001 From: TIHan Date: Sat, 27 Jul 2024 18:32:08 -0700 Subject: [PATCH 52/83] Fixing flags for OSX --- .../System/Buffers/BoundedMemory.Unix.cs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs index 26626b50cbe9f..16d76449849f3 100644 --- a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs +++ b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs @@ -128,11 +128,20 @@ private AllocHGlobalHandle(IntPtr buffer, ulong allocationSize) internal static AllocHGlobalHandle Allocate(nint byteLength, PoisonPagePlacement placement) { + int flags = 0; + if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + { + flags = MAP_PRIVATE | MAP_ANONYMOUS_OSX; + } + else + { + flags = MAP_PRIVATE | MAP_ANONYMOUS; + } // Allocate number of pages to incorporate required (byteLength bytes of) memory and an additional page to create a poison page. int pageSize = Environment.SystemPageSize; int allocationSize = (int)(((byteLength / pageSize) + ((byteLength % pageSize) == 0 ? 0 : 1) + 1) * pageSize); - IntPtr buffer = mmap(IntPtr.Zero, (ulong)allocationSize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + IntPtr buffer = mmap(IntPtr.Zero, (ulong)allocationSize, PROT_READ | PROT_WRITE, flags, -1, 0); if (buffer == IntPtr.Zero) { @@ -174,6 +183,7 @@ protected override bool ReleaseHandle() // Defined in const int MAP_PRIVATE = 0x2; const int MAP_ANONYMOUS = 0x20; + const int MAP_ANONYMOUS_OSX = 0x1000; const int PROT_NONE = 0x0; const int PROT_READ = 0x1; const int PROT_WRITE = 0x2; From 54ae1b2a90fede621e6cb263e903fb7d76e46dd7 Mon Sep 17 00:00:00 2001 From: TIHan Date: Sat, 27 Jul 2024 19:22:23 -0700 Subject: [PATCH 53/83] Fixup unix impl --- .../System/Buffers/BoundedMemory.Unix.cs | 188 +++++++++++------- 1 file changed, 116 insertions(+), 72 deletions(-) diff --git a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs index 16d76449849f3..6f4add5d16939 100644 --- a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs +++ b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs @@ -7,28 +7,111 @@ namespace System.Buffers { public static unsafe partial class BoundedMemory { + private static readonly int SystemPageSize = Environment.SystemPageSize; + private static UnixImplementation AllocateWithoutDataPopulationUnix(int elementCount, PoisonPagePlacement placement) where T : unmanaged { - return new UnixImplementation(elementCount, placement); + long cb, totalBytesToAllocate; + checked + { + cb = elementCount * sizeof(T); + totalBytesToAllocate = cb; + + // We only need to round the count up if it's not an exact multiple + // of the system page size. + + long leftoverBytes = totalBytesToAllocate % SystemPageSize; + if (leftoverBytes != 0) + { + totalBytesToAllocate += SystemPageSize - leftoverBytes; + } + + // Finally, account for the poison pages at the front and back. + + totalBytesToAllocate += 2 * SystemPageSize; + } + + // Reserve and commit the entire range as NOACCESS. + + MMapHandle handle = MMapHandle.Allocate( + address: IntPtr.Zero, + length: checked((nuint)totalBytesToAllocate), + prot: UnsafeNativeMethods.PROT_NONE, + flags: UnsafeNativeMethods.MAP_PRIVATE | UnsafeNativeMethods.MAP_ANONYMOUS); + + if (handle == null || handle.IsInvalid) + { + int lastError = Marshal.GetLastPInvokeError(); + handle?.Dispose(); + throw new InvalidOperationException($"mmap failed unexpectedly with {lastError}."); + } + + // Done allocating! Now carve out a READWRITE section bookended by the NOACCESS + // pages and return that carved-out section to the caller. Since memory protection + // flags only apply at page-level granularity, we need to "left-align" or "right- + // align" the section we carve out so that it's guaranteed adjacent to one of + // the NOACCESS bookend pages. + + return new UnixImplementation( + handle: handle, + byteOffsetIntoHandle: (placement == PoisonPagePlacement.Before) + ? SystemPageSize /* just after leading poison page */ + : checked((int)(totalBytesToAllocate - SystemPageSize - cb)) /* just before trailing poison page */, + elementCount: elementCount) + { + Protection = UnsafeNativeMethods.PROT_WRITE | UnsafeNativeMethods.PROT_READ + }; } private sealed class UnixImplementation : BoundedMemory where T : unmanaged { - private readonly AllocHGlobalHandle _handle; + private readonly MMapHandle _handle; + private readonly int _byteOffsetIntoHandle; private readonly int _elementCount; private readonly BoundedMemoryManager _memoryManager; + private int _prot; - public UnixImplementation(int elementCount, PoisonPagePlacement placement) + internal UnixImplementation(MMapHandle handle, int byteOffsetIntoHandle, int elementCount) { - _handle = AllocHGlobalHandle.Allocate(checked(elementCount * (nint)sizeof(T)), placement); + _handle = handle; + _byteOffsetIntoHandle = byteOffsetIntoHandle; _elementCount = elementCount; _memoryManager = new BoundedMemoryManager(this); + _prot = UnsafeNativeMethods.PROT_NONE; } - public override bool IsReadonly => false; + public override bool IsReadonly => (Protection != (UnsafeNativeMethods.PROT_WRITE | UnsafeNativeMethods.PROT_READ)); public override int Length => _elementCount; + internal int Protection + { + get + { + return _prot; + } + set + { + bool refAdded = false; + try + { + _handle.DangerousAddRef(ref refAdded); + if (UnsafeNativeMethods.mprotect(_handle.DangerousGetHandle(), _handle.Length, _prot) != 0) + { + throw new InvalidOperationException($"mprotected failed with {Marshal.GetLastPInvokeError()}."); + } + _prot = value; + } + finally + { + if (refAdded) + { + _handle.DangerousRelease(); + } + } + } + } + public override Memory Memory => _memoryManager.Memory; public override Span Span @@ -39,7 +122,7 @@ public override Span Span try { _handle.DangerousAddRef(ref refAdded); - return new Span((void*)_handle.DangerousGetHandle(), _elementCount); + return new Span((void*)(_handle.DangerousGetHandle() + _byteOffsetIntoHandle), _elementCount); } finally { @@ -58,12 +141,12 @@ public override void Dispose() public override void MakeReadonly() { - // no-op + Protection = UnsafeNativeMethods.PROT_READ; } public override void MakeWriteable() { - // no-op + Protection = UnsafeNativeMethods.PROT_WRITE | UnsafeNativeMethods.PROT_READ; } private sealed class BoundedMemoryManager : MemoryManager @@ -95,7 +178,7 @@ public override MemoryHandle Pin(int elementIndex) try { _impl._handle.DangerousAddRef(ref refAdded); - return new MemoryHandle((T*)_impl._handle.DangerousGetHandle() + elementIndex); + return new MemoryHandle((T*)(_impl._handle.DangerousGetHandle() + _impl._byteOffsetIntoHandle) + elementIndex); } finally { @@ -113,89 +196,50 @@ public override void Unpin() } } - private sealed class AllocHGlobalHandle : SafeHandle + private sealed class MMapHandle : SafeHandle { - private IntPtr buffer; - private ulong allocationSize; + public nuint Length { get; private set; } // Called by P/Invoke when returning SafeHandles - private AllocHGlobalHandle(IntPtr buffer, ulong allocationSize) + public MMapHandle(nuint length) : base(IntPtr.Zero, ownsHandle: true) { - this.buffer = buffer; - this.allocationSize = allocationSize; + Length = length; } - internal static AllocHGlobalHandle Allocate(nint byteLength, PoisonPagePlacement placement) + internal static MMapHandle Allocate(IntPtr address, nuint length, int prot, int flags) { - int flags = 0; - if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) - { - flags = MAP_PRIVATE | MAP_ANONYMOUS_OSX; - } - else - { - flags = MAP_PRIVATE | MAP_ANONYMOUS; - } - - // Allocate number of pages to incorporate required (byteLength bytes of) memory and an additional page to create a poison page. - int pageSize = Environment.SystemPageSize; - int allocationSize = (int)(((byteLength / pageSize) + ((byteLength % pageSize) == 0 ? 0 : 1) + 1) * pageSize); - IntPtr buffer = mmap(IntPtr.Zero, (ulong)allocationSize, PROT_READ | PROT_WRITE, flags, -1, 0); - - if (buffer == IntPtr.Zero) - { - throw new InvalidOperationException($"Memory allocation failed with error {Marshal.GetLastPInvokeError()}."); - } - - // Depending on the PoisonPagePlacement requirement (before/after) initialise the baseAddress and poisonPageAddress to point to the location - // in the buffer. Here the baseAddress points to the first valid allocation and poisonPageAddress points to the first invalid location. - // For `PoisonPagePlacement.Before` the first page is made inaccessible using mprotect and baseAddress points to the start of the second page. - // The allocation and protection is at the granularity of a page. Thus, `PoisonPagePlacement.Before` configuration has an additional accessible - // memory at the end of the page (bytes equivalent to `pageSize - (byteLength % pageSize)`). - // For `PoisonPagePlacement.After`, we adjust the baseAddress so that inaccessible memory is at the `byteLength` offset from the baseAddress. - IntPtr baseAddress = buffer + pageSize; - IntPtr poisonPageAddress = buffer; - if (placement == PoisonPagePlacement.After) - { - baseAddress = buffer + (allocationSize - pageSize - byteLength); - poisonPageAddress = buffer + (allocationSize - pageSize); - } - - // Protect the page before/after based on the poison page placement. - if (mprotect(poisonPageAddress, (ulong) pageSize, PROT_NONE) == -1) - { - throw new InvalidOperationException($"Failed to mark page as a poison page using mprotect with error :{Marshal.GetLastPInvokeError()}."); - } - - AllocHGlobalHandle retVal = new AllocHGlobalHandle(buffer, (ulong)allocationSize); - retVal.SetHandle(baseAddress); // this base address would be used as the start of Span that is used during unit testing. + MMapHandle retVal = new MMapHandle(length); + retVal.SetHandle(UnsafeNativeMethods.mmap(address, length, prot, flags, -1, 0)); return retVal; } + // Do not provide a finalizer - SafeHandle's critical finalizer will + // call ReleaseHandle for you. + public override bool IsInvalid => (handle == IntPtr.Zero); - protected override bool ReleaseHandle() - { - return munmap(buffer, allocationSize) == 0; - } + protected override bool ReleaseHandle() => + UnsafeNativeMethods.munmap(handle, Length) == 0; + } - // Defined in - const int MAP_PRIVATE = 0x2; - const int MAP_ANONYMOUS = 0x20; - const int MAP_ANONYMOUS_OSX = 0x1000; - const int PROT_NONE = 0x0; - const int PROT_READ = 0x1; - const int PROT_WRITE = 0x2; + private static class UnsafeNativeMethods + { + // Defined in + public const int MAP_PRIVATE = 0x2; + public static readonly int MAP_ANONYMOUS = OperatingSystem.IsLinux() ? 0x20 : 0x1000; + public const int PROT_NONE = 0x0; + public const int PROT_READ = 0x1; + public const int PROT_WRITE = 0x2; [DllImport("libc", SetLastError = true)] - static extern IntPtr mmap(IntPtr address, ulong length, int prot, int flags, int fd, int offset); + public static extern IntPtr mmap(IntPtr address, ulong length, int prot, int flags, int fd, int offset); [DllImport("libc", SetLastError = true)] - static extern IntPtr munmap(IntPtr address, ulong length); + public static extern IntPtr munmap(IntPtr address, ulong length); [DllImport("libc", SetLastError = true)] - static extern int mprotect(IntPtr address, ulong length, int prot); + public static extern int mprotect(IntPtr address, ulong length, int prot); } } } From 62f29eb58949c37af7b7a97a944ff9ead55f4255 Mon Sep 17 00:00:00 2001 From: TIHan Date: Sat, 27 Jul 2024 19:25:18 -0700 Subject: [PATCH 54/83] Trying to fix build --- .../TestUtilities/System/Buffers/BoundedMemory.Creation.cs | 2 ++ .../tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs | 4 +--- .../TestUtilities/System/Buffers/BoundedMemory.Windows.cs | 4 +--- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Creation.cs b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Creation.cs index 1e8118af8da03..7a2eb32d0fed5 100644 --- a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Creation.cs +++ b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Creation.cs @@ -10,6 +10,8 @@ namespace System.Buffers /// public static partial class BoundedMemory { + private static readonly int SystemPageSize = Environment.SystemPageSize; + /// /// Allocates a new region which is immediately preceded by /// or immediately followed by a poison (MEM_NOACCESS) page. If diff --git a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs index 6f4add5d16939..cceee3dcd4519 100644 --- a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs +++ b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs @@ -7,8 +7,6 @@ namespace System.Buffers { public static unsafe partial class BoundedMemory { - private static readonly int SystemPageSize = Environment.SystemPageSize; - private static UnixImplementation AllocateWithoutDataPopulationUnix(int elementCount, PoisonPagePlacement placement) where T : unmanaged { long cb, totalBytesToAllocate; @@ -223,7 +221,7 @@ protected override bool ReleaseHandle() => UnsafeNativeMethods.munmap(handle, Length) == 0; } - private static class UnsafeNativeMethods + private static partial class UnsafeNativeMethods { // Defined in public const int MAP_PRIVATE = 0x2; diff --git a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Windows.cs b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Windows.cs index d6dc353ed2daa..bb8843c757abd 100644 --- a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Windows.cs +++ b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Windows.cs @@ -7,8 +7,6 @@ namespace System.Buffers { public static unsafe partial class BoundedMemory { - private static readonly int SystemPageSize = Environment.SystemPageSize; - private static WindowsImplementation AllocateWithoutDataPopulationWindows(int elementCount, PoisonPagePlacement placement) where T : unmanaged { long cb, totalBytesToAllocate; @@ -297,7 +295,7 @@ protected override bool ReleaseHandle() => UnsafeNativeMethods.VirtualFree(handle, IntPtr.Zero, VirtualAllocAllocationType.MEM_RELEASE) != 0; } - private static class UnsafeNativeMethods + private static partial class UnsafeNativeMethods { private const string KERNEL32_LIB = "kernel32.dll"; From 04c988ae5ca133ed5422670301bcc9c02f3268ec Mon Sep 17 00:00:00 2001 From: TIHan Date: Sat, 27 Jul 2024 21:37:12 -0700 Subject: [PATCH 55/83] Fix osx calls --- .../System/Buffers/BoundedMemory.Unix.cs | 57 +++++++++++++++++-- 1 file changed, 51 insertions(+), 6 deletions(-) diff --git a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs index cceee3dcd4519..c4657e33dad68 100644 --- a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs +++ b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs @@ -230,14 +230,59 @@ private static partial class UnsafeNativeMethods public const int PROT_READ = 0x1; public const int PROT_WRITE = 0x2; - [DllImport("libc", SetLastError = true)] - public static extern IntPtr mmap(IntPtr address, ulong length, int prot, int flags, int fd, int offset); + private static class Linux + { + [DllImport("libc", SetLastError = true)] + public static extern IntPtr mmap(IntPtr address, ulong length, int prot, int flags, int fd, int offset); + + [DllImport("libc", SetLastError = true)] + public static extern IntPtr munmap(IntPtr address, ulong length); - [DllImport("libc", SetLastError = true)] - public static extern IntPtr munmap(IntPtr address, ulong length); + [DllImport("libc", SetLastError = true)] + public static extern int mprotect(IntPtr address, ulong length, int prot); + } - [DllImport("libc", SetLastError = true)] - public static extern int mprotect(IntPtr address, ulong length, int prot); + private static class Osx + { + [DllImport("libSystem", SetLastError = true)] + public static extern IntPtr mmap(IntPtr address, ulong length, int prot, int flags, int fd, int offset); + + [DllImport("libSystem", SetLastError = true)] + public static extern IntPtr munmap(IntPtr address, ulong length); + + [DllImport("libSystem", SetLastError = true)] + public static extern int mprotect(IntPtr address, ulong length, int prot); + } + + public static IntPtr mmap(IntPtr address, ulong length, int prot, int flags, int fd, int offset) + { + if (OperatingSystem.IsLinux()) + { + return Linux.mmap(address, length, prot, flags, fd, offset); + } + + return Osx.mmap(address, length, prot, flags, fd, offset); + } + + public static IntPtr munmap(IntPtr address, ulong length) + { + if (OperatingSystem.IsLinux()) + { + return Linux.munmap(address, length); + } + + return Osx.munmap(address, length); + } + + public static int mprotect(IntPtr address, ulong length, int prot) + { + if (OperatingSystem.IsLinux()) + { + return Linux.mprotect(address, length, prot); + } + + return Osx.mprotect(address, length, prot); + } } } } From 34b0aa10f306a46a3c7a52f6bfb84922cc7d4885 Mon Sep 17 00:00:00 2001 From: TIHan Date: Sun, 28 Jul 2024 12:45:57 -0700 Subject: [PATCH 56/83] Fix unix impl - forgot to use 'value' --- .../tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs index c4657e33dad68..a5f386fb576e5 100644 --- a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs +++ b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs @@ -94,7 +94,7 @@ internal int Protection try { _handle.DangerousAddRef(ref refAdded); - if (UnsafeNativeMethods.mprotect(_handle.DangerousGetHandle(), _handle.Length, _prot) != 0) + if (UnsafeNativeMethods.mprotect(_handle.DangerousGetHandle(), _handle.Length, value) != 0) { throw new InvalidOperationException($"mprotected failed with {Marshal.GetLastPInvokeError()}."); } From 9c9d9752e84aab456de325e901cf849cd4a87b84 Mon Sep 17 00:00:00 2001 From: TIHan Date: Sun, 28 Jul 2024 14:36:15 -0700 Subject: [PATCH 57/83] Added default impl --- .../System/Buffers/BoundedMemory.Creation.cs | 8 +- .../System/Buffers/BoundedMemory.Default.cs | 146 ++++++++++++++++++ .../tests/TestUtilities/TestUtilities.csproj | 1 + .../HardwareIntrinsics/Arm/Sve/Sve_r.csproj | 1 + .../HardwareIntrinsics/Arm/Sve/Sve_ro.csproj | 1 + 5 files changed, 155 insertions(+), 2 deletions(-) create mode 100644 src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Default.cs diff --git a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Creation.cs b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Creation.cs index 7a2eb32d0fed5..cec236105aafa 100644 --- a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Creation.cs +++ b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Creation.cs @@ -11,7 +11,7 @@ namespace System.Buffers public static partial class BoundedMemory { private static readonly int SystemPageSize = Environment.SystemPageSize; - + /// /// Allocates a new region which is immediately preceded by /// or immediately followed by a poison (MEM_NOACCESS) page. If @@ -84,10 +84,14 @@ private static BoundedMemory AllocateWithoutDataPopulation(int elementCoun { return AllocateWithoutDataPopulationWindows(elementCount, placement); } - else + else if (RuntimeInformation.IsOSPlatform(OSPlatform.FreeBSD) || RuntimeInformation.IsOSPlatform(OSPlatform.OSX) || RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) { return AllocateWithoutDataPopulationUnix(elementCount, placement); } + else + { + return AllocateWithoutDataPopulationDefault(elementCount, placement); + } } } } diff --git a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Default.cs b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Default.cs new file mode 100644 index 0000000000000..fc8134859f92f --- /dev/null +++ b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Default.cs @@ -0,0 +1,146 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.InteropServices; + +namespace System.Buffers +{ + public static unsafe partial class BoundedMemory + { + private static DefaultImplementation AllocateWithoutDataPopulationDefault(int elementCount, PoisonPagePlacement placement) where T : unmanaged + { + // On non-Windows platforms, we don't yet have support for changing the permissions of individual pages. + // We'll instead use AllocHGlobal / FreeHGlobal to carve out a r+w section of unmanaged memory. + + return new DefaultImplementation(elementCount); + } + + private sealed class DefaultImplementation : BoundedMemory where T : unmanaged + { + private readonly AllocHGlobalHandle _handle; + private readonly int _elementCount; + private readonly BoundedMemoryManager _memoryManager; + + public DefaultImplementation(int elementCount) + { + _handle = AllocHGlobalHandle.Allocate(checked(elementCount * (nint)sizeof(T))); + _elementCount = elementCount; + _memoryManager = new BoundedMemoryManager(this); + } + + public override bool IsReadonly => false; + + public override int Length => _elementCount; + + public override Memory Memory => _memoryManager.Memory; + + public override Span Span + { + get + { + bool refAdded = false; + try + { + _handle.DangerousAddRef(ref refAdded); + return new Span((void*)_handle.DangerousGetHandle(), _elementCount); + } + finally + { + if (refAdded) + { + _handle.DangerousRelease(); + } + } + } + } + + public override void Dispose() + { + _handle.Dispose(); + } + + public override void MakeReadonly() + { + // no-op + } + + public override void MakeWriteable() + { + // no-op + } + + private sealed class BoundedMemoryManager : MemoryManager + { + private readonly DefaultImplementation _impl; + + public BoundedMemoryManager(DefaultImplementation impl) + { + _impl = impl; + } + + public override Memory Memory => CreateMemory(_impl._elementCount); + + protected override void Dispose(bool disposing) + { + // no-op; the handle will be disposed separately + } + + public override Span GetSpan() => _impl.Span; + + public override MemoryHandle Pin(int elementIndex) + { + if ((uint)elementIndex > (uint)_impl._elementCount) + { + throw new ArgumentOutOfRangeException(paramName: nameof(elementIndex)); + } + + bool refAdded = false; + try + { + _impl._handle.DangerousAddRef(ref refAdded); + return new MemoryHandle((T*)_impl._handle.DangerousGetHandle() + elementIndex); + } + finally + { + if (refAdded) + { + _impl._handle.DangerousRelease(); + } + } + } + + public override void Unpin() + { + // no-op - we don't unpin native memory + } + } + } + + private sealed class AllocHGlobalHandle : SafeHandle + { + // Called by P/Invoke when returning SafeHandles + private AllocHGlobalHandle() + : base(IntPtr.Zero, ownsHandle: true) + { + } + + internal static AllocHGlobalHandle Allocate(nint byteLength) + { + AllocHGlobalHandle retVal = new AllocHGlobalHandle(); + retVal.SetHandle(Marshal.AllocHGlobal(byteLength)); // this is for unit testing; don't bother setting up a CER on Full Framework + return retVal; + } + + // Do not provide a finalizer - SafeHandle's critical finalizer will + // call ReleaseHandle for you. + + public override bool IsInvalid => (handle == IntPtr.Zero); + + protected override bool ReleaseHandle() + { + Marshal.FreeHGlobal(handle); + return true; + } + } + } +} \ No newline at end of file diff --git a/src/libraries/Common/tests/TestUtilities/TestUtilities.csproj b/src/libraries/Common/tests/TestUtilities/TestUtilities.csproj index 1566f163697bc..51fc1add6ef9f 100644 --- a/src/libraries/Common/tests/TestUtilities/TestUtilities.csproj +++ b/src/libraries/Common/tests/TestUtilities/TestUtilities.csproj @@ -23,6 +23,7 @@ + diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Sve/Sve_r.csproj b/src/tests/JIT/HardwareIntrinsics/Arm/Sve/Sve_r.csproj index dd4135fc1c866..5cecbd0119695 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Sve/Sve_r.csproj +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Sve/Sve_r.csproj @@ -14,6 +14,7 @@ + diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Sve/Sve_ro.csproj b/src/tests/JIT/HardwareIntrinsics/Arm/Sve/Sve_ro.csproj index e23748ff74c8b..cf917acd96487 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Sve/Sve_ro.csproj +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Sve/Sve_ro.csproj @@ -14,6 +14,7 @@ + From 921b0474232fd43bcdd30cc1be90d3d0b670b5b7 Mon Sep 17 00:00:00 2001 From: TIHan Date: Sun, 28 Jul 2024 16:26:23 -0700 Subject: [PATCH 58/83] Updating error messages --- .../tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs index a5f386fb576e5..5f937112b4eed 100644 --- a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs +++ b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs @@ -41,7 +41,7 @@ private static UnixImplementation AllocateWithoutDataPopulationUnix(int el { int lastError = Marshal.GetLastPInvokeError(); handle?.Dispose(); - throw new InvalidOperationException($"mmap failed unexpectedly with {lastError}."); + throw new InvalidOperationException($"mmap failed unexpectedly with errno: {lastError}."); } // Done allocating! Now carve out a READWRITE section bookended by the NOACCESS @@ -96,7 +96,7 @@ internal int Protection _handle.DangerousAddRef(ref refAdded); if (UnsafeNativeMethods.mprotect(_handle.DangerousGetHandle(), _handle.Length, value) != 0) { - throw new InvalidOperationException($"mprotected failed with {Marshal.GetLastPInvokeError()}."); + throw new InvalidOperationException($"mprotect failed with errno: {Marshal.GetLastPInvokeError()}. Length: {_handle.Length}, Value: {value}"); } _prot = value; } From 50a9790f00d392de0915264eace1b8931d002b82 Mon Sep 17 00:00:00 2001 From: TIHan Date: Sun, 28 Jul 2024 17:11:56 -0700 Subject: [PATCH 59/83] Attempt mprotect to determine which MAP_ANONYMOUS value to use --- src/tests/JIT/HardwareIntrinsics/Arm/Sve/Sve_ro.csproj | 1 + 1 file changed, 1 insertion(+) diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Sve/Sve_ro.csproj b/src/tests/JIT/HardwareIntrinsics/Arm/Sve/Sve_ro.csproj index cf917acd96487..3f5158feb5602 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Sve/Sve_ro.csproj +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Sve/Sve_ro.csproj @@ -6,6 +6,7 @@ Embedded True + From bf292ce58efdcabd09d0f74af3d4c38c785d7d60 Mon Sep 17 00:00:00 2001 From: TIHan Date: Sun, 28 Jul 2024 17:12:37 -0700 Subject: [PATCH 60/83] Attempt mprotect to determine which MAP_ANONYMOUS value to use --- .../System/Buffers/BoundedMemory.Unix.cs | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs index 5f937112b4eed..d99307dd66259 100644 --- a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs +++ b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs @@ -1,6 +1,8 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Net; +using System.Reflection.Metadata; using System.Runtime.InteropServices; namespace System.Buffers @@ -31,11 +33,21 @@ private static UnixImplementation AllocateWithoutDataPopulationUnix(int el // Reserve and commit the entire range as NOACCESS. + var flags = UnsafeNativeMethods.MAP_PRIVATE | UnsafeNativeMethods.MAP_ANONYMOUS_1; + + var nativeHandle = UnsafeNativeMethods.mmap(IntPtr.Zero, checked((nuint)totalBytesToAllocate), UnsafeNativeMethods.PROT_NONE, flags, -1, 0); + if (UnsafeNativeMethods.mprotect(nativeHandle, (ulong)totalBytesToAllocate, UnsafeNativeMethods.PROT_NONE) != 0) + { + // If 'mprotect' failed, it is most likely due to MAP_ANONYMOUS being the wrong value on Linux, so use the other value. + flags = UnsafeNativeMethods.MAP_PRIVATE | UnsafeNativeMethods.MAP_ANONYMOUS; + } + UnsafeNativeMethods.munmap(nativeHandle, (ulong)totalBytesToAllocate); + MMapHandle handle = MMapHandle.Allocate( address: IntPtr.Zero, length: checked((nuint)totalBytesToAllocate), prot: UnsafeNativeMethods.PROT_NONE, - flags: UnsafeNativeMethods.MAP_PRIVATE | UnsafeNativeMethods.MAP_ANONYMOUS); + flags: flags); if (handle == null || handle.IsInvalid) { @@ -225,7 +237,8 @@ private static partial class UnsafeNativeMethods { // Defined in public const int MAP_PRIVATE = 0x2; - public static readonly int MAP_ANONYMOUS = OperatingSystem.IsLinux() ? 0x20 : 0x1000; + public const int MAP_ANONYMOUS_1 = 0x1000; + public static readonly int MAP_ANONYMOUS = OperatingSystem.IsLinux() ? 0x20 : MAP_ANONYMOUS_1; public const int PROT_NONE = 0x0; public const int PROT_READ = 0x1; public const int PROT_WRITE = 0x2; From 59ebe26609e9fb8d9502ffb3776227fee1c466a7 Mon Sep 17 00:00:00 2001 From: TIHan Date: Sun, 28 Jul 2024 19:02:43 -0700 Subject: [PATCH 61/83] Add a way to query MAP_ANONYMOUS --- .../System/Buffers/BoundedMemory.Unix.cs | 17 ++++------------- .../TestUtilities/System/Buffers/CMakeLists.txt | 3 +++ .../System/Buffers/CrossplatVirtualAlloc.cpp | 17 +++++++++++++++++ .../tests/TestUtilities/TestUtilities.csproj | 1 + .../JIT/HardwareIntrinsics/Arm/Sve/Sve_r.csproj | 1 + .../HardwareIntrinsics/Arm/Sve/Sve_ro.csproj | 2 +- 6 files changed, 27 insertions(+), 14 deletions(-) create mode 100644 src/libraries/Common/tests/TestUtilities/System/Buffers/CMakeLists.txt create mode 100644 src/libraries/Common/tests/TestUtilities/System/Buffers/CrossplatVirtualAlloc.cpp diff --git a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs index d99307dd66259..e845d45230f8d 100644 --- a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs +++ b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs @@ -33,21 +33,11 @@ private static UnixImplementation AllocateWithoutDataPopulationUnix(int el // Reserve and commit the entire range as NOACCESS. - var flags = UnsafeNativeMethods.MAP_PRIVATE | UnsafeNativeMethods.MAP_ANONYMOUS_1; - - var nativeHandle = UnsafeNativeMethods.mmap(IntPtr.Zero, checked((nuint)totalBytesToAllocate), UnsafeNativeMethods.PROT_NONE, flags, -1, 0); - if (UnsafeNativeMethods.mprotect(nativeHandle, (ulong)totalBytesToAllocate, UnsafeNativeMethods.PROT_NONE) != 0) - { - // If 'mprotect' failed, it is most likely due to MAP_ANONYMOUS being the wrong value on Linux, so use the other value. - flags = UnsafeNativeMethods.MAP_PRIVATE | UnsafeNativeMethods.MAP_ANONYMOUS; - } - UnsafeNativeMethods.munmap(nativeHandle, (ulong)totalBytesToAllocate); - MMapHandle handle = MMapHandle.Allocate( address: IntPtr.Zero, length: checked((nuint)totalBytesToAllocate), prot: UnsafeNativeMethods.PROT_NONE, - flags: flags); + flags: UnsafeNativeMethods.MAP_PRIVATE | UnsafeNativeMethods.GetMapAnonymousFlag()); if (handle == null || handle.IsInvalid) { @@ -237,12 +227,13 @@ private static partial class UnsafeNativeMethods { // Defined in public const int MAP_PRIVATE = 0x2; - public const int MAP_ANONYMOUS_1 = 0x1000; - public static readonly int MAP_ANONYMOUS = OperatingSystem.IsLinux() ? 0x20 : MAP_ANONYMOUS_1; public const int PROT_NONE = 0x0; public const int PROT_READ = 0x1; public const int PROT_WRITE = 0x2; + [DllImport("CrossplatVirtualAlloc")] + public static extern int GetMapAnonymousFlag(); + private static class Linux { [DllImport("libc", SetLastError = true)] diff --git a/src/libraries/Common/tests/TestUtilities/System/Buffers/CMakeLists.txt b/src/libraries/Common/tests/TestUtilities/System/Buffers/CMakeLists.txt new file mode 100644 index 0000000000000..6c7af07db127d --- /dev/null +++ b/src/libraries/Common/tests/TestUtilities/System/Buffers/CMakeLists.txt @@ -0,0 +1,3 @@ +include_directories(${INC_PLATFORM_DIR}) +add_library(CrossplatVirtualAlloc SHARED CrossplatVirtualAlloc.cpp) +target_link_libraries(CrossplatVirtualAlloc PRIVATE platformdefines) \ No newline at end of file diff --git a/src/libraries/Common/tests/TestUtilities/System/Buffers/CrossplatVirtualAlloc.cpp b/src/libraries/Common/tests/TestUtilities/System/Buffers/CrossplatVirtualAlloc.cpp new file mode 100644 index 0000000000000..a5957cefe5a8f --- /dev/null +++ b/src/libraries/Common/tests/TestUtilities/System/Buffers/CrossplatVirtualAlloc.cpp @@ -0,0 +1,17 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#include +#include +#ifdef _WIN32 +#include +#else +#include +#endif + +#ifdef _WIN32 +extern "C" DLL_EXPORT int GetMapAnonymousFlag() +{ + return MAP_ANONYMOUS; +} +#endif \ No newline at end of file diff --git a/src/libraries/Common/tests/TestUtilities/TestUtilities.csproj b/src/libraries/Common/tests/TestUtilities/TestUtilities.csproj index 51fc1add6ef9f..fa1f2be47b285 100644 --- a/src/libraries/Common/tests/TestUtilities/TestUtilities.csproj +++ b/src/libraries/Common/tests/TestUtilities/TestUtilities.csproj @@ -49,6 +49,7 @@ + diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Sve/Sve_r.csproj b/src/tests/JIT/HardwareIntrinsics/Arm/Sve/Sve_r.csproj index 5cecbd0119695..eb5d6f5b3345d 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Sve/Sve_r.csproj +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Sve/Sve_r.csproj @@ -16,5 +16,6 @@ + diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Sve/Sve_ro.csproj b/src/tests/JIT/HardwareIntrinsics/Arm/Sve/Sve_ro.csproj index 3f5158feb5602..5deef95d409d9 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Sve/Sve_ro.csproj +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Sve/Sve_ro.csproj @@ -6,7 +6,6 @@ Embedded True - @@ -17,5 +16,6 @@ + From 489b919f5355d20e0bf97bc5facd89f13ca0913b Mon Sep 17 00:00:00 2001 From: TIHan Date: Sun, 28 Jul 2024 19:03:08 -0700 Subject: [PATCH 62/83] Add a way to query MAP_ANONYMOUS --- .../TestUtilities/System/Buffers/CrossplatVirtualAlloc.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libraries/Common/tests/TestUtilities/System/Buffers/CrossplatVirtualAlloc.cpp b/src/libraries/Common/tests/TestUtilities/System/Buffers/CrossplatVirtualAlloc.cpp index a5957cefe5a8f..e77527fb26d09 100644 --- a/src/libraries/Common/tests/TestUtilities/System/Buffers/CrossplatVirtualAlloc.cpp +++ b/src/libraries/Common/tests/TestUtilities/System/Buffers/CrossplatVirtualAlloc.cpp @@ -9,7 +9,7 @@ #include #endif -#ifdef _WIN32 +#ifndef _WIN32 extern "C" DLL_EXPORT int GetMapAnonymousFlag() { return MAP_ANONYMOUS; From b3922ff48b26668920236b8aa307f4560658d2da Mon Sep 17 00:00:00 2001 From: TIHan Date: Sun, 28 Jul 2024 20:30:12 -0700 Subject: [PATCH 63/83] trying to figure out cmake reference --- src/tests/JIT/HardwareIntrinsics/Arm/Sve/Sve_r.csproj | 1 - src/tests/JIT/HardwareIntrinsics/Arm/Sve/Sve_ro.csproj | 1 - 2 files changed, 2 deletions(-) diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Sve/Sve_r.csproj b/src/tests/JIT/HardwareIntrinsics/Arm/Sve/Sve_r.csproj index eb5d6f5b3345d..5cecbd0119695 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Sve/Sve_r.csproj +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Sve/Sve_r.csproj @@ -16,6 +16,5 @@ - diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Sve/Sve_ro.csproj b/src/tests/JIT/HardwareIntrinsics/Arm/Sve/Sve_ro.csproj index 5deef95d409d9..cf917acd96487 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Sve/Sve_ro.csproj +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Sve/Sve_ro.csproj @@ -16,6 +16,5 @@ - From 5c49435ff46cc6eebdafa4396d3befd62e459f34 Mon Sep 17 00:00:00 2001 From: TIHan Date: Sun, 28 Jul 2024 20:53:39 -0700 Subject: [PATCH 64/83] trying to figure out cmake reference --- src/tests/JIT/HardwareIntrinsics/Arm/Sve/Sve_r.csproj | 1 + src/tests/JIT/HardwareIntrinsics/Arm/Sve/Sve_ro.csproj | 1 + 2 files changed, 2 insertions(+) diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Sve/Sve_r.csproj b/src/tests/JIT/HardwareIntrinsics/Arm/Sve/Sve_r.csproj index 5cecbd0119695..3508f753ac897 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Sve/Sve_r.csproj +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Sve/Sve_r.csproj @@ -16,5 +16,6 @@ + diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Sve/Sve_ro.csproj b/src/tests/JIT/HardwareIntrinsics/Arm/Sve/Sve_ro.csproj index cf917acd96487..e22009675338b 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Sve/Sve_ro.csproj +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Sve/Sve_ro.csproj @@ -16,5 +16,6 @@ + From 8b495762bc28724621104d563a3ba135d953b502 Mon Sep 17 00:00:00 2001 From: TIHan Date: Sun, 28 Jul 2024 21:08:51 -0700 Subject: [PATCH 65/83] trying to figure out cmake reference --- .../System/Buffers/BoundedMemory.Unix.cs | 2 +- .../TestUtilities/System/Buffers/CMakeLists.txt | 4 ++-- .../System/Buffers/CrossplatVirtualAlloc.cpp | 17 ----------------- 3 files changed, 3 insertions(+), 20 deletions(-) delete mode 100644 src/libraries/Common/tests/TestUtilities/System/Buffers/CrossplatVirtualAlloc.cpp diff --git a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs index e845d45230f8d..62da4e6abc113 100644 --- a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs +++ b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs @@ -231,7 +231,7 @@ private static partial class UnsafeNativeMethods public const int PROT_READ = 0x1; public const int PROT_WRITE = 0x2; - [DllImport("CrossplatVirtualAlloc")] + [DllImport("GetMapAnonymousFlag")] public static extern int GetMapAnonymousFlag(); private static class Linux diff --git a/src/libraries/Common/tests/TestUtilities/System/Buffers/CMakeLists.txt b/src/libraries/Common/tests/TestUtilities/System/Buffers/CMakeLists.txt index 6c7af07db127d..432219669ad6f 100644 --- a/src/libraries/Common/tests/TestUtilities/System/Buffers/CMakeLists.txt +++ b/src/libraries/Common/tests/TestUtilities/System/Buffers/CMakeLists.txt @@ -1,3 +1,3 @@ include_directories(${INC_PLATFORM_DIR}) -add_library(CrossplatVirtualAlloc SHARED CrossplatVirtualAlloc.cpp) -target_link_libraries(CrossplatVirtualAlloc PRIVATE platformdefines) \ No newline at end of file +add_library(GetMapAnonymousFlag SHARED GetMapAnonymousFlag.cpp) +target_link_libraries(GetMapAnonymousFlag PRIVATE platformdefines) \ No newline at end of file diff --git a/src/libraries/Common/tests/TestUtilities/System/Buffers/CrossplatVirtualAlloc.cpp b/src/libraries/Common/tests/TestUtilities/System/Buffers/CrossplatVirtualAlloc.cpp deleted file mode 100644 index e77527fb26d09..0000000000000 --- a/src/libraries/Common/tests/TestUtilities/System/Buffers/CrossplatVirtualAlloc.cpp +++ /dev/null @@ -1,17 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -#include -#include -#ifdef _WIN32 -#include -#else -#include -#endif - -#ifndef _WIN32 -extern "C" DLL_EXPORT int GetMapAnonymousFlag() -{ - return MAP_ANONYMOUS; -} -#endif \ No newline at end of file From 754b1b577d143d9bade6bce28e24669be4babbeb Mon Sep 17 00:00:00 2001 From: TIHan Date: Sun, 28 Jul 2024 22:30:20 -0700 Subject: [PATCH 66/83] forgot to add cpp files --- .../System/Buffers/GetMapAnonymousFlag.cpp | 17 +++++++++++++++++ .../HardwareIntrinsics/Arm/Sve/CMakeLists.txt | 3 +++ .../Arm/Sve/GetMapAnonymousFlag.cpp | 17 +++++++++++++++++ 3 files changed, 37 insertions(+) create mode 100644 src/libraries/Common/tests/TestUtilities/System/Buffers/GetMapAnonymousFlag.cpp create mode 100644 src/tests/JIT/HardwareIntrinsics/Arm/Sve/CMakeLists.txt create mode 100644 src/tests/JIT/HardwareIntrinsics/Arm/Sve/GetMapAnonymousFlag.cpp diff --git a/src/libraries/Common/tests/TestUtilities/System/Buffers/GetMapAnonymousFlag.cpp b/src/libraries/Common/tests/TestUtilities/System/Buffers/GetMapAnonymousFlag.cpp new file mode 100644 index 0000000000000..e77527fb26d09 --- /dev/null +++ b/src/libraries/Common/tests/TestUtilities/System/Buffers/GetMapAnonymousFlag.cpp @@ -0,0 +1,17 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#include +#include +#ifdef _WIN32 +#include +#else +#include +#endif + +#ifndef _WIN32 +extern "C" DLL_EXPORT int GetMapAnonymousFlag() +{ + return MAP_ANONYMOUS; +} +#endif \ No newline at end of file diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Sve/CMakeLists.txt b/src/tests/JIT/HardwareIntrinsics/Arm/Sve/CMakeLists.txt new file mode 100644 index 0000000000000..432219669ad6f --- /dev/null +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Sve/CMakeLists.txt @@ -0,0 +1,3 @@ +include_directories(${INC_PLATFORM_DIR}) +add_library(GetMapAnonymousFlag SHARED GetMapAnonymousFlag.cpp) +target_link_libraries(GetMapAnonymousFlag PRIVATE platformdefines) \ No newline at end of file diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Sve/GetMapAnonymousFlag.cpp b/src/tests/JIT/HardwareIntrinsics/Arm/Sve/GetMapAnonymousFlag.cpp new file mode 100644 index 0000000000000..e77527fb26d09 --- /dev/null +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Sve/GetMapAnonymousFlag.cpp @@ -0,0 +1,17 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#include +#include +#ifdef _WIN32 +#include +#else +#include +#endif + +#ifndef _WIN32 +extern "C" DLL_EXPORT int GetMapAnonymousFlag() +{ + return MAP_ANONYMOUS; +} +#endif \ No newline at end of file From dbaa496f185444ca68303908368b689c906e42d6 Mon Sep 17 00:00:00 2001 From: TIHan Date: Mon, 29 Jul 2024 01:24:29 -0700 Subject: [PATCH 67/83] Use MemoryMappedFile and then use mprotect --- .../System/Buffers/BoundedMemory.Unix.cs | 203 ++++++------------ .../System/Buffers/CMakeLists.txt | 3 - .../System/Buffers/GetMapAnonymousFlag.cpp | 17 -- .../tests/TestUtilities/TestUtilities.csproj | 1 - .../HardwareIntrinsics/Arm/Sve/CMakeLists.txt | 3 - .../Arm/Sve/GetMapAnonymousFlag.cpp | 17 -- .../HardwareIntrinsics/Arm/Sve/Sve_r.csproj | 1 - .../HardwareIntrinsics/Arm/Sve/Sve_ro.csproj | 1 - 8 files changed, 62 insertions(+), 184 deletions(-) delete mode 100644 src/libraries/Common/tests/TestUtilities/System/Buffers/CMakeLists.txt delete mode 100644 src/libraries/Common/tests/TestUtilities/System/Buffers/GetMapAnonymousFlag.cpp delete mode 100644 src/tests/JIT/HardwareIntrinsics/Arm/Sve/CMakeLists.txt delete mode 100644 src/tests/JIT/HardwareIntrinsics/Arm/Sve/GetMapAnonymousFlag.cpp diff --git a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs index 62da4e6abc113..18a6c432f38aa 100644 --- a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs +++ b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs @@ -1,8 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System.Net; -using System.Reflection.Metadata; +using System.IO.MemoryMappedFiles; using System.Runtime.InteropServices; namespace System.Buffers @@ -11,107 +10,26 @@ public static unsafe partial class BoundedMemory { private static UnixImplementation AllocateWithoutDataPopulationUnix(int elementCount, PoisonPagePlacement placement) where T : unmanaged { - long cb, totalBytesToAllocate; - checked - { - cb = elementCount * sizeof(T); - totalBytesToAllocate = cb; - - // We only need to round the count up if it's not an exact multiple - // of the system page size. - - long leftoverBytes = totalBytesToAllocate % SystemPageSize; - if (leftoverBytes != 0) - { - totalBytesToAllocate += SystemPageSize - leftoverBytes; - } - - // Finally, account for the poison pages at the front and back. - - totalBytesToAllocate += 2 * SystemPageSize; - } - - // Reserve and commit the entire range as NOACCESS. - - MMapHandle handle = MMapHandle.Allocate( - address: IntPtr.Zero, - length: checked((nuint)totalBytesToAllocate), - prot: UnsafeNativeMethods.PROT_NONE, - flags: UnsafeNativeMethods.MAP_PRIVATE | UnsafeNativeMethods.GetMapAnonymousFlag()); - - if (handle == null || handle.IsInvalid) - { - int lastError = Marshal.GetLastPInvokeError(); - handle?.Dispose(); - throw new InvalidOperationException($"mmap failed unexpectedly with errno: {lastError}."); - } - - // Done allocating! Now carve out a READWRITE section bookended by the NOACCESS - // pages and return that carved-out section to the caller. Since memory protection - // flags only apply at page-level granularity, we need to "left-align" or "right- - // align" the section we carve out so that it's guaranteed adjacent to one of - // the NOACCESS bookend pages. - - return new UnixImplementation( - handle: handle, - byteOffsetIntoHandle: (placement == PoisonPagePlacement.Before) - ? SystemPageSize /* just after leading poison page */ - : checked((int)(totalBytesToAllocate - SystemPageSize - cb)) /* just before trailing poison page */, - elementCount: elementCount) - { - Protection = UnsafeNativeMethods.PROT_WRITE | UnsafeNativeMethods.PROT_READ - }; + return new UnixImplementation(elementCount, placement); } private sealed class UnixImplementation : BoundedMemory where T : unmanaged { - private readonly MMapHandle _handle; - private readonly int _byteOffsetIntoHandle; + private readonly MmfHandle _handle; private readonly int _elementCount; private readonly BoundedMemoryManager _memoryManager; - private int _prot; - internal UnixImplementation(MMapHandle handle, int byteOffsetIntoHandle, int elementCount) + public UnixImplementation(int elementCount, PoisonPagePlacement placement) { - _handle = handle; - _byteOffsetIntoHandle = byteOffsetIntoHandle; + _handle = MmfHandle.Allocate(checked(elementCount * (nint)sizeof(T)), placement); _elementCount = elementCount; _memoryManager = new BoundedMemoryManager(this); - _prot = UnsafeNativeMethods.PROT_NONE; } - public override bool IsReadonly => (Protection != (UnsafeNativeMethods.PROT_WRITE | UnsafeNativeMethods.PROT_READ)); + public override bool IsReadonly => false; public override int Length => _elementCount; - internal int Protection - { - get - { - return _prot; - } - set - { - bool refAdded = false; - try - { - _handle.DangerousAddRef(ref refAdded); - if (UnsafeNativeMethods.mprotect(_handle.DangerousGetHandle(), _handle.Length, value) != 0) - { - throw new InvalidOperationException($"mprotect failed with errno: {Marshal.GetLastPInvokeError()}. Length: {_handle.Length}, Value: {value}"); - } - _prot = value; - } - finally - { - if (refAdded) - { - _handle.DangerousRelease(); - } - } - } - } - public override Memory Memory => _memoryManager.Memory; public override Span Span @@ -122,7 +40,7 @@ public override Span Span try { _handle.DangerousAddRef(ref refAdded); - return new Span((void*)(_handle.DangerousGetHandle() + _byteOffsetIntoHandle), _elementCount); + return new Span((void*)_handle.DangerousGetHandle(), _elementCount); } finally { @@ -141,12 +59,12 @@ public override void Dispose() public override void MakeReadonly() { - Protection = UnsafeNativeMethods.PROT_READ; + // no-op } public override void MakeWriteable() { - Protection = UnsafeNativeMethods.PROT_WRITE | UnsafeNativeMethods.PROT_READ; + // no-op } private sealed class BoundedMemoryManager : MemoryManager @@ -178,7 +96,7 @@ public override MemoryHandle Pin(int elementIndex) try { _impl._handle.DangerousAddRef(ref refAdded); - return new MemoryHandle((T*)(_impl._handle.DangerousGetHandle() + _impl._byteOffsetIntoHandle) + elementIndex); + return new MemoryHandle((T*)_impl._handle.DangerousGetHandle() + elementIndex); } finally { @@ -196,51 +114,79 @@ public override void Unpin() } } - private sealed class MMapHandle : SafeHandle + private sealed class MmfHandle : SafeHandle { - public nuint Length { get; private set; } + private MemoryMappedFile mmf; + private MemoryMappedViewAccessor view; + private IntPtr buffer; + private ulong allocationSize; // Called by P/Invoke when returning SafeHandles - public MMapHandle(nuint length) + private MmfHandle(MemoryMappedFile mmf, MemoryMappedViewAccessor view, IntPtr buffer, ulong allocationSize) : base(IntPtr.Zero, ownsHandle: true) { - Length = length; + this.mmf = mmf; + this.view = view; + this.buffer = buffer; + this.allocationSize = allocationSize; } - internal static MMapHandle Allocate(IntPtr address, nuint length, int prot, int flags) + internal static MmfHandle Allocate(nint byteLength, PoisonPagePlacement placement) { - MMapHandle retVal = new MMapHandle(length); - retVal.SetHandle(UnsafeNativeMethods.mmap(address, length, prot, flags, -1, 0)); + // Allocate number of pages to incorporate required (byteLength bytes of) memory and an additional page to create a poison page. + int pageSize = Environment.SystemPageSize; + int allocationSize = (int)(((byteLength / pageSize) + ((byteLength % pageSize) == 0 ? 0 : 1) + 1) * pageSize); + var mmf = MemoryMappedFile.CreateNew(null, (long)allocationSize, MemoryMappedFileAccess.ReadWrite); + var view = mmf.CreateViewAccessor(); + + IntPtr buffer = view.SafeMemoryMappedViewHandle.DangerousGetHandle(); + + if (buffer == IntPtr.Zero) + { + throw new InvalidOperationException($"Memory allocation failed with error {Marshal.GetLastPInvokeError()}."); + } + + // Depending on the PoisonPagePlacement requirement (before/after) initialise the baseAddress and poisonPageAddress to point to the location + // in the buffer. Here the baseAddress points to the first valid allocation and poisonPageAddress points to the first invalid location. + // For `PoisonPagePlacement.Before` the first page is made inaccessible using mprotect and baseAddress points to the start of the second page. + // The allocation and protection is at the granularity of a page. Thus, `PoisonPagePlacement.Before` configuration has an additional accessible + // memory at the end of the page (bytes equivalent to `pageSize - (byteLength % pageSize)`). + // For `PoisonPagePlacement.After`, we adjust the baseAddress so that inaccessible memory is at the `byteLength` offset from the baseAddress. + IntPtr baseAddress = buffer + pageSize; + IntPtr poisonPageAddress = buffer; + if (placement == PoisonPagePlacement.After) + { + baseAddress = buffer + (allocationSize - pageSize - byteLength); + poisonPageAddress = buffer + (allocationSize - pageSize); + } + + // Protect the page before/after based on the poison page placement. + if (mprotect(poisonPageAddress, (ulong)pageSize, PROT_NONE) == -1) + { + throw new InvalidOperationException($"Failed to mark page as a poison page using mprotect with error :{Marshal.GetLastPInvokeError()}."); + } + + MmfHandle retVal = new MmfHandle(mmf, view, buffer, (ulong)allocationSize); + retVal.SetHandle(baseAddress); // this base address would be used as the start of Span that is used during unit testing. return retVal; } - // Do not provide a finalizer - SafeHandle's critical finalizer will - // call ReleaseHandle for you. - public override bool IsInvalid => (handle == IntPtr.Zero); - protected override bool ReleaseHandle() => - UnsafeNativeMethods.munmap(handle, Length) == 0; - } + protected override bool ReleaseHandle() + { + view.Dispose(); + mmf.Dispose(); + return true; + } - private static partial class UnsafeNativeMethods - { // Defined in - public const int MAP_PRIVATE = 0x2; public const int PROT_NONE = 0x0; public const int PROT_READ = 0x1; public const int PROT_WRITE = 0x2; - [DllImport("GetMapAnonymousFlag")] - public static extern int GetMapAnonymousFlag(); - private static class Linux { - [DllImport("libc", SetLastError = true)] - public static extern IntPtr mmap(IntPtr address, ulong length, int prot, int flags, int fd, int offset); - - [DllImport("libc", SetLastError = true)] - public static extern IntPtr munmap(IntPtr address, ulong length); [DllImport("libc", SetLastError = true)] public static extern int mprotect(IntPtr address, ulong length, int prot); @@ -248,36 +194,11 @@ private static class Linux private static class Osx { - [DllImport("libSystem", SetLastError = true)] - public static extern IntPtr mmap(IntPtr address, ulong length, int prot, int flags, int fd, int offset); - - [DllImport("libSystem", SetLastError = true)] - public static extern IntPtr munmap(IntPtr address, ulong length); [DllImport("libSystem", SetLastError = true)] public static extern int mprotect(IntPtr address, ulong length, int prot); } - public static IntPtr mmap(IntPtr address, ulong length, int prot, int flags, int fd, int offset) - { - if (OperatingSystem.IsLinux()) - { - return Linux.mmap(address, length, prot, flags, fd, offset); - } - - return Osx.mmap(address, length, prot, flags, fd, offset); - } - - public static IntPtr munmap(IntPtr address, ulong length) - { - if (OperatingSystem.IsLinux()) - { - return Linux.munmap(address, length); - } - - return Osx.munmap(address, length); - } - public static int mprotect(IntPtr address, ulong length, int prot) { if (OperatingSystem.IsLinux()) diff --git a/src/libraries/Common/tests/TestUtilities/System/Buffers/CMakeLists.txt b/src/libraries/Common/tests/TestUtilities/System/Buffers/CMakeLists.txt deleted file mode 100644 index 432219669ad6f..0000000000000 --- a/src/libraries/Common/tests/TestUtilities/System/Buffers/CMakeLists.txt +++ /dev/null @@ -1,3 +0,0 @@ -include_directories(${INC_PLATFORM_DIR}) -add_library(GetMapAnonymousFlag SHARED GetMapAnonymousFlag.cpp) -target_link_libraries(GetMapAnonymousFlag PRIVATE platformdefines) \ No newline at end of file diff --git a/src/libraries/Common/tests/TestUtilities/System/Buffers/GetMapAnonymousFlag.cpp b/src/libraries/Common/tests/TestUtilities/System/Buffers/GetMapAnonymousFlag.cpp deleted file mode 100644 index e77527fb26d09..0000000000000 --- a/src/libraries/Common/tests/TestUtilities/System/Buffers/GetMapAnonymousFlag.cpp +++ /dev/null @@ -1,17 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -#include -#include -#ifdef _WIN32 -#include -#else -#include -#endif - -#ifndef _WIN32 -extern "C" DLL_EXPORT int GetMapAnonymousFlag() -{ - return MAP_ANONYMOUS; -} -#endif \ No newline at end of file diff --git a/src/libraries/Common/tests/TestUtilities/TestUtilities.csproj b/src/libraries/Common/tests/TestUtilities/TestUtilities.csproj index fa1f2be47b285..51fc1add6ef9f 100644 --- a/src/libraries/Common/tests/TestUtilities/TestUtilities.csproj +++ b/src/libraries/Common/tests/TestUtilities/TestUtilities.csproj @@ -49,7 +49,6 @@ - diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Sve/CMakeLists.txt b/src/tests/JIT/HardwareIntrinsics/Arm/Sve/CMakeLists.txt deleted file mode 100644 index 432219669ad6f..0000000000000 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Sve/CMakeLists.txt +++ /dev/null @@ -1,3 +0,0 @@ -include_directories(${INC_PLATFORM_DIR}) -add_library(GetMapAnonymousFlag SHARED GetMapAnonymousFlag.cpp) -target_link_libraries(GetMapAnonymousFlag PRIVATE platformdefines) \ No newline at end of file diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Sve/GetMapAnonymousFlag.cpp b/src/tests/JIT/HardwareIntrinsics/Arm/Sve/GetMapAnonymousFlag.cpp deleted file mode 100644 index e77527fb26d09..0000000000000 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Sve/GetMapAnonymousFlag.cpp +++ /dev/null @@ -1,17 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -#include -#include -#ifdef _WIN32 -#include -#else -#include -#endif - -#ifndef _WIN32 -extern "C" DLL_EXPORT int GetMapAnonymousFlag() -{ - return MAP_ANONYMOUS; -} -#endif \ No newline at end of file diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Sve/Sve_r.csproj b/src/tests/JIT/HardwareIntrinsics/Arm/Sve/Sve_r.csproj index 3508f753ac897..5cecbd0119695 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Sve/Sve_r.csproj +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Sve/Sve_r.csproj @@ -16,6 +16,5 @@ - diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Sve/Sve_ro.csproj b/src/tests/JIT/HardwareIntrinsics/Arm/Sve/Sve_ro.csproj index e22009675338b..cf917acd96487 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Sve/Sve_ro.csproj +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Sve/Sve_ro.csproj @@ -16,6 +16,5 @@ - From 6081591b7ac37b6e3c1714e8f51be269ef3ecd35 Mon Sep 17 00:00:00 2001 From: TIHan Date: Mon, 29 Jul 2024 01:25:07 -0700 Subject: [PATCH 68/83] Minor cleanup --- .../TestUtilities/System/Buffers/BoundedMemory.Unix.cs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs index 18a6c432f38aa..6988554be5bf3 100644 --- a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs +++ b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs @@ -136,16 +136,11 @@ internal static MmfHandle Allocate(nint byteLength, PoisonPagePlacement placemen // Allocate number of pages to incorporate required (byteLength bytes of) memory and an additional page to create a poison page. int pageSize = Environment.SystemPageSize; int allocationSize = (int)(((byteLength / pageSize) + ((byteLength % pageSize) == 0 ? 0 : 1) + 1) * pageSize); + var mmf = MemoryMappedFile.CreateNew(null, (long)allocationSize, MemoryMappedFileAccess.ReadWrite); var view = mmf.CreateViewAccessor(); - IntPtr buffer = view.SafeMemoryMappedViewHandle.DangerousGetHandle(); - if (buffer == IntPtr.Zero) - { - throw new InvalidOperationException($"Memory allocation failed with error {Marshal.GetLastPInvokeError()}."); - } - // Depending on the PoisonPagePlacement requirement (before/after) initialise the baseAddress and poisonPageAddress to point to the location // in the buffer. Here the baseAddress points to the first valid allocation and poisonPageAddress points to the first invalid location. // For `PoisonPagePlacement.Before` the first page is made inaccessible using mprotect and baseAddress points to the start of the second page. From 7e3057a1fdc685891ec934adec8e404ad6127fc5 Mon Sep 17 00:00:00 2001 From: TIHan Date: Wed, 31 Jul 2024 13:19:57 -0700 Subject: [PATCH 69/83] Added XplatVirtualAlloc, hopefully it will work --- .../Common/tests/TestUtilities/CMakeLists.txt | 3 ++ .../System/Buffers/BoundedMemory.Unix.cs | 11 +++++ .../tests/TestUtilities/TestUtilities.csproj | 1 + .../tests/TestUtilities/XplatVirtualAlloc.cpp | 46 +++++++++++++++++++ .../JIT/HardwareIntrinsics/CMakeLists.txt | 3 ++ .../HardwareIntrinsics_Arm_r.csproj | 1 + .../HardwareIntrinsics_Arm_ro.csproj | 1 + .../HardwareIntrinsics/XplatVirtualAlloc.cpp | 46 +++++++++++++++++++ 8 files changed, 112 insertions(+) create mode 100644 src/libraries/Common/tests/TestUtilities/CMakeLists.txt create mode 100644 src/libraries/Common/tests/TestUtilities/XplatVirtualAlloc.cpp create mode 100644 src/tests/JIT/HardwareIntrinsics/CMakeLists.txt create mode 100644 src/tests/JIT/HardwareIntrinsics/XplatVirtualAlloc.cpp diff --git a/src/libraries/Common/tests/TestUtilities/CMakeLists.txt b/src/libraries/Common/tests/TestUtilities/CMakeLists.txt new file mode 100644 index 0000000000000..1d13919bee32a --- /dev/null +++ b/src/libraries/Common/tests/TestUtilities/CMakeLists.txt @@ -0,0 +1,3 @@ +include_directories(${INC_PLATFORM_DIR}) +add_library(XplatVirtualAlloc SHARED XplatVirtualAlloc.cpp) +target_link_libraries(XplatVirtualAlloc PRIVATE platformdefines) diff --git a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs index 6988554be5bf3..c7871f74a7151 100644 --- a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs +++ b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs @@ -136,6 +136,10 @@ internal static MmfHandle Allocate(nint byteLength, PoisonPagePlacement placemen // Allocate number of pages to incorporate required (byteLength bytes of) memory and an additional page to create a poison page. int pageSize = Environment.SystemPageSize; int allocationSize = (int)(((byteLength / pageSize) + ((byteLength % pageSize) == 0 ? 0 : 1) + 1) * pageSize); + + // WILL REMOVE THIS, just need to test CI to make sure we can call it. + var ptr = AllocWithGuard((nuint)allocationSize); + Free(ptr, (nuint)allocationSize); var mmf = MemoryMappedFile.CreateNew(null, (long)allocationSize, MemoryMappedFileAccess.ReadWrite); var view = mmf.CreateViewAccessor(); @@ -180,6 +184,13 @@ protected override bool ReleaseHandle() public const int PROT_READ = 0x1; public const int PROT_WRITE = 0x2; + [DllImport(nameof("XplatVirtualAlloc"))] + public static extern byte* AllocWithGuard(nuint size); + + + [DllImport(nameof("XplatVirtualAlloc"))] + public static extern void Free(byte* ptr, nuint size); + private static class Linux { diff --git a/src/libraries/Common/tests/TestUtilities/TestUtilities.csproj b/src/libraries/Common/tests/TestUtilities/TestUtilities.csproj index 51fc1add6ef9f..3f249f0ff2b5c 100644 --- a/src/libraries/Common/tests/TestUtilities/TestUtilities.csproj +++ b/src/libraries/Common/tests/TestUtilities/TestUtilities.csproj @@ -49,6 +49,7 @@ + diff --git a/src/libraries/Common/tests/TestUtilities/XplatVirtualAlloc.cpp b/src/libraries/Common/tests/TestUtilities/XplatVirtualAlloc.cpp new file mode 100644 index 0000000000000..c0b1be74b6947 --- /dev/null +++ b/src/libraries/Common/tests/TestUtilities/XplatVirtualAlloc.cpp @@ -0,0 +1,46 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#include +#include +#ifdef _WIN32 +#include +#else +#include +#endif + +extern "C" DLL_EXPORT void* AllocWithGuard(uintptr_t size) +{ +#ifdef _WIN32 + void* reservePtr = VirtualAlloc(nullptr, size * 2, MEM_RESERVE, PAGE_READWRITE); + if (reservePtr != nullptr) + { + void* ptr = VirtualAlloc(reservePtr, size, MEM_COMMIT, PAGE_READWRITE); + if (ptr == nullptr) + { + VirtualFree(reservePtr, 0, MEM_RELEASE); + } + return ptr; + } +#else + uint8_t* ptr = (uint8_t*)mmap(nullptr, size * 2, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (ptr != MAP_FAILED) + { + if (mprotect(ptr + size, size, PROT_NONE) == 0) + { + return ptr; + } + munmap(ptr, size * 2); + } +#endif + return nullptr; +} + +extern "C" DLL_EXPORT void Free(void* ptr, uintptr_t size) +{ +#ifdef _WIN32 + VirtualFree(ptr, 0, MEM_RELEASE); +#else + munmap(ptr, size * 2); +#endif +} diff --git a/src/tests/JIT/HardwareIntrinsics/CMakeLists.txt b/src/tests/JIT/HardwareIntrinsics/CMakeLists.txt new file mode 100644 index 0000000000000..1d13919bee32a --- /dev/null +++ b/src/tests/JIT/HardwareIntrinsics/CMakeLists.txt @@ -0,0 +1,3 @@ +include_directories(${INC_PLATFORM_DIR}) +add_library(XplatVirtualAlloc SHARED XplatVirtualAlloc.cpp) +target_link_libraries(XplatVirtualAlloc PRIVATE platformdefines) diff --git a/src/tests/JIT/HardwareIntrinsics/HardwareIntrinsics_Arm_r.csproj b/src/tests/JIT/HardwareIntrinsics/HardwareIntrinsics_Arm_r.csproj index 05eba017a9df1..3ec255439a8a9 100644 --- a/src/tests/JIT/HardwareIntrinsics/HardwareIntrinsics_Arm_r.csproj +++ b/src/tests/JIT/HardwareIntrinsics/HardwareIntrinsics_Arm_r.csproj @@ -12,6 +12,7 @@ + diff --git a/src/tests/JIT/HardwareIntrinsics/HardwareIntrinsics_Arm_ro.csproj b/src/tests/JIT/HardwareIntrinsics/HardwareIntrinsics_Arm_ro.csproj index 356f5e1fa6ad2..5ae271fc46b0c 100644 --- a/src/tests/JIT/HardwareIntrinsics/HardwareIntrinsics_Arm_ro.csproj +++ b/src/tests/JIT/HardwareIntrinsics/HardwareIntrinsics_Arm_ro.csproj @@ -12,6 +12,7 @@ + diff --git a/src/tests/JIT/HardwareIntrinsics/XplatVirtualAlloc.cpp b/src/tests/JIT/HardwareIntrinsics/XplatVirtualAlloc.cpp new file mode 100644 index 0000000000000..c0b1be74b6947 --- /dev/null +++ b/src/tests/JIT/HardwareIntrinsics/XplatVirtualAlloc.cpp @@ -0,0 +1,46 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#include +#include +#ifdef _WIN32 +#include +#else +#include +#endif + +extern "C" DLL_EXPORT void* AllocWithGuard(uintptr_t size) +{ +#ifdef _WIN32 + void* reservePtr = VirtualAlloc(nullptr, size * 2, MEM_RESERVE, PAGE_READWRITE); + if (reservePtr != nullptr) + { + void* ptr = VirtualAlloc(reservePtr, size, MEM_COMMIT, PAGE_READWRITE); + if (ptr == nullptr) + { + VirtualFree(reservePtr, 0, MEM_RELEASE); + } + return ptr; + } +#else + uint8_t* ptr = (uint8_t*)mmap(nullptr, size * 2, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (ptr != MAP_FAILED) + { + if (mprotect(ptr + size, size, PROT_NONE) == 0) + { + return ptr; + } + munmap(ptr, size * 2); + } +#endif + return nullptr; +} + +extern "C" DLL_EXPORT void Free(void* ptr, uintptr_t size) +{ +#ifdef _WIN32 + VirtualFree(ptr, 0, MEM_RELEASE); +#else + munmap(ptr, size * 2); +#endif +} From 3b251823f8c5d5b13dd2b937f50109061644a3d9 Mon Sep 17 00:00:00 2001 From: TIHan Date: Wed, 31 Jul 2024 13:23:18 -0700 Subject: [PATCH 70/83] Fix build --- .../tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs index c7871f74a7151..c63d91148a7c4 100644 --- a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs +++ b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs @@ -184,11 +184,11 @@ protected override bool ReleaseHandle() public const int PROT_READ = 0x1; public const int PROT_WRITE = 0x2; - [DllImport(nameof("XplatVirtualAlloc"))] + [DllImport("XplatVirtualAlloc")] public static extern byte* AllocWithGuard(nuint size); - [DllImport(nameof("XplatVirtualAlloc"))] + [DllImport("XplatVirtualAlloc")] public static extern void Free(byte* ptr, nuint size); private static class Linux From 5ce97380a72c9c8e1ceb2281d5439b72a9c0acfb Mon Sep 17 00:00:00 2001 From: TIHan Date: Wed, 31 Jul 2024 14:18:20 -0700 Subject: [PATCH 71/83] Fix build --- .../JIT/HardwareIntrinsics/HardwareIntrinsics_Arm_r.csproj | 2 ++ .../JIT/HardwareIntrinsics/HardwareIntrinsics_Arm_ro.csproj | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/tests/JIT/HardwareIntrinsics/HardwareIntrinsics_Arm_r.csproj b/src/tests/JIT/HardwareIntrinsics/HardwareIntrinsics_Arm_r.csproj index 3ec255439a8a9..c77d965a6a3b4 100644 --- a/src/tests/JIT/HardwareIntrinsics/HardwareIntrinsics_Arm_r.csproj +++ b/src/tests/JIT/HardwareIntrinsics/HardwareIntrinsics_Arm_r.csproj @@ -1,5 +1,7 @@ + + true 8 true diff --git a/src/tests/JIT/HardwareIntrinsics/HardwareIntrinsics_Arm_ro.csproj b/src/tests/JIT/HardwareIntrinsics/HardwareIntrinsics_Arm_ro.csproj index 5ae271fc46b0c..478f088247d8f 100644 --- a/src/tests/JIT/HardwareIntrinsics/HardwareIntrinsics_Arm_ro.csproj +++ b/src/tests/JIT/HardwareIntrinsics/HardwareIntrinsics_Arm_ro.csproj @@ -1,5 +1,7 @@ + + true 8 true From ff0fd3965590def586dfab74c99db2af13776830 Mon Sep 17 00:00:00 2001 From: Will Smith Date: Wed, 31 Jul 2024 15:57:14 -0700 Subject: [PATCH 72/83] Update hwintrinsiccodegenarm64.cpp --- src/coreclr/jit/hwintrinsiccodegenarm64.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp index f6a490c3517d0..bc7953f34eea9 100644 --- a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp @@ -2074,7 +2074,6 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) GetEmitter()->emitIns_R(INS_sve_wrffr, emitSize, op4Reg, opt); } } - FALLTHROUGH; } case NI_Sve_GatherVector: From eda7fc480ce0cd64f230d9c50a9c523e6fefad91 Mon Sep 17 00:00:00 2001 From: TIHan Date: Wed, 31 Jul 2024 20:17:14 -0700 Subject: [PATCH 73/83] Trying to fix build --- .../ref/System.Runtime.Intrinsics.cs | 22 ------------------- 1 file changed, 22 deletions(-) diff --git a/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs b/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs index 73620653ae341..65d2c4f5e7648 100644 --- a/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs +++ b/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs @@ -5062,22 +5062,6 @@ internal Arm64() { } public static unsafe System.Numerics.Vector LoadVectorNonTemporal(System.Numerics.Vector mask, ushort* address) { throw null; } public static unsafe System.Numerics.Vector LoadVectorNonTemporal(System.Numerics.Vector mask, uint* address) { throw null; } public static unsafe System.Numerics.Vector LoadVectorNonTemporal(System.Numerics.Vector mask, ulong* address) { throw null; } - public static unsafe System.Numerics.Vector LoadVector128AndReplicateToVector(System.Numerics.Vector mask, byte* address) { throw null; } - public static unsafe System.Numerics.Vector LoadVector128AndReplicateToVector(System.Numerics.Vector mask, double* address) { throw null; } - public static unsafe System.Numerics.Vector LoadVector128AndReplicateToVector(System.Numerics.Vector mask, short* address) { throw null; } - public static unsafe System.Numerics.Vector LoadVector128AndReplicateToVector(System.Numerics.Vector mask, int* address) { throw null; } - public static unsafe System.Numerics.Vector LoadVector128AndReplicateToVector(System.Numerics.Vector mask, long* address) { throw null; } - public static unsafe System.Numerics.Vector LoadVector128AndReplicateToVector(System.Numerics.Vector mask, sbyte* address) { throw null; } - public static unsafe System.Numerics.Vector LoadVector128AndReplicateToVector(System.Numerics.Vector mask, float* address) { throw null; } - public static unsafe System.Numerics.Vector LoadVector128AndReplicateToVector(System.Numerics.Vector mask, ushort* address) { throw null; } - public static unsafe System.Numerics.Vector LoadVector128AndReplicateToVector(System.Numerics.Vector mask, uint* address) { throw null; } - public static unsafe System.Numerics.Vector LoadVector128AndReplicateToVector(System.Numerics.Vector mask, ulong* address) { throw null; } - public static unsafe System.Numerics.Vector LoadVectorByteZeroExtendToInt16(System.Numerics.Vector mask, byte* address) { throw null; } - public static unsafe System.Numerics.Vector LoadVectorByteZeroExtendToInt32(System.Numerics.Vector mask, byte* address) { throw null; } - public static unsafe System.Numerics.Vector LoadVectorByteZeroExtendToInt64(System.Numerics.Vector mask, byte* address) { throw null; } - public static unsafe System.Numerics.Vector LoadVectorByteZeroExtendToUInt16(System.Numerics.Vector mask, byte* address) { throw null; } - public static unsafe System.Numerics.Vector LoadVectorByteZeroExtendToUInt32(System.Numerics.Vector mask, byte* address) { throw null; } - public static unsafe System.Numerics.Vector LoadVectorByteZeroExtendToUInt64(System.Numerics.Vector mask, byte* address) { throw null; } public static unsafe System.Numerics.Vector LoadVectorFirstFaulting(System.Numerics.Vector mask, byte* address) { throw null; } public static unsafe System.Numerics.Vector LoadVectorFirstFaulting(System.Numerics.Vector mask, double* address) { throw null; } @@ -5090,12 +5074,6 @@ internal Arm64() { } public static unsafe System.Numerics.Vector LoadVectorFirstFaulting(System.Numerics.Vector mask, uint* address) { throw null; } public static unsafe System.Numerics.Vector LoadVectorFirstFaulting(System.Numerics.Vector mask, ulong* address) { throw null; } - public static unsafe System.Numerics.Vector LoadVectorInt16SignExtendToInt32(System.Numerics.Vector mask, short* address) { throw null; } - public static unsafe System.Numerics.Vector LoadVectorInt16SignExtendToInt64(System.Numerics.Vector mask, short* address) { throw null; } - public static unsafe System.Numerics.Vector LoadVectorInt16SignExtendToUInt32(System.Numerics.Vector mask, short* address) { throw null; } - public static unsafe System.Numerics.Vector LoadVectorInt16SignExtendToUInt64(System.Numerics.Vector mask, short* address) { throw null; } - public static unsafe System.Numerics.Vector LoadVectorInt32SignExtendToInt64(System.Numerics.Vector mask, int* address) { throw null; } - public static unsafe System.Numerics.Vector LoadVectorInt32SignExtendToUInt64(System.Numerics.Vector mask, int* address) { throw null; } public static unsafe System.Numerics.Vector LoadVectorSByteNonFaultingSignExtendToInt16(sbyte* address) { throw null; } public static unsafe System.Numerics.Vector LoadVectorSByteNonFaultingSignExtendToInt32(sbyte* address) { throw null; } public static unsafe System.Numerics.Vector LoadVectorSByteNonFaultingSignExtendToInt64(sbyte* address) { throw null; } From 2d71632728b84c242347036ce984a22ca79a5594 Mon Sep 17 00:00:00 2001 From: TIHan Date: Thu, 1 Aug 2024 10:53:36 -0700 Subject: [PATCH 74/83] Use SystemNative --- .../Common/tests/TestUtilities/CMakeLists.txt | 3 - .../System/Buffers/BoundedMemory.Creation.cs | 1 + .../System/Buffers/BoundedMemory.Unix.cs | 84 ++++++++----------- .../tests/TestUtilities/TestUtilities.csproj | 1 - .../tests/TestUtilities/XplatVirtualAlloc.cpp | 46 ---------- .../JIT/HardwareIntrinsics/CMakeLists.txt | 3 - .../HardwareIntrinsics_Arm_r.csproj | 3 - .../HardwareIntrinsics_Arm_ro.csproj | 3 - .../HardwareIntrinsics/XplatVirtualAlloc.cpp | 46 ---------- 9 files changed, 36 insertions(+), 154 deletions(-) delete mode 100644 src/libraries/Common/tests/TestUtilities/CMakeLists.txt delete mode 100644 src/libraries/Common/tests/TestUtilities/XplatVirtualAlloc.cpp delete mode 100644 src/tests/JIT/HardwareIntrinsics/CMakeLists.txt delete mode 100644 src/tests/JIT/HardwareIntrinsics/XplatVirtualAlloc.cpp diff --git a/src/libraries/Common/tests/TestUtilities/CMakeLists.txt b/src/libraries/Common/tests/TestUtilities/CMakeLists.txt deleted file mode 100644 index 1d13919bee32a..0000000000000 --- a/src/libraries/Common/tests/TestUtilities/CMakeLists.txt +++ /dev/null @@ -1,3 +0,0 @@ -include_directories(${INC_PLATFORM_DIR}) -add_library(XplatVirtualAlloc SHARED XplatVirtualAlloc.cpp) -target_link_libraries(XplatVirtualAlloc PRIVATE platformdefines) diff --git a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Creation.cs b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Creation.cs index cec236105aafa..78bfa92dad969 100644 --- a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Creation.cs +++ b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Creation.cs @@ -10,6 +10,7 @@ namespace System.Buffers /// public static partial class BoundedMemory { + public static bool UnixBoundsEnabled { get; set; } private static readonly int SystemPageSize = Environment.SystemPageSize; /// diff --git a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs index c63d91148a7c4..76482db404d32 100644 --- a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs +++ b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using Microsoft.Win32.SafeHandles; using System.IO.MemoryMappedFiles; using System.Runtime.InteropServices; @@ -15,13 +16,13 @@ private static UnixImplementation AllocateWithoutDataPopulationUnix(int el private sealed class UnixImplementation : BoundedMemory where T : unmanaged { - private readonly MmfHandle _handle; + private readonly MMapHandle _handle; private readonly int _elementCount; private readonly BoundedMemoryManager _memoryManager; public UnixImplementation(int elementCount, PoisonPagePlacement placement) { - _handle = MmfHandle.Allocate(checked(elementCount * (nint)sizeof(T)), placement); + _handle = MMapHandle.Allocate(checked(elementCount * (nint)sizeof(T)), placement); _elementCount = elementCount; _memoryManager = new BoundedMemoryManager(this); } @@ -114,37 +115,27 @@ public override void Unpin() } } - private sealed class MmfHandle : SafeHandle + private sealed class MMapHandle : SafeHandle { - private MemoryMappedFile mmf; - private MemoryMappedViewAccessor view; private IntPtr buffer; private ulong allocationSize; // Called by P/Invoke when returning SafeHandles - private MmfHandle(MemoryMappedFile mmf, MemoryMappedViewAccessor view, IntPtr buffer, ulong allocationSize) + private MMapHandle(IntPtr buffer, ulong allocationSize) : base(IntPtr.Zero, ownsHandle: true) { - this.mmf = mmf; - this.view = view; this.buffer = buffer; this.allocationSize = allocationSize; } - internal static MmfHandle Allocate(nint byteLength, PoisonPagePlacement placement) + internal static MMapHandle Allocate(nint byteLength, PoisonPagePlacement placement) { // Allocate number of pages to incorporate required (byteLength bytes of) memory and an additional page to create a poison page. int pageSize = Environment.SystemPageSize; int allocationSize = (int)(((byteLength / pageSize) + ((byteLength % pageSize) == 0 ? 0 : 1) + 1) * pageSize); + IntPtr buffer = MMap(0, (ulong)allocationSize, MemoryMappedProtections.PROT_READ | MemoryMappedProtections.PROT_WRITE, MemoryMappedFlags.MAP_PRIVATE | MemoryMappedFlags.MAP_ANONYMOUS, -1, 0); - // WILL REMOVE THIS, just need to test CI to make sure we can call it. - var ptr = AllocWithGuard((nuint)allocationSize); - Free(ptr, (nuint)allocationSize); - - var mmf = MemoryMappedFile.CreateNew(null, (long)allocationSize, MemoryMappedFileAccess.ReadWrite); - var view = mmf.CreateViewAccessor(); - IntPtr buffer = view.SafeMemoryMappedViewHandle.DangerousGetHandle(); - + Console.WriteLine(buffer); // Depending on the PoisonPagePlacement requirement (before/after) initialise the baseAddress and poisonPageAddress to point to the location // in the buffer. Here the baseAddress points to the first valid allocation and poisonPageAddress points to the first invalid location. // For `PoisonPagePlacement.Before` the first page is made inaccessible using mprotect and baseAddress points to the start of the second page. @@ -160,12 +151,12 @@ internal static MmfHandle Allocate(nint byteLength, PoisonPagePlacement placemen } // Protect the page before/after based on the poison page placement. - if (mprotect(poisonPageAddress, (ulong)pageSize, PROT_NONE) == -1) + if (MProtect(poisonPageAddress, (ulong)pageSize, MemoryMappedProtections.PROT_NONE) == -1) { throw new InvalidOperationException($"Failed to mark page as a poison page using mprotect with error :{Marshal.GetLastPInvokeError()}."); } - MmfHandle retVal = new MmfHandle(mmf, view, buffer, (ulong)allocationSize); + MMapHandle retVal = new MMapHandle(buffer, (ulong)allocationSize); retVal.SetHandle(baseAddress); // this base address would be used as the start of Span that is used during unit testing. return retVal; } @@ -174,46 +165,41 @@ internal static MmfHandle Allocate(nint byteLength, PoisonPagePlacement placemen protected override bool ReleaseHandle() { - view.Dispose(); - mmf.Dispose(); + MUnmap(buffer, allocationSize); return true; } - // Defined in - public const int PROT_NONE = 0x0; - public const int PROT_READ = 0x1; - public const int PROT_WRITE = 0x2; - - [DllImport("XplatVirtualAlloc")] - public static extern byte* AllocWithGuard(nuint size); - - - [DllImport("XplatVirtualAlloc")] - public static extern void Free(byte* ptr, nuint size); - - private static class Linux + [Flags] + private enum MemoryMappedProtections { - - [DllImport("libc", SetLastError = true)] - public static extern int mprotect(IntPtr address, ulong length, int prot); + PROT_NONE = 0x0, + PROT_READ = 0x1, + PROT_WRITE = 0x2, + PROT_EXEC = 0x4 } - private static class Osx + [Flags] + private enum MemoryMappedFlags { - - [DllImport("libSystem", SetLastError = true)] - public static extern int mprotect(IntPtr address, ulong length, int prot); + MAP_SHARED = 0x01, + MAP_PRIVATE = 0x02, + MAP_ANONYMOUS = 0x10, } - public static int mprotect(IntPtr address, ulong length, int prot) - { - if (OperatingSystem.IsLinux()) - { - return Linux.mprotect(address, length, prot); - } + private const string SystemNative = "libSystem.Native"; - return Osx.mprotect(address, length, prot); - } + // NOTE: Shim returns null pointer on failure, not non-null MAP_FAILED sentinel. + [DllImport(SystemNative, EntryPoint = "SystemNative_MMap", SetLastError = true)] + private static extern IntPtr MMap( + IntPtr addr, ulong len, + MemoryMappedProtections prot, MemoryMappedFlags flags, + IntPtr fd, long offset); + + [DllImport(SystemNative, EntryPoint = "SystemNative_MProtect", SetLastError = true)] + private static extern int MProtect(IntPtr addr, ulong len, MemoryMappedProtections prot); + + [DllImport(SystemNative, EntryPoint = "SystemNative_MUnmap", SetLastError = true)] + internal static extern int MUnmap(IntPtr addr, ulong len); } } } diff --git a/src/libraries/Common/tests/TestUtilities/TestUtilities.csproj b/src/libraries/Common/tests/TestUtilities/TestUtilities.csproj index 3f249f0ff2b5c..51fc1add6ef9f 100644 --- a/src/libraries/Common/tests/TestUtilities/TestUtilities.csproj +++ b/src/libraries/Common/tests/TestUtilities/TestUtilities.csproj @@ -49,7 +49,6 @@ - diff --git a/src/libraries/Common/tests/TestUtilities/XplatVirtualAlloc.cpp b/src/libraries/Common/tests/TestUtilities/XplatVirtualAlloc.cpp deleted file mode 100644 index c0b1be74b6947..0000000000000 --- a/src/libraries/Common/tests/TestUtilities/XplatVirtualAlloc.cpp +++ /dev/null @@ -1,46 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -#include -#include -#ifdef _WIN32 -#include -#else -#include -#endif - -extern "C" DLL_EXPORT void* AllocWithGuard(uintptr_t size) -{ -#ifdef _WIN32 - void* reservePtr = VirtualAlloc(nullptr, size * 2, MEM_RESERVE, PAGE_READWRITE); - if (reservePtr != nullptr) - { - void* ptr = VirtualAlloc(reservePtr, size, MEM_COMMIT, PAGE_READWRITE); - if (ptr == nullptr) - { - VirtualFree(reservePtr, 0, MEM_RELEASE); - } - return ptr; - } -#else - uint8_t* ptr = (uint8_t*)mmap(nullptr, size * 2, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - if (ptr != MAP_FAILED) - { - if (mprotect(ptr + size, size, PROT_NONE) == 0) - { - return ptr; - } - munmap(ptr, size * 2); - } -#endif - return nullptr; -} - -extern "C" DLL_EXPORT void Free(void* ptr, uintptr_t size) -{ -#ifdef _WIN32 - VirtualFree(ptr, 0, MEM_RELEASE); -#else - munmap(ptr, size * 2); -#endif -} diff --git a/src/tests/JIT/HardwareIntrinsics/CMakeLists.txt b/src/tests/JIT/HardwareIntrinsics/CMakeLists.txt deleted file mode 100644 index 1d13919bee32a..0000000000000 --- a/src/tests/JIT/HardwareIntrinsics/CMakeLists.txt +++ /dev/null @@ -1,3 +0,0 @@ -include_directories(${INC_PLATFORM_DIR}) -add_library(XplatVirtualAlloc SHARED XplatVirtualAlloc.cpp) -target_link_libraries(XplatVirtualAlloc PRIVATE platformdefines) diff --git a/src/tests/JIT/HardwareIntrinsics/HardwareIntrinsics_Arm_r.csproj b/src/tests/JIT/HardwareIntrinsics/HardwareIntrinsics_Arm_r.csproj index c77d965a6a3b4..05eba017a9df1 100644 --- a/src/tests/JIT/HardwareIntrinsics/HardwareIntrinsics_Arm_r.csproj +++ b/src/tests/JIT/HardwareIntrinsics/HardwareIntrinsics_Arm_r.csproj @@ -1,7 +1,5 @@ - - true 8 true @@ -14,7 +12,6 @@ - diff --git a/src/tests/JIT/HardwareIntrinsics/HardwareIntrinsics_Arm_ro.csproj b/src/tests/JIT/HardwareIntrinsics/HardwareIntrinsics_Arm_ro.csproj index 478f088247d8f..356f5e1fa6ad2 100644 --- a/src/tests/JIT/HardwareIntrinsics/HardwareIntrinsics_Arm_ro.csproj +++ b/src/tests/JIT/HardwareIntrinsics/HardwareIntrinsics_Arm_ro.csproj @@ -1,7 +1,5 @@ - - true 8 true @@ -14,7 +12,6 @@ - diff --git a/src/tests/JIT/HardwareIntrinsics/XplatVirtualAlloc.cpp b/src/tests/JIT/HardwareIntrinsics/XplatVirtualAlloc.cpp deleted file mode 100644 index c0b1be74b6947..0000000000000 --- a/src/tests/JIT/HardwareIntrinsics/XplatVirtualAlloc.cpp +++ /dev/null @@ -1,46 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -#include -#include -#ifdef _WIN32 -#include -#else -#include -#endif - -extern "C" DLL_EXPORT void* AllocWithGuard(uintptr_t size) -{ -#ifdef _WIN32 - void* reservePtr = VirtualAlloc(nullptr, size * 2, MEM_RESERVE, PAGE_READWRITE); - if (reservePtr != nullptr) - { - void* ptr = VirtualAlloc(reservePtr, size, MEM_COMMIT, PAGE_READWRITE); - if (ptr == nullptr) - { - VirtualFree(reservePtr, 0, MEM_RELEASE); - } - return ptr; - } -#else - uint8_t* ptr = (uint8_t*)mmap(nullptr, size * 2, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - if (ptr != MAP_FAILED) - { - if (mprotect(ptr + size, size, PROT_NONE) == 0) - { - return ptr; - } - munmap(ptr, size * 2); - } -#endif - return nullptr; -} - -extern "C" DLL_EXPORT void Free(void* ptr, uintptr_t size) -{ -#ifdef _WIN32 - VirtualFree(ptr, 0, MEM_RELEASE); -#else - munmap(ptr, size * 2); -#endif -} From 929b13bc26ecd228f5da1224002a0970d2e520b1 Mon Sep 17 00:00:00 2001 From: TIHan Date: Thu, 1 Aug 2024 10:56:19 -0700 Subject: [PATCH 75/83] fix pinvoke --- .../System/Buffers/BoundedMemory.Unix.cs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs index 76482db404d32..681224fb4b732 100644 --- a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs +++ b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs @@ -133,7 +133,7 @@ internal static MMapHandle Allocate(nint byteLength, PoisonPagePlacement placeme // Allocate number of pages to incorporate required (byteLength bytes of) memory and an additional page to create a poison page. int pageSize = Environment.SystemPageSize; int allocationSize = (int)(((byteLength / pageSize) + ((byteLength % pageSize) == 0 ? 0 : 1) + 1) * pageSize); - IntPtr buffer = MMap(0, (ulong)allocationSize, MemoryMappedProtections.PROT_READ | MemoryMappedProtections.PROT_WRITE, MemoryMappedFlags.MAP_PRIVATE | MemoryMappedFlags.MAP_ANONYMOUS, -1, 0); + IntPtr buffer = MMap(0, (ulong)allocationSize, (int)(MemoryMappedProtections.PROT_READ | MemoryMappedProtections.PROT_WRITE), (int)(MemoryMappedFlags.MAP_PRIVATE | MemoryMappedFlags.MAP_ANONYMOUS), -1, 0); Console.WriteLine(buffer); // Depending on the PoisonPagePlacement requirement (before/after) initialise the baseAddress and poisonPageAddress to point to the location @@ -151,7 +151,7 @@ internal static MMapHandle Allocate(nint byteLength, PoisonPagePlacement placeme } // Protect the page before/after based on the poison page placement. - if (MProtect(poisonPageAddress, (ulong)pageSize, MemoryMappedProtections.PROT_NONE) == -1) + if (MProtect(poisonPageAddress, (ulong)pageSize, (int)MemoryMappedProtections.PROT_NONE) == -1) { throw new InvalidOperationException($"Failed to mark page as a poison page using mprotect with error :{Marshal.GetLastPInvokeError()}."); } @@ -190,13 +190,10 @@ private enum MemoryMappedFlags // NOTE: Shim returns null pointer on failure, not non-null MAP_FAILED sentinel. [DllImport(SystemNative, EntryPoint = "SystemNative_MMap", SetLastError = true)] - private static extern IntPtr MMap( - IntPtr addr, ulong len, - MemoryMappedProtections prot, MemoryMappedFlags flags, - IntPtr fd, long offset); + private static extern IntPtr MMap(IntPtr addr, ulong len, int prot, int flags, IntPtr fd, long offset); [DllImport(SystemNative, EntryPoint = "SystemNative_MProtect", SetLastError = true)] - private static extern int MProtect(IntPtr addr, ulong len, MemoryMappedProtections prot); + private static extern int MProtect(IntPtr addr, ulong len, int prot); [DllImport(SystemNative, EntryPoint = "SystemNative_MUnmap", SetLastError = true)] internal static extern int MUnmap(IntPtr addr, ulong len); From 0a327c6a61c7b80d4797e1010ced7eedb18de727 Mon Sep 17 00:00:00 2001 From: TIHan Date: Thu, 1 Aug 2024 10:56:45 -0700 Subject: [PATCH 76/83] Get rid of writeline --- .../tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs index 681224fb4b732..dfcc742f0ec42 100644 --- a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs +++ b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs @@ -135,7 +135,6 @@ internal static MMapHandle Allocate(nint byteLength, PoisonPagePlacement placeme int allocationSize = (int)(((byteLength / pageSize) + ((byteLength % pageSize) == 0 ? 0 : 1) + 1) * pageSize); IntPtr buffer = MMap(0, (ulong)allocationSize, (int)(MemoryMappedProtections.PROT_READ | MemoryMappedProtections.PROT_WRITE), (int)(MemoryMappedFlags.MAP_PRIVATE | MemoryMappedFlags.MAP_ANONYMOUS), -1, 0); - Console.WriteLine(buffer); // Depending on the PoisonPagePlacement requirement (before/after) initialise the baseAddress and poisonPageAddress to point to the location // in the buffer. Here the baseAddress points to the first valid allocation and poisonPageAddress points to the first invalid location. // For `PoisonPagePlacement.Before` the first page is made inaccessible using mprotect and baseAddress points to the start of the second page. From e716ae57a97b2f51790f5d6b927bc026059418bc Mon Sep 17 00:00:00 2001 From: TIHan Date: Thu, 1 Aug 2024 16:22:40 -0700 Subject: [PATCH 77/83] Add mono check --- .../TestUtilities/System/Buffers/BoundedMemory.Creation.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Creation.cs b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Creation.cs index 78bfa92dad969..05d91e7419a89 100644 --- a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Creation.cs +++ b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Creation.cs @@ -66,6 +66,8 @@ public static BoundedMemory AllocateFromExistingData(T[] data, PoisonPageP return AllocateFromExistingData(new ReadOnlySpan(data), placement); } + private static bool IsMonoRuntime => Type.GetType("Mono.RuntimeStructs") != null; + private static void FillRandom(Span buffer) { // Loop over a Random instance manually since Random.NextBytes(Span) doesn't @@ -85,8 +87,9 @@ private static BoundedMemory AllocateWithoutDataPopulation(int elementCoun { return AllocateWithoutDataPopulationWindows(elementCount, placement); } - else if (RuntimeInformation.IsOSPlatform(OSPlatform.FreeBSD) || RuntimeInformation.IsOSPlatform(OSPlatform.OSX) || RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + else if (!IsMonoRuntime && (RuntimeInformation.IsOSPlatform(OSPlatform.FreeBSD) || RuntimeInformation.IsOSPlatform(OSPlatform.OSX) || RuntimeInformation.IsOSPlatform(OSPlatform.Linux))) { + // Not supported for mono as the implementtion uses SystemNative from CoreClr. return AllocateWithoutDataPopulationUnix(elementCount, placement); } else From 028b2c71c3548360999bc9d281a6004279ea4741 Mon Sep 17 00:00:00 2001 From: TIHan Date: Thu, 1 Aug 2024 16:24:35 -0700 Subject: [PATCH 78/83] Fix misspelled word --- .../TestUtilities/System/Buffers/BoundedMemory.Creation.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Creation.cs b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Creation.cs index 05d91e7419a89..b9b0251f21e66 100644 --- a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Creation.cs +++ b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Creation.cs @@ -89,7 +89,7 @@ private static BoundedMemory AllocateWithoutDataPopulation(int elementCoun } else if (!IsMonoRuntime && (RuntimeInformation.IsOSPlatform(OSPlatform.FreeBSD) || RuntimeInformation.IsOSPlatform(OSPlatform.OSX) || RuntimeInformation.IsOSPlatform(OSPlatform.Linux))) { - // Not supported for mono as the implementtion uses SystemNative from CoreClr. + // Not supported for mono as the implementation uses SystemNative from CoreClr. return AllocateWithoutDataPopulationUnix(elementCount, placement); } else From 2b5215b44e6f38b37c0fbc389de620d9df0c0e87 Mon Sep 17 00:00:00 2001 From: TIHan Date: Thu, 1 Aug 2024 16:52:54 -0700 Subject: [PATCH 79/83] Check for wasm --- .../System/Buffers/BoundedMemory.Creation.cs | 10 +++++----- .../TestUtilities/System/Buffers/BoundedMemory.Unix.cs | 7 ++++--- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Creation.cs b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Creation.cs index b9b0251f21e66..3711164689f77 100644 --- a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Creation.cs +++ b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Creation.cs @@ -66,7 +66,8 @@ public static BoundedMemory AllocateFromExistingData(T[] data, PoisonPageP return AllocateFromExistingData(new ReadOnlySpan(data), placement); } - private static bool IsMonoRuntime => Type.GetType("Mono.RuntimeStructs") != null; + private static bool IsBrowser => RuntimeInformation.IsOSPlatform(OSPlatform.Create("BROWSER")); + private static bool IsWasi => RuntimeInformation.IsOSPlatform(OSPlatform.Create("WASI")); private static void FillRandom(Span buffer) { @@ -87,14 +88,13 @@ private static BoundedMemory AllocateWithoutDataPopulation(int elementCoun { return AllocateWithoutDataPopulationWindows(elementCount, placement); } - else if (!IsMonoRuntime && (RuntimeInformation.IsOSPlatform(OSPlatform.FreeBSD) || RuntimeInformation.IsOSPlatform(OSPlatform.OSX) || RuntimeInformation.IsOSPlatform(OSPlatform.Linux))) + else if (IsWasi || IsBrowser) { - // Not supported for mono as the implementation uses SystemNative from CoreClr. - return AllocateWithoutDataPopulationUnix(elementCount, placement); + return AllocateWithoutDataPopulationDefault(elementCount, placement); } else { - return AllocateWithoutDataPopulationDefault(elementCount, placement); + return AllocateWithoutDataPopulationUnix(elementCount, placement); } } } diff --git a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs index dfcc742f0ec42..6e819c7483575 100644 --- a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs +++ b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System; using Microsoft.Win32.SafeHandles; using System.IO.MemoryMappedFiles; using System.Runtime.InteropServices; @@ -188,13 +189,13 @@ private enum MemoryMappedFlags private const string SystemNative = "libSystem.Native"; // NOTE: Shim returns null pointer on failure, not non-null MAP_FAILED sentinel. - [DllImport(SystemNative, EntryPoint = "SystemNative_MMap", SetLastError = true)] + [DllImport(Interop.Libraries.SystemNative, EntryPoint = "SystemNative_MMap", SetLastError = true)] private static extern IntPtr MMap(IntPtr addr, ulong len, int prot, int flags, IntPtr fd, long offset); - [DllImport(SystemNative, EntryPoint = "SystemNative_MProtect", SetLastError = true)] + [DllImport(Interop.Libraries.SystemNative, EntryPoint = "SystemNative_MProtect", SetLastError = true)] private static extern int MProtect(IntPtr addr, ulong len, int prot); - [DllImport(SystemNative, EntryPoint = "SystemNative_MUnmap", SetLastError = true)] + [DllImport(Interop.Libraries.SystemNative, EntryPoint = "SystemNative_MUnmap", SetLastError = true)] internal static extern int MUnmap(IntPtr addr, ulong len); } } From 8924c6c8f22733efbc4537787ccbb8f25fc7ef01 Mon Sep 17 00:00:00 2001 From: TIHan Date: Thu, 1 Aug 2024 16:55:13 -0700 Subject: [PATCH 80/83] Fix build --- .../TestUtilities/System/Buffers/BoundedMemory.Unix.cs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs index 6e819c7483575..dfcc742f0ec42 100644 --- a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs +++ b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Unix.cs @@ -1,7 +1,6 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System; using Microsoft.Win32.SafeHandles; using System.IO.MemoryMappedFiles; using System.Runtime.InteropServices; @@ -189,13 +188,13 @@ private enum MemoryMappedFlags private const string SystemNative = "libSystem.Native"; // NOTE: Shim returns null pointer on failure, not non-null MAP_FAILED sentinel. - [DllImport(Interop.Libraries.SystemNative, EntryPoint = "SystemNative_MMap", SetLastError = true)] + [DllImport(SystemNative, EntryPoint = "SystemNative_MMap", SetLastError = true)] private static extern IntPtr MMap(IntPtr addr, ulong len, int prot, int flags, IntPtr fd, long offset); - [DllImport(Interop.Libraries.SystemNative, EntryPoint = "SystemNative_MProtect", SetLastError = true)] + [DllImport(SystemNative, EntryPoint = "SystemNative_MProtect", SetLastError = true)] private static extern int MProtect(IntPtr addr, ulong len, int prot); - [DllImport(Interop.Libraries.SystemNative, EntryPoint = "SystemNative_MUnmap", SetLastError = true)] + [DllImport(SystemNative, EntryPoint = "SystemNative_MUnmap", SetLastError = true)] internal static extern int MUnmap(IntPtr addr, ulong len); } } From 5e53a9734319e03665c6b7dee6585b5059e88ccf Mon Sep 17 00:00:00 2001 From: TIHan Date: Thu, 1 Aug 2024 16:59:11 -0700 Subject: [PATCH 81/83] Use IsBrowser --- .../TestUtilities/System/Buffers/BoundedMemory.Creation.cs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Creation.cs b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Creation.cs index 3711164689f77..303b7d989132c 100644 --- a/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Creation.cs +++ b/src/libraries/Common/tests/TestUtilities/System/Buffers/BoundedMemory.Creation.cs @@ -66,9 +66,6 @@ public static BoundedMemory AllocateFromExistingData(T[] data, PoisonPageP return AllocateFromExistingData(new ReadOnlySpan(data), placement); } - private static bool IsBrowser => RuntimeInformation.IsOSPlatform(OSPlatform.Create("BROWSER")); - private static bool IsWasi => RuntimeInformation.IsOSPlatform(OSPlatform.Create("WASI")); - private static void FillRandom(Span buffer) { // Loop over a Random instance manually since Random.NextBytes(Span) doesn't @@ -88,7 +85,7 @@ private static BoundedMemory AllocateWithoutDataPopulation(int elementCoun { return AllocateWithoutDataPopulationWindows(elementCount, placement); } - else if (IsWasi || IsBrowser) + else if (OperatingSystem.IsBrowser()) { return AllocateWithoutDataPopulationDefault(elementCount, placement); } From badf7884ce8f4742cffeb4c4fff2a1ee6cb2cd1b Mon Sep 17 00:00:00 2001 From: Mikhail Ablakatov Date: Tue, 6 Aug 2024 00:34:34 +0000 Subject: [PATCH 82/83] fixup: various fixes after merging with main --- src/coreclr/jit/hwintrinsiclistarm64sve.h | 12 ++++++------ src/coreclr/jit/lowerarmarch.cpp | 12 ++++++++++++ src/coreclr/jit/lsraarm64.cpp | 6 ------ 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/src/coreclr/jit/hwintrinsiclistarm64sve.h b/src/coreclr/jit/hwintrinsiclistarm64sve.h index 901f20947bd4d..19fb5e49a1dab 100644 --- a/src/coreclr/jit/hwintrinsiclistarm64sve.h +++ b/src/coreclr/jit/hwintrinsiclistarm64sve.h @@ -145,7 +145,7 @@ HARDWARE_INTRINSIC(Sve, LoadVectorByteNonFaultingZeroExtendToInt64, HARDWARE_INTRINSIC(Sve, LoadVectorByteNonFaultingZeroExtendToUInt16, -1, 1, {INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorByteNonFaultingZeroExtendToUInt32, -1, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorByteNonFaultingZeroExtendToUInt64, -1, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1b, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendFirstFaulting, -1, 2, {INS_invalid, INS_invalid, INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1b, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendFirstFaulting, -1, -1, {INS_invalid, INS_invalid, INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1b, INS_sve_ldff1b, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialSideEffectMask) HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendToInt16, -1, 2, {INS_invalid, INS_invalid, INS_sve_ld1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendToInt32, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorByteZeroExtendToInt64, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1b, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) @@ -157,14 +157,14 @@ HARDWARE_INTRINSIC(Sve, LoadVectorInt16NonFaultingSignExtendToInt32, HARDWARE_INTRINSIC(Sve, LoadVectorInt16NonFaultingSignExtendToInt64, -1, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sh, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorInt16NonFaultingSignExtendToUInt32, -1, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sh, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorInt16NonFaultingSignExtendToUInt64, -1, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sh, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorInt16SignExtendFirstFaulting, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1sh, INS_sve_ldff1sh, INS_sve_ldff1sh, INS_sve_ldff1sh, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, LoadVectorInt16SignExtendFirstFaulting, -1, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1sh, INS_sve_ldff1sh, INS_sve_ldff1sh, INS_sve_ldff1sh, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialSideEffectMask) HARDWARE_INTRINSIC(Sve, LoadVectorInt16SignExtendToInt32, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorInt16SignExtendToInt64, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sh, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorInt16SignExtendToUInt32, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sh, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorInt16SignExtendToUInt64, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sh, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorInt32NonFaultingSignExtendToInt64, -1, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sw, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorInt32NonFaultingSignExtendToUInt64, -1, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sw, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorInt32SignExtendFirstFaulting, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1sw, INS_sve_ldff1sw, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, LoadVectorInt32SignExtendFirstFaulting, -1, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1sw, INS_sve_ldff1sw, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialSideEffectMask) HARDWARE_INTRINSIC(Sve, LoadVectorInt32SignExtendToInt64, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sw, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorInt32SignExtendToUInt64, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sw, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorNonFaulting, -1, 1, {INS_sve_ldnf1b, INS_sve_ldnf1b, INS_sve_ldnf1h, INS_sve_ldnf1h, INS_sve_ldnf1w, INS_sve_ldnf1w, INS_sve_ldnf1d, INS_sve_ldnf1d, INS_sve_ldnf1w, INS_sve_ldnf1d}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) @@ -175,7 +175,7 @@ HARDWARE_INTRINSIC(Sve, LoadVectorSByteNonFaultingSignExtendToInt64, HARDWARE_INTRINSIC(Sve, LoadVectorSByteNonFaultingSignExtendToUInt16, -1, -1, {INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorSByteNonFaultingSignExtendToUInt32, -1, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorSByteNonFaultingSignExtendToUInt64, -1, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1sb, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorSByteSignExtendFirstFaulting, -1, 2, {INS_invalid, INS_invalid, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, LoadVectorSByteSignExtendFirstFaulting, -1, -1, {INS_invalid, INS_invalid, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_sve_ldff1sb, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialSideEffectMask) HARDWARE_INTRINSIC(Sve, LoadVectorSByteSignExtendToInt16, -1, 2, {INS_invalid, INS_invalid, INS_sve_ld1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorSByteSignExtendToInt32, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorSByteSignExtendToInt64, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1sb, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) @@ -186,14 +186,14 @@ HARDWARE_INTRINSIC(Sve, LoadVectorUInt16NonFaultingZeroExtendToInt32, HARDWARE_INTRINSIC(Sve, LoadVectorUInt16NonFaultingZeroExtendToInt64, -1, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1h, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorUInt16NonFaultingZeroExtendToUInt32, -1, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1h, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorUInt16NonFaultingZeroExtendToUInt64, -1, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1h, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorUInt16ZeroExtendFirstFaulting, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1h, INS_sve_ldff1h, INS_sve_ldff1h, INS_sve_ldff1h, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, LoadVectorUInt16ZeroExtendFirstFaulting, -1, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1h, INS_sve_ldff1h, INS_sve_ldff1h, INS_sve_ldff1h, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialSideEffectMask) HARDWARE_INTRINSIC(Sve, LoadVectorUInt16ZeroExtendToInt32, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1h, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorUInt16ZeroExtendToInt64, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1h, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorUInt16ZeroExtendToUInt32, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1h, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorUInt16ZeroExtendToUInt64, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1h, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorUInt32NonFaultingZeroExtendToInt64, -1, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1w, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorUInt32NonFaultingZeroExtendToUInt64, -1, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldnf1w, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) -HARDWARE_INTRINSIC(Sve, LoadVectorUInt32ZeroExtendFirstFaulting, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1w, INS_sve_ldff1w, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, LoadVectorUInt32ZeroExtendFirstFaulting, -1, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ldff1w, INS_sve_ldff1w, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialSideEffectMask) HARDWARE_INTRINSIC(Sve, LoadVectorUInt32ZeroExtendToInt64, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, LoadVectorUInt32ZeroExtendToUInt64, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ld1w, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, Max, -1, -1, {INS_sve_smax, INS_sve_umax, INS_sve_smax, INS_sve_umax, INS_sve_smax, INS_sve_umax, INS_sve_smax, INS_sve_umax, INS_sve_fmax, INS_sve_fmax}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) diff --git a/src/coreclr/jit/lowerarmarch.cpp b/src/coreclr/jit/lowerarmarch.cpp index c68034f6cd61e..bf8cf2261f20c 100644 --- a/src/coreclr/jit/lowerarmarch.cpp +++ b/src/coreclr/jit/lowerarmarch.cpp @@ -1819,7 +1819,13 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node) StoreFFRValue(node); break; } + case NI_Sve_LoadVectorByteZeroExtendFirstFaulting: case NI_Sve_LoadVectorFirstFaulting: + case NI_Sve_LoadVectorInt16SignExtendFirstFaulting: + case NI_Sve_LoadVectorInt32SignExtendFirstFaulting: + case NI_Sve_LoadVectorSByteSignExtendFirstFaulting: + case NI_Sve_LoadVectorUInt16ZeroExtendFirstFaulting: + case NI_Sve_LoadVectorUInt32ZeroExtendFirstFaulting: { LIR::Use use; bool foundUse = BlockRange().TryGetUse(node, &use); @@ -4129,7 +4135,13 @@ void Lowering::StoreFFRValue(GenTreeHWIntrinsic* node) switch (node->GetHWIntrinsicId()) { case NI_Sve_GatherVectorFirstFaulting: + case NI_Sve_LoadVectorByteZeroExtendFirstFaulting: case NI_Sve_LoadVectorFirstFaulting: + case NI_Sve_LoadVectorInt16SignExtendFirstFaulting: + case NI_Sve_LoadVectorInt32SignExtendFirstFaulting: + case NI_Sve_LoadVectorSByteSignExtendFirstFaulting: + case NI_Sve_LoadVectorUInt16ZeroExtendFirstFaulting: + case NI_Sve_LoadVectorUInt32ZeroExtendFirstFaulting: case NI_Sve_SetFfr: break; diff --git a/src/coreclr/jit/lsraarm64.cpp b/src/coreclr/jit/lsraarm64.cpp index 3bad80a928701..3906f91352773 100644 --- a/src/coreclr/jit/lsraarm64.cpp +++ b/src/coreclr/jit/lsraarm64.cpp @@ -2085,13 +2085,7 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou { switch (intrin.id) { - case NI_Sve_LoadVectorByteZeroExtendFirstFaulting: - case NI_Sve_LoadVectorInt16SignExtendFirstFaulting: - case NI_Sve_LoadVectorInt32SignExtendFirstFaulting: case NI_Sve_LoadVectorNonTemporal: - case NI_Sve_LoadVectorSByteSignExtendFirstFaulting: - case NI_Sve_LoadVectorUInt16ZeroExtendFirstFaulting: - case NI_Sve_LoadVectorUInt32ZeroExtendFirstFaulting: case NI_Sve_LoadVector128AndReplicateToVector: case NI_Sve_StoreAndZip: assert(intrinsicTree->OperIsMemoryLoadOrStore()); From 1ef21b79f171bc42482a8f9eb9d37b01cbfbe866 Mon Sep 17 00:00:00 2001 From: Mikhail Ablakatov Date: Wed, 7 Aug 2024 16:38:33 +0000 Subject: [PATCH 83/83] fixup: restore FFR register --- src/coreclr/jit/hwintrinsiccodegenarm64.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp index 3a58da8115405..c0be745ace660 100644 --- a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp @@ -2412,6 +2412,13 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) case NI_Sve_LoadVectorByteZeroExtendFirstFaulting: case NI_Sve_LoadVectorSByteSignExtendFirstFaulting: { + if (intrin.numOperands == 3) + { + // We have extra argument which means there is a "use" of FFR here. Restore it back in FFR register. + assert(op3Reg != REG_NA); + GetEmitter()->emitIns_R(INS_sve_wrffr, emitSize, op3Reg, opt); + } + GetEmitter()->emitIns_R_R_R_R(ins, emitSize, targetReg, op1Reg, op2Reg, REG_ZR, opt); break; }