diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 4093f3b796bdf..66f284702cbdb 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -26445,6 +26445,10 @@ bool GenTreeHWIntrinsic::OperIsMemoryStore(GenTree** pAddr) const addr = Op(2); break; + case NI_Sve_Scatter: + addr = Op(2); + break; + #endif // TARGET_ARM64 default: @@ -26486,7 +26490,11 @@ bool GenTreeHWIntrinsic::OperIsMemoryStore(GenTree** pAddr) const if (addr != nullptr) { +#ifdef TARGET_ARM64 + assert(varTypeIsI(addr) || (varTypeIsSIMD(addr) && ((intrinsicId >= NI_Sve_Scatter)))); +#else assert(varTypeIsI(addr)); +#endif return true; } diff --git a/src/coreclr/jit/hwintrinsic.cpp b/src/coreclr/jit/hwintrinsic.cpp index 4f016940840b2..06b6eb429caa9 100644 --- a/src/coreclr/jit/hwintrinsic.cpp +++ b/src/coreclr/jit/hwintrinsic.cpp @@ -1869,7 +1869,11 @@ GenTree* Compiler::impHWIntrinsic(NamedIntrinsic intrinsic, case NI_Sve_GatherVectorUInt32ZeroExtend: case NI_Sve_GatherVectorWithByteOffsets: assert(varTypeIsSIMD(op3->TypeGet())); - retNode->AsHWIntrinsic()->SetAuxiliaryJitType(getBaseJitTypeOfSIMDType(sigReader.op3ClsHnd)); + if (numArgs == 3) + { + retNode->AsHWIntrinsic()->SetAuxiliaryJitType( + getBaseJitTypeOfSIMDType(sigReader.op3ClsHnd)); + } break; #endif @@ -1885,6 +1889,23 @@ GenTree* Compiler::impHWIntrinsic(NamedIntrinsic intrinsic, assert(!isScalar); retNode = gtNewSimdHWIntrinsicNode(nodeRetType, op1, op2, op3, op4, intrinsic, simdBaseJitType, simdSize); + + switch (intrinsic) + { +#if defined(TARGET_ARM64) + case NI_Sve_Scatter: + assert(varTypeIsSIMD(op3->TypeGet())); + if (numArgs == 4) + { + retNode->AsHWIntrinsic()->SetAuxiliaryJitType( + getBaseJitTypeOfSIMDType(sigReader.op3ClsHnd)); + } + break; +#endif + + default: + break; + } break; } diff --git a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp index 2fa052fcbdcad..bb9e340d03d37 100644 --- a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp @@ -2052,6 +2052,39 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) GetEmitter()->emitIns_R_R(ins, emitSize, targetReg, op1Reg, opt); break; + case NI_Sve_Scatter: + { + if (!varTypeIsSIMD(intrin.op2->gtType)) + { + // Scatter(Vector mask, T1* address, Vector indicies, Vector data) + assert(intrin.numOperands == 4); + emitAttr baseSize = emitActualTypeSize(intrin.baseType); + + if (baseSize == EA_8BYTE) + { + // Index is multiplied by 8 + GetEmitter()->emitIns_R_R_R_R(ins, emitSize, op4Reg, op1Reg, op2Reg, op3Reg, opt, + INS_SCALABLE_OPTS_LSL_N); + } + else + { + // Index is sign or zero extended to 64bits, then multiplied by 4 + assert(baseSize == EA_4BYTE); + opt = varTypeIsUnsigned(node->GetAuxiliaryType()) ? INS_OPTS_SCALABLE_S_UXTW + : INS_OPTS_SCALABLE_S_SXTW; + GetEmitter()->emitIns_R_R_R_R(ins, emitSize, op4Reg, op1Reg, op2Reg, op3Reg, opt, + INS_SCALABLE_OPTS_MOD_N); + } + } + else + { + // Scatter(Vector mask, Vector addresses, Vector data) + assert(intrin.numOperands == 3); + GetEmitter()->emitIns_R_R_R_I(ins, emitSize, op3Reg, op1Reg, op2Reg, 0, opt); + } + break; + } + case NI_Sve_StoreNarrowing: opt = emitter::optGetSveInsOpt(emitTypeSize(intrin.baseType)); GetEmitter()->emitIns_R_R_R_I(ins, emitSize, op3Reg, op1Reg, op2Reg, 0, opt); diff --git a/src/coreclr/jit/hwintrinsiclistarm64sve.h b/src/coreclr/jit/hwintrinsiclistarm64sve.h index 0dcee81c83e4d..d594128674660 100644 --- a/src/coreclr/jit/hwintrinsiclistarm64sve.h +++ b/src/coreclr/jit/hwintrinsiclistarm64sve.h @@ -222,6 +222,7 @@ HARDWARE_INTRINSIC(Sve, SaturatingIncrementBy64BitElementCount, HARDWARE_INTRINSIC(Sve, SaturatingIncrementBy8BitElementCount, 0, 3, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqincb, INS_sve_uqincb, INS_sve_sqincb, INS_sve_uqincb, INS_invalid, INS_invalid}, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasEnumOperand|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics) HARDWARE_INTRINSIC(Sve, SaturatingIncrementByActiveElementCount, -1, 2, true, {INS_invalid, INS_sve_sqincp, INS_sve_sqincp, INS_sve_sqincp, INS_sve_sqincp, INS_sve_sqincp, INS_sve_sqincp, INS_sve_sqincp, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics) HARDWARE_INTRINSIC(Sve, Scale, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fscale, INS_sve_fscale}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve, Scatter, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_st1w, INS_sve_st1w, INS_sve_st1d, INS_sve_st1d, INS_sve_st1w, INS_sve_st1d}, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, ShiftLeftLogical, -1, -1, false, {INS_sve_lsl, INS_sve_lsl, INS_sve_lsl, INS_sve_lsl, INS_sve_lsl, INS_sve_lsl, INS_sve_lsl, INS_sve_lsl, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) HARDWARE_INTRINSIC(Sve, ShiftRightArithmetic, -1, -1, false, {INS_sve_asr, INS_invalid, INS_sve_asr, INS_invalid, INS_sve_asr, INS_invalid, INS_sve_asr, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) HARDWARE_INTRINSIC(Sve, ShiftRightArithmeticForDivide, -1, -1, false, {INS_sve_asrd, INS_invalid, INS_sve_asrd, INS_invalid, INS_sve_asrd, INS_invalid, INS_sve_asrd, INS_invalid, INS_invalid, INS_invalid}, HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand) diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs index 079541962cc9f..2097724df5699 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs @@ -4175,7 +4175,7 @@ internal Arm64() { } /// svuint8_t svcls[_s8]_z(svbool_t pg, svint8_t op) /// CLS Ztied.B, Pg/M, Zop.B /// - public static unsafe Vector LeadingSignCount(Vector value){ throw new PlatformNotSupportedException(); } + public static unsafe Vector LeadingSignCount(Vector value) { throw new PlatformNotSupportedException(); } /// /// svuint16_t svcls[_s16]_m(svuint16_t inactive, svbool_t pg, svint16_t op) @@ -7144,6 +7144,120 @@ internal Arm64() { } public static unsafe Vector Scale(Vector left, Vector right) { throw new PlatformNotSupportedException(); } + // Non-truncating store + + // + // void svst1_scatter_[s64]offset[_f64](svbool_t pg, float64_t *base, svint64_t offsets, svfloat64_t data) + // ST1D Zdata.D, Pg, [Xbase, Zoffsets.D] + // + public static unsafe void Scatter(Vector mask, double* address, Vector indicies, Vector data) { throw new PlatformNotSupportedException(); } + + // + // void svst1_scatter[_u64base_f64](svbool_t pg, svuint64_t bases, svfloat64_t data) + // ST1D Zdata.D, Pg, [Zbases.D, #0] + // + public static unsafe void Scatter(Vector mask, Vector addresses, Vector data) { throw new PlatformNotSupportedException(); } + + // + // void svst1_scatter_[u64]offset[_f64](svbool_t pg, float64_t *base, svuint64_t offsets, svfloat64_t data) + // ST1D Zdata.D, Pg, [Xbase, Zoffsets.D] + // + public static unsafe void Scatter(Vector mask, double* address, Vector indicies, Vector data) { throw new PlatformNotSupportedException(); } + + // + // void svst1_scatter_[s32]offset[_s32](svbool_t pg, int32_t *base, svint32_t offsets, svint32_t data) + // ST1W Zdata.S, Pg, [Xbase, Zoffsets.S, SXTW] + // + public static unsafe void Scatter(Vector mask, int* address, Vector indicies, Vector data) { throw new PlatformNotSupportedException(); } + + // + // void svst1_scatter[_u32base_s32](svbool_t pg, svuint32_t bases, svint32_t data) + // ST1W Zdata.S, Pg, [Zbases.S, #0] + // + // Removed as per #103297 + // public static unsafe void Scatter(Vector mask, Vector addresses, Vector data) { throw new PlatformNotSupportedException(); } + + // + // void svst1_scatter_[u32]offset[_s32](svbool_t pg, int32_t *base, svuint32_t offsets, svint32_t data) + // ST1W Zdata.S, Pg, [Xbase, Zoffsets.S, UXTW] + // + public static unsafe void Scatter(Vector mask, int* address, Vector indicies, Vector data) { throw new PlatformNotSupportedException(); } + + // + // void svst1_scatter_[s64]offset[_s64](svbool_t pg, int64_t *base, svint64_t offsets, svint64_t data) + // ST1D Zdata.D, Pg, [Xbase, Zoffsets.D] + // + public static unsafe void Scatter(Vector mask, long* address, Vector indicies, Vector data) { throw new PlatformNotSupportedException(); } + + // + // void svst1_scatter[_u64base_s64](svbool_t pg, svuint64_t bases, svint64_t data) + // ST1D Zdata.D, Pg, [Zbases.D, #0] + // + public static unsafe void Scatter(Vector mask, Vector addresses, Vector data) { throw new PlatformNotSupportedException(); } + + // + // void svst1_scatter_[u64]offset[_s64](svbool_t pg, int64_t *base, svuint64_t offsets, svint64_t data) + // ST1D Zdata.D, Pg, [Xbase, Zoffsets.D] + // + public static unsafe void Scatter(Vector mask, long* address, Vector indicies, Vector data) { throw new PlatformNotSupportedException(); } + + // + // void svst1_scatter_[s32]offset[_f32](svbool_t pg, float32_t *base, svint32_t offsets, svfloat32_t data) + // ST1W Zdata.S, Pg, [Xbase, Zoffsets.S, SXTW] + // + public static unsafe void Scatter(Vector mask, float* address, Vector indicies, Vector data) { throw new PlatformNotSupportedException(); } + + // + // void svst1_scatter[_u32base_f32](svbool_t pg, svuint32_t bases, svfloat32_t data) + // ST1W Zdata.S, Pg, [Zbases.S, #0] + // + // Removed as per #103297 + // public static unsafe void Scatter(Vector mask, Vector addresses, Vector data) { throw new PlatformNotSupportedException(); } + + // + // void svst1_scatter_[u32]offset[_f32](svbool_t pg, float32_t *base, svuint32_t offsets, svfloat32_t data) + // ST1W Zdata.S, Pg, [Xbase, Zoffsets.S, UXTW] + // + public static unsafe void Scatter(Vector mask, float* address, Vector indicies, Vector data) { throw new PlatformNotSupportedException(); } + + // + // void svst1_scatter_[s32]offset[_u32](svbool_t pg, uint32_t *base, svint32_t offsets, svuint32_t data) + // ST1W Zdata.S, Pg, [Xbase, Zoffsets.S, SXTW] + // + public static unsafe void Scatter(Vector mask, uint* address, Vector indicies, Vector data) { throw new PlatformNotSupportedException(); } + + // + // void svst1_scatter[_u32base_u32](svbool_t pg, svuint32_t bases, svuint32_t data) + // ST1W Zdata.S, Pg, [Zbases.S, #0] + // + // Removed as per #103297 + // public static unsafe void Scatter(Vector mask, Vector addresses, Vector data) { throw new PlatformNotSupportedException(); } + + // + // void svst1_scatter_[u32]offset[_u32](svbool_t pg, uint32_t *base, svuint32_t offsets, svuint32_t data) + // ST1W Zdata.S, Pg, [Xbase, Zoffsets.S, UXTW] + // + public static unsafe void Scatter(Vector mask, uint* address, Vector indicies, Vector data) { throw new PlatformNotSupportedException(); } + + // + // void svst1_scatter_[s64]offset[_u64](svbool_t pg, uint64_t *base, svint64_t offsets, svuint64_t data) + // ST1D Zdata.D, Pg, [Xbase, Zoffsets.D] + // + public static unsafe void Scatter(Vector mask, ulong* address, Vector indicies, Vector data) { throw new PlatformNotSupportedException(); } + + // + // void svst1_scatter[_u64base_u64](svbool_t pg, svuint64_t bases, svuint64_t data) + // ST1D Zdata.D, Pg, [Zbases.D, #0] + // + public static unsafe void Scatter(Vector mask, Vector addresses, Vector data) { throw new PlatformNotSupportedException(); } + + // + // void svst1_scatter_[u64]offset[_u64](svbool_t pg, uint64_t *base, svuint64_t offsets, svuint64_t data) + // ST1D Zdata.D, Pg, [Xbase, Zoffsets.D] + // + public static unsafe void Scatter(Vector mask, ulong* address, Vector indicies, Vector data) { throw new PlatformNotSupportedException(); } + + /// Logical shift left /// diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs index ae978c4362f24..c6b116cc49a44 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs @@ -7189,6 +7189,120 @@ internal Arm64() { } public static unsafe Vector Scale(Vector left, Vector right) => Scale(left, right); + /// Non-truncating store + + // + // void svst1_scatter_[s64]offset[_f64](svbool_t pg, float64_t *base, svint64_t offsets, svfloat64_t data) + // ST1D Zdata.D, Pg, [Xbase, Zoffsets.D] + // + public static unsafe void Scatter(Vector mask, double* address, Vector indicies, Vector data) => Scatter(mask, address, indicies, data); + + // + // void svst1_scatter[_u64base_f64](svbool_t pg, svuint64_t bases, svfloat64_t data) + // ST1D Zdata.D, Pg, [Zbases.D, #0] + // + public static unsafe void Scatter(Vector mask, Vector addresses, Vector data) => Scatter(mask, addresses, data); + + // + // void svst1_scatter_[u64]offset[_f64](svbool_t pg, float64_t *base, svuint64_t offsets, svfloat64_t data) + // ST1D Zdata.D, Pg, [Xbase, Zoffsets.D] + // + public static unsafe void Scatter(Vector mask, double* address, Vector indicies, Vector data) => Scatter(mask, address, indicies, data); + + // + // void svst1_scatter_[s32]offset[_s32](svbool_t pg, int32_t *base, svint32_t offsets, svint32_t data) + // ST1W Zdata.S, Pg, [Xbase, Zoffsets.S, SXTW] + // + public static unsafe void Scatter(Vector mask, int* address, Vector indicies, Vector data) => Scatter(mask, address, indicies, data); + + // + // void svst1_scatter[_u32base_s32](svbool_t pg, svuint32_t bases, svint32_t data) + // ST1W Zdata.S, Pg, [Zbases.S, #0] + // + // Removed as per #103297 + // public static unsafe void Scatter(Vector mask, Vector addresses, Vector data) => Scatter(mask, addresses, data); + + // + // void svst1_scatter_[u32]offset[_s32](svbool_t pg, int32_t *base, svuint32_t offsets, svint32_t data) + // ST1W Zdata.S, Pg, [Xbase, Zoffsets.S, UXTW] + // + public static unsafe void Scatter(Vector mask, int* address, Vector indicies, Vector data) => Scatter(mask, address, indicies, data); + + // + // void svst1_scatter_[s64]offset[_s64](svbool_t pg, int64_t *base, svint64_t offsets, svint64_t data) + // ST1D Zdata.D, Pg, [Xbase, Zoffsets.D] + // + public static unsafe void Scatter(Vector mask, long* address, Vector indicies, Vector data) => Scatter(mask, address, indicies, data); + + // + // void svst1_scatter[_u64base_s64](svbool_t pg, svuint64_t bases, svint64_t data) + // ST1D Zdata.D, Pg, [Zbases.D, #0] + // + public static unsafe void Scatter(Vector mask, Vector addresses, Vector data) => Scatter(mask, addresses, data); + + // + // void svst1_scatter_[u64]offset[_s64](svbool_t pg, int64_t *base, svuint64_t offsets, svint64_t data) + // ST1D Zdata.D, Pg, [Xbase, Zoffsets.D] + // + public static unsafe void Scatter(Vector mask, long* address, Vector indicies, Vector data) => Scatter(mask, address, indicies, data); + + // + // void svst1_scatter_[s32]offset[_f32](svbool_t pg, float32_t *base, svint32_t offsets, svfloat32_t data) + // ST1W Zdata.S, Pg, [Xbase, Zoffsets.S, SXTW] + // + public static unsafe void Scatter(Vector mask, float* address, Vector indicies, Vector data) => Scatter(mask, address, indicies, data); + + // + // void svst1_scatter[_u32base_f32](svbool_t pg, svuint32_t bases, svfloat32_t data) + // ST1W Zdata.S, Pg, [Zbases.S, #0] + // + // Removed as per #103297 + // public static unsafe void Scatter(Vector mask, Vector addresses, Vector data) => Scatter(mask, addresses, data); + + // + // void svst1_scatter_[u32]offset[_f32](svbool_t pg, float32_t *base, svuint32_t offsets, svfloat32_t data) + // ST1W Zdata.S, Pg, [Xbase, Zoffsets.S, UXTW] + // + public static unsafe void Scatter(Vector mask, float* address, Vector indicies, Vector data) => Scatter(mask, address, indicies, data); + + // + // void svst1_scatter_[s32]offset[_u32](svbool_t pg, uint32_t *base, svint32_t offsets, svuint32_t data) + // ST1W Zdata.S, Pg, [Xbase, Zoffsets.S, SXTW] + // + public static unsafe void Scatter(Vector mask, uint* address, Vector indicies, Vector data) => Scatter(mask, address, indicies, data); + + // + // void svst1_scatter[_u32base_u32](svbool_t pg, svuint32_t bases, svuint32_t data) + // ST1W Zdata.S, Pg, [Zbases.S, #0] + // + // Removed as per #103297 + // public static unsafe void Scatter(Vector mask, Vector addresses, Vector data) => Scatter(mask, addresses, data); + + // + // void svst1_scatter_[u32]offset[_u32](svbool_t pg, uint32_t *base, svuint32_t offsets, svuint32_t data) + // ST1W Zdata.S, Pg, [Xbase, Zoffsets.S, UXTW] + // + public static unsafe void Scatter(Vector mask, uint* address, Vector indicies, Vector data) => Scatter(mask, address, indicies, data); + + // + // void svst1_scatter_[s64]offset[_u64](svbool_t pg, uint64_t *base, svint64_t offsets, svuint64_t data) + // ST1D Zdata.D, Pg, [Xbase, Zoffsets.D] + // + public static unsafe void Scatter(Vector mask, ulong* address, Vector indicies, Vector data) => Scatter(mask, address, indicies, data); + + // + // void svst1_scatter[_u64base_u64](svbool_t pg, svuint64_t bases, svuint64_t data) + // ST1D Zdata.D, Pg, [Zbases.D, #0] + // + public static unsafe void Scatter(Vector mask, Vector addresses, Vector data) => Scatter(mask, addresses, data); + + // + // void svst1_scatter_[u64]offset[_u64](svbool_t pg, uint64_t *base, svuint64_t offsets, svuint64_t data) + // ST1D Zdata.D, Pg, [Xbase, Zoffsets.D] + // + public static unsafe void Scatter(Vector mask, ulong* address, Vector indicies, Vector data) => Scatter(mask, address, indicies, data); + + /// Logical shift left /// diff --git a/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs b/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs index 3df9a05b405ba..ba7c19442f3bf 100644 --- a/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs +++ b/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs @@ -5266,6 +5266,25 @@ internal Arm64() { } public static System.Numerics.Vector Scale(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } public static System.Numerics.Vector Scale(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static unsafe void Scatter(System.Numerics.Vector mask, double* address, System.Numerics.Vector indicies, System.Numerics.Vector data) { throw null; } + public static unsafe void Scatter(System.Numerics.Vector mask, System.Numerics.Vector addresses, System.Numerics.Vector data) { throw null; } + public static unsafe void Scatter(System.Numerics.Vector mask, double* address, System.Numerics.Vector indicies, System.Numerics.Vector data) { throw null; } + public static unsafe void Scatter(System.Numerics.Vector mask, int* address, System.Numerics.Vector indicies, System.Numerics.Vector data) { throw null; } + // public static unsafe void Scatter(System.Numerics.Vector mask, System.Numerics.Vector addresses, System.Numerics.Vector data) { throw null; } + public static unsafe void Scatter(System.Numerics.Vector mask, int* address, System.Numerics.Vector indicies, System.Numerics.Vector data) { throw null; } + public static unsafe void Scatter(System.Numerics.Vector mask, long* address, System.Numerics.Vector indicies, System.Numerics.Vector data) { throw null; } + public static unsafe void Scatter(System.Numerics.Vector mask, System.Numerics.Vector addresses, System.Numerics.Vector data) { throw null; } + public static unsafe void Scatter(System.Numerics.Vector mask, long* address, System.Numerics.Vector indicies, System.Numerics.Vector data) { throw null; } + public static unsafe void Scatter(System.Numerics.Vector mask, float* address, System.Numerics.Vector indicies, System.Numerics.Vector data) { throw null; } + // public static unsafe void Scatter(System.Numerics.Vector mask, System.Numerics.Vector addresses, System.Numerics.Vector data) { throw null; } + public static unsafe void Scatter(System.Numerics.Vector mask, float* address, System.Numerics.Vector indicies, System.Numerics.Vector data) { throw null; } + public static unsafe void Scatter(System.Numerics.Vector mask, uint* address, System.Numerics.Vector indicies, System.Numerics.Vector data) { throw null; } + // public static unsafe void Scatter(System.Numerics.Vector mask, System.Numerics.Vector addresses, System.Numerics.Vector data) { throw null; } + public static unsafe void Scatter(System.Numerics.Vector mask, uint* address, System.Numerics.Vector indicies, System.Numerics.Vector data) { throw null; } + public static unsafe void Scatter(System.Numerics.Vector mask, ulong* address, System.Numerics.Vector indicies, System.Numerics.Vector data) { throw null; } + public static unsafe void Scatter(System.Numerics.Vector mask, System.Numerics.Vector addresses, System.Numerics.Vector data) { throw null; } + public static unsafe void Scatter(System.Numerics.Vector mask, ulong* address, System.Numerics.Vector indicies, System.Numerics.Vector data) { throw null; } + public static System.Numerics.Vector ShiftLeftLogical(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } public static System.Numerics.Vector ShiftLeftLogical(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } public static System.Numerics.Vector ShiftLeftLogical(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } diff --git a/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs b/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs index 7455aebc0ad32..5e875a40aff90 100644 --- a/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs +++ b/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs @@ -4031,6 +4031,22 @@ ("SveVecPairBinOpTest.template", new Dictionary { ["TestName"] = "Sve_Scale_float_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Scale", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSingle()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt32()", ["ValidateEntry"] = "result[i] != Helpers.Scale(left[i], right[i])"}), ("SveVecPairBinOpTest.template", new Dictionary { ["TestName"] = "Sve_Scale_double_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Scale", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetDouble()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt64()", ["ValidateEntry"] = "result[i] != Helpers.Scale(left[i], right[i])"}), + ("SveScatterVectorBases.template", new Dictionary { ["TestName"] = "Sve_Scatter_bases_double_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Scatter", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt64", ["NarrowingType"] = "Double", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()", ["NextValueOp3"] = "TestLibrary.Generator.GetDouble()"}), + ("SveScatterVectorBases.template", new Dictionary { ["TestName"] = "Sve_Scatter_bases_long_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Scatter", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt64", ["NarrowingType"] = "Int64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt64()"}), + ("SveScatterVectorBases.template", new Dictionary { ["TestName"] = "Sve_Scatter_bases_ulong_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Scatter", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt64", ["NarrowingType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt64()"}), + ("SveScatterVectorIndices.template", new Dictionary { ["TestName"] = "Sve_Scatter_indices_double_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Scatter", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2BaseType"] = "Double", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskDouble()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt64()", ["NextValueOp4"] = "TestLibrary.Generator.GetDouble()"}), + ("SveScatterVectorIndices.template", new Dictionary { ["TestName"] = "Sve_Scatter_indices_double_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Scatter", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2BaseType"] = "Double", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskDouble()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt64()", ["NextValueOp4"] = "TestLibrary.Generator.GetDouble()"}), + ("SveScatterVectorIndices.template", new Dictionary { ["TestName"] = "Sve_Scatter_indices_int_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Scatter", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2BaseType"] = "Int32", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt32()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt32()", ["NextValueOp4"] = "TestLibrary.Generator.GetInt32()"}), + ("SveScatterVectorIndices.template", new Dictionary { ["TestName"] = "Sve_Scatter_indices_int_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Scatter", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2BaseType"] = "Int32", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt32()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp4"] = "TestLibrary.Generator.GetInt32()"}), + ("SveScatterVectorIndices.template", new Dictionary { ["TestName"] = "Sve_Scatter_indices_long_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Scatter", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2BaseType"] = "Int64", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt64()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt64()", ["NextValueOp4"] = "TestLibrary.Generator.GetInt64()"}), + ("SveScatterVectorIndices.template", new Dictionary { ["TestName"] = "Sve_Scatter_indices_long_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Scatter", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2BaseType"] = "Int64", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt64()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt64()", ["NextValueOp4"] = "TestLibrary.Generator.GetInt64()"}), + ("SveScatterVectorIndices.template", new Dictionary { ["TestName"] = "Sve_Scatter_indices_float_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Scatter", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2BaseType"] = "Single", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskSingle()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt32()", ["NextValueOp4"] = "TestLibrary.Generator.GetSingle()"}), + ("SveScatterVectorIndices.template", new Dictionary { ["TestName"] = "Sve_Scatter_indices_float_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Scatter", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2BaseType"] = "Single", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskSingle()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp4"] = "TestLibrary.Generator.GetSingle()"}), + ("SveScatterVectorIndices.template", new Dictionary { ["TestName"] = "Sve_Scatter_indices_uint_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Scatter", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2BaseType"] = "UInt32", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt32()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt32()", ["NextValueOp4"] = "TestLibrary.Generator.GetUInt32()"}), + ("SveScatterVectorIndices.template", new Dictionary { ["TestName"] = "Sve_Scatter_indices_uint_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Scatter", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2BaseType"] = "UInt32", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt32()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp4"] = "TestLibrary.Generator.GetUInt32()"}), + ("SveScatterVectorIndices.template", new Dictionary { ["TestName"] = "Sve_Scatter_indices_ulong_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Scatter", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2BaseType"] = "UInt64", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt64()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt64()", ["NextValueOp4"] = "TestLibrary.Generator.GetUInt64()"}), + ("SveScatterVectorIndices.template", new Dictionary { ["TestName"] = "Sve_Scatter_indices_ulong_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Scatter", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2BaseType"] = "UInt64", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt64()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt64()", ["NextValueOp4"] = "TestLibrary.Generator.GetUInt64()"}), + ("SveVecBinOpDifferentTypesTest.template", new Dictionary { ["TestName"] = "Sve_ShiftLeftLogical_sbyte_byte", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "ShiftLeftLogical", ["RetVectorType"] = "Vector", ["RetBaseType"] = "SByte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "SByte", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSByte()", ["NextValueOp2"] = "TestLibrary.Generator.GetByte()", ["ValidateIterResult"] = "(sbyte)Helpers.ShiftLeft((byte)left[i], (ulong)right[i]) != result[i]", ["GetIterResult"] = "(sbyte)Helpers.ShiftLeft((byte)left[i], (ulong)right[i])"}), ("SveVecBinOpDifferentTypesTest.template", new Dictionary { ["TestName"] = "Sve_ShiftLeftLogical_short_ushort", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "ShiftLeftLogical", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int16", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt16()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt16()", ["ValidateIterResult"] = "(short)Helpers.ShiftLeft((ushort)left[i], (ulong)right[i]) != result[i]", ["GetIterResult"] = "(short)Helpers.ShiftLeft((ushort)left[i], (ulong)right[i])"}), ("SveVecBinOpDifferentTypesTest.template", new Dictionary { ["TestName"] = "Sve_ShiftLeftLogical_int_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "ShiftLeftLogical", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["ValidateIterResult"] = "(int)Helpers.ShiftLeft((uint)left[i], (ulong)right[i]) != result[i]", ["GetIterResult"] = "(int)Helpers.ShiftLeft((uint)left[i], (ulong)right[i])"}), diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveScatterVectorBases.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveScatterVectorBases.template new file mode 100644 index 0000000000000..6103c423254ff --- /dev/null +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveScatterVectorBases.template @@ -0,0 +1,499 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +/****************************************************************************** + * This file is auto-generated from a template file by the GenerateTests.csx * + * script in tests\src\JIT\HardwareIntrinsics\Arm\Shared. In order to make * + * changes, please update the corresponding template and run according to the * + * directions listed in the file. * + ******************************************************************************/ + +using System; +using System.Collections.Generic; +using System.Numerics; +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.Arm; +using Xunit; + +namespace JIT.HardwareIntrinsics.Arm._Sve +{ + public static partial class Program + { + [Fact] + public static void {TestName}() + { + var test = new SveScatterVectorBasesTest__{TestName}(); + + if (test.IsSupported) + { + // Validates basic functionality works, using Unsafe.Read + test.RunBasicScenario_UnsafeRead(); + + // Validates basic functionality works + test.RunBasicScenario_Load(); + + // Validates fully masked out load works. + test.RunBasicScenario_FalseMask(); + + // Validates fully masked out load with invalid address works. + test.RunBasicScenario_NonFaulting(); + + // Validates calling via reflection works, using Unsafe.Read + test.RunReflectionScenario_UnsafeRead(); + + // Validates passing a local works, using Unsafe.Read + test.RunLclVarScenario_UnsafeRead(); + + // Validates passing an instance member of a class works + test.RunClassFldScenario(); + + // Validates passing the field of a local struct works + test.RunStructLclFldScenario(); + + // Validates passing an instance member of a struct works + test.RunStructFldScenario(); + } + else + { + // Validates we throw on unsupported hardware + test.RunUnsupportedScenario(); + } + + if (!test.Succeeded) + { + throw new Exception("One or more scenarios did not complete as expected."); + } + } + } + + public sealed unsafe class SveScatterVectorBasesTest__{TestName} + { + private struct DataTable + { + private byte[] inMask; + private byte[] inData; + public byte[] outArray; + private byte[] inAddress; + + private GCHandle inMaskHandle; + private GCHandle inAddressHandle; + private GCHandle inDataHandle; + private GCHandle outHandle; + + private ulong alignment; + + public DataTable({Op1BaseType}[] inMask, {Op1BaseType}[] inData, {Op1BaseType}[] outArray, {Op2BaseType}[] inAddress, int alignment) + { + int sizeOfInMask = inMask.Length * Unsafe.SizeOf<{Op1BaseType}>(); + int sizeOfInData = inData.Length * Unsafe.SizeOf<{Op1BaseType}>(); + int sizeOfOutArray = outArray.Length * Unsafe.SizeOf<{Op1BaseType}>(); + int sizeOfInOffset = inAddress.Length * Unsafe.SizeOf<{Op2BaseType}>(); + + if ((alignment != 64 && alignment != 16 && alignment != 8) || (alignment * 2) < sizeOfInMask || (alignment * 2) < sizeOfInOffset || (alignment * 2) < sizeOfInData) + { + throw new ArgumentException($"Invalid value of alignment: {alignment}, sizeOfInMask: {sizeOfInMask}, sizeOfInOffset: {sizeOfInOffset}, sizeOfInData: {sizeOfInData}"); + } + + this.inMask = new byte[alignment * 2]; + this.inData = new byte[alignment * 2]; + this.outArray = new byte[(alignment * 2) + sizeOfOutArray]; + this.inAddress = new byte[alignment * 2]; + + this.inMaskHandle = GCHandle.Alloc(this.inMask, GCHandleType.Pinned); + this.inAddressHandle = GCHandle.Alloc(this.inAddress, GCHandleType.Pinned); + this.inDataHandle = GCHandle.Alloc(this.inData, GCHandleType.Pinned); + this.outHandle = GCHandle.Alloc(this.outArray, GCHandleType.Pinned); + + this.alignment = (ulong)alignment; + + Unsafe.CopyBlockUnaligned(ref Unsafe.AsRef(inMaskArrayPtr), ref Unsafe.As<{Op1BaseType}, byte>(ref inMask[0]), (uint)sizeOfInMask); + Unsafe.CopyBlockUnaligned(ref Unsafe.AsRef(inDataArrayPtr), ref Unsafe.As<{Op1BaseType}, byte>(ref inData[0]), (uint)sizeOfInData); + Unsafe.CopyBlockUnaligned(ref Unsafe.AsRef(outArrayPtr), ref Unsafe.As<{Op1BaseType}, byte>(ref outArray[0]), (uint)sizeOfOutArray); + + // Add the base pointer to the offsets within outArray to create complete pointers. + for (var i = 0; i < inAddress.Length; i++) { inAddress[i] += ({Op2BaseType})outArrayPtr; } + Unsafe.CopyBlockUnaligned(ref Unsafe.AsRef(inAddressArrayPtr), ref Unsafe.As<{Op2BaseType}, byte>(ref inAddress[0]), (uint)sizeOfInOffset); + } + + public void* inMaskArrayPtr => Align((byte*)(inMaskHandle.AddrOfPinnedObject().ToPointer()), alignment); + public void* inAddressArrayPtr => Align((byte*)(inAddressHandle.AddrOfPinnedObject().ToPointer()), alignment); + public void* inDataArrayPtr => Align((byte*)(inDataHandle.AddrOfPinnedObject().ToPointer()), alignment); + public void* outArrayPtr => Align((byte*)(outHandle.AddrOfPinnedObject().ToPointer()), alignment); + + public void Dispose() + { + inMaskHandle.Free(); + inDataHandle.Free(); + inAddressHandle.Free(); + outHandle.Free(); + } + + private static unsafe void* Align(byte* buffer, ulong expectedAlignment) + { + return (void*)(((ulong)buffer + expectedAlignment - 1) & ~(expectedAlignment - 1)); + } + + public void ResetOutArray() + { + for (int i = 0; i < this.outArray.Length; i++) + { + this.outArray[i] = 0; + } + } + } + + public static void MakeDistinct({Op2BaseType}[] input) + { + HashSet<{Op2BaseType}> dict = new (); + for(var i = 0; i < input.Length;) + { + if(!dict.Add(input[i])) + { + // value already exist + input[i] += sizeof({Op1BaseType}); + continue; + } + i++; + } + } + + private struct TestStruct + { + public {Op1VectorType}<{Op1BaseType}> _mask; + public {Op1VectorType}<{Op1BaseType}> _data; + public {Op2VectorType}<{Op2BaseType}> _address; + + public static TestStruct Create(DataTable _dataTable) + { + var testStruct = new TestStruct(); + + for (var i = 0; i < MaskElementCount; i++) { _maskArr[i] = {NextValueOp1}; } + for (var i = 0; i < DataElementCount; i++) { _dataArr[i] = {NextValueOp3}; } + _dataTable.ResetOutArray(); + + // Fill full of offsets into the data buffer. + for (var i = 0; i < AddressElementCount; i++) { _addressArr[i] = ({NextValueOp2} % ({Op2BaseType})OutElementCount ) * sizeof({Op2BaseType}); } + MakeDistinct(_addressArr); + + // Add the base pointer to the offsets within outArray to create complete pointers. + for (var i = 0; i < AddressElementCount; i++) { _addressArr[i] += ({Op2BaseType})_dataTable.outArrayPtr; } + + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1VectorType}<{Op1BaseType}>, byte>(ref testStruct._mask), ref Unsafe.As<{Op1BaseType}, byte>(ref _maskArr[0]), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1VectorType}<{Op1BaseType}>, byte>(ref testStruct._data), ref Unsafe.As<{Op1BaseType}, byte>(ref _dataArr[0]), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op2VectorType}<{Op2BaseType}>, byte>(ref testStruct._address), ref Unsafe.As<{Op2BaseType}, byte>(ref _addressArr[0]), (uint)Unsafe.SizeOf<{Op2VectorType}<{Op2BaseType}>>()); + + return testStruct; + } + + public void RunStructFldScenario(SveScatterVectorBasesTest__{TestName} testClass) + { + {Isa}.{Method}(_mask, _address, _data); + testClass.ValidateResult(_mask, _data, _address); + } + } + + private static readonly int LargestVectorSize = 64; + + // A large enough buffer to hold many values. + // Values in address will point to locations within this buffer. + private static readonly int OutElementCount = 1024; + private static readonly int MaskElementCount = Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>() / sizeof({Op1BaseType}); + private static readonly int DataElementCount = Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>() / sizeof({Op1BaseType}); + private static readonly int AddressElementCount = Unsafe.SizeOf<{Op2VectorType}<{Op2BaseType}>>() / sizeof({Op2BaseType}); + + private static {Op1BaseType}[] _maskArr = new {Op1BaseType}[MaskElementCount]; + private static {Op1BaseType}[] _dataArr = new {Op1BaseType}[DataElementCount]; + private static {Op2BaseType}[] _addressArr = new {Op2BaseType}[AddressElementCount]; + private static {Op1BaseType}[] _falseData = new {Op1BaseType}[DataElementCount]; + + private {Op1VectorType}<{Op1BaseType}> _mask; + private {Op1VectorType}<{Op1BaseType}> _data; + private {Op2VectorType}<{Op2BaseType}> _address; + private {Op1VectorType}<{Op1BaseType}> _falseFld; + + private DataTable _dataTable; + + public SveScatterVectorBasesTest__{TestName}() + { + Succeeded = true; + + for (var i = 0; i < MaskElementCount; i++) { _maskArr[i] = {NextValueOp1}; } + for (var i = 0; i < DataElementCount; i++) { _dataArr[i] = {NextValueOp3}; } + + // Fill full of offsets into the data buffer. They wil be expanded to full pointers inside the DataTable constructor. + for (var i = 0; i < AddressElementCount; i++) { _addressArr[i] = ({NextValueOp2} % ({Op2BaseType})OutElementCount) * sizeof({Op2BaseType}); } + MakeDistinct(_addressArr); + + for (var i = 0; i < DataElementCount; i++) { _falseData[i] = {NextValueOp3}; } + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1VectorType}<{Op1BaseType}>, byte>(ref _falseFld), ref Unsafe.As<{Op1BaseType}, byte>(ref _falseData[0]), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + + _dataTable = new DataTable(_maskArr, _dataArr, new {Op1BaseType}[OutElementCount], _addressArr, LargestVectorSize); + + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1VectorType}<{Op1BaseType}>, byte>(ref _mask), ref Unsafe.As<{Op1BaseType}, byte>(ref _maskArr[0]), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1VectorType}<{Op1BaseType}>, byte>(ref _data), ref Unsafe.As<{Op1BaseType}, byte>(ref _dataArr[0]), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op2VectorType}<{Op2BaseType}>, byte>(ref _address), ref Unsafe.As<{Op2BaseType}, byte>(ref _addressArr[0]), (uint)Unsafe.SizeOf<{Op2VectorType}<{Op2BaseType}>>()); + } + + public bool IsSupported => {Isa}.IsSupported; + + public bool Succeeded { get; set; } + + public void RunBasicScenario_UnsafeRead() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunBasicScenario_UnsafeRead)); + + _dataTable.ResetOutArray(); + {Isa}.{Method}( + Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inMaskArrayPtr), + Unsafe.Read<{Op2VectorType}<{Op2BaseType}>>(_dataTable.inAddressArrayPtr), + Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inDataArrayPtr) + ); + + ValidateResult(_dataTable.inMaskArrayPtr, _dataTable.inDataArrayPtr, _dataTable.inAddressArrayPtr); + + } + + public void RunBasicScenario_Load() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunBasicScenario_Load)); + + {Op1VectorType}<{Op1BaseType}> loadMask1 = {Isa}.CreateTrueMask{Op1BaseType}(SveMaskPattern.All); + {Op2VectorType}<{Op2BaseType}> loadMask2 = {Isa}.CreateTrueMask{Op2BaseType}(SveMaskPattern.All); + _dataTable.ResetOutArray(); + + {Isa}.{Method}( + {Isa}.LoadVector(loadMask1, ({Op1BaseType}*)(_dataTable.inMaskArrayPtr)), + {Isa}.LoadVector(loadMask2, ({Op2BaseType}*)(_dataTable.inAddressArrayPtr)), + {Isa}.LoadVector(loadMask1, ({Op1BaseType}*)(_dataTable.inDataArrayPtr)) + ); + + ValidateResult(_dataTable.inMaskArrayPtr, _dataTable.inDataArrayPtr, _dataTable.inAddressArrayPtr); + } + + public void RunBasicScenario_FalseMask() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunBasicScenario_FalseMask)); + + {Op1VectorType}<{Op1BaseType}> falseMask = {Isa}.CreateFalseMask{Op1BaseType}(); + + _dataTable.ResetOutArray(); + + {Isa}.{Method}( + falseMask, + Unsafe.Read<{Op2VectorType}<{Op2BaseType}>>(_dataTable.inAddressArrayPtr), + Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inDataArrayPtr) + ); + + ValidateZeroResult(_dataTable.inDataArrayPtr, _dataTable.inAddressArrayPtr); + } + + public void RunBasicScenario_NonFaulting() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunBasicScenario_NonFaulting)); + + {Op1VectorType}<{Op1BaseType}> falseMask = {Isa}.CreateFalseMask{Op1BaseType}(); + _dataTable.ResetOutArray(); + + try + { + {Isa}.{Method}( + falseMask, + {Op2VectorType}<{Op2BaseType}>.Zero, + Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inDataArrayPtr) + ); + + ValidateZeroResult(_dataTable.inDataArrayPtr, _dataTable.inAddressArrayPtr); + } + catch + { + Succeeded = false; + } + } + + public void RunReflectionScenario_UnsafeRead() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunReflectionScenario_UnsafeRead)); + + _dataTable.ResetOutArray(); + + typeof(Sve).GetMethod(nameof({Isa}.{Method}), new Type[] { typeof({Op1VectorType}<{Op1BaseType}>), typeof({Op2VectorType}<{Op2BaseType}>), typeof({Op1VectorType}<{Op1BaseType}>) }) + .Invoke(null, new object[] { + Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inMaskArrayPtr), + Unsafe.Read<{Op2VectorType}<{Op2BaseType}>>(_dataTable.inAddressArrayPtr), + Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inDataArrayPtr), + }); + + ValidateResult(_dataTable.inMaskArrayPtr, _dataTable.inDataArrayPtr, _dataTable.inAddressArrayPtr); + } + + public void RunLclVarScenario_UnsafeRead() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunLclVarScenario_UnsafeRead)); + + var mask = Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inMaskArrayPtr); + var address = Unsafe.Read<{Op2VectorType}<{Op2BaseType}>>(_dataTable.inAddressArrayPtr); + var data = Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inDataArrayPtr); + + _dataTable.ResetOutArray(); + {Isa}.{Method}(mask, address, data); + + ValidateResult(mask, data, address); + } + + public void RunClassFldScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunClassFldScenario)); + _dataTable.ResetOutArray(); + {Isa}.{Method}(_mask, _address, _data); + + ValidateResult(_mask, _data, _address); + } + + public void RunStructLclFldScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunStructLclFldScenario)); + var test = TestStruct.Create(_dataTable); + + {Isa}.{Method}(test._mask, test._address, test._data); + + ValidateResult(test._mask, test._data, test._address); + } + + public void RunStructFldScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunStructFldScenario)); + + var test = TestStruct.Create(_dataTable); + + test.RunStructFldScenario(this); + } + + public void RunUnsupportedScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunUnsupportedScenario)); + + Succeeded = false; + + try + { + RunBasicScenario_Load(); + } + catch (PlatformNotSupportedException) + { + Succeeded = true; + } + } + + private void ValidateResult({Op1VectorType}<{Op1BaseType}> mask, {Op1VectorType}<{Op1BaseType}> data, {Op2VectorType}<{Op2BaseType}> address, [CallerMemberName] string method = "") + { + {Op1BaseType}[] maskArray = new {Op1BaseType}[MaskElementCount]; + {Op1BaseType}[] dataArray = new {Op1BaseType}[DataElementCount]; + {Op2BaseType}[] addressArray = new {Op2BaseType}[AddressElementCount]; + + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref maskArray[0]), mask); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref dataArray[0]), data); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op2BaseType}, byte>(ref addressArray[0]), address); + + ValidateResult(maskArray, dataArray, addressArray, method); + } + + private void ValidateResult(void* mask, void* data, void* address, [CallerMemberName] string method = "") + { + {Op1BaseType}[] maskArray = new {Op1BaseType}[MaskElementCount]; + {Op1BaseType}[] dataArray = new {Op1BaseType}[DataElementCount]; + {Op2BaseType}[] addressArray = new {Op2BaseType}[AddressElementCount]; + + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref maskArray[0]), ref Unsafe.AsRef(mask), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref dataArray[0]), ref Unsafe.AsRef(data), (uint)(Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>())); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op2BaseType}, byte>(ref addressArray[0]), ref Unsafe.AsRef(address), (uint)(Unsafe.SizeOf<{Op2VectorType}<{Op2BaseType}>>())); + + ValidateResult(maskArray, dataArray, addressArray, method); + } + + private void ValidateResult({Op1BaseType}[] mask, {Op1BaseType}[] data, {Op2BaseType}[] address, [CallerMemberName] string method = "") + { + bool succeeded = true; + + var actualResult = new {NarrowingType}[DataElementCount]; + for (var i = 0; i < DataElementCount; i++) + { + actualResult[i] = *({NarrowingType}*)(address[i]); + } + + for (var i = 0; i < DataElementCount; i++) + { + {Op1BaseType} expectedResult = mask[i] == 0 ? 0 : data[i]; + if (actualResult[i] != ({NarrowingType})expectedResult) + { + succeeded = false; + break; + } + } + + if (!succeeded) + { + TestLibrary.TestFramework.LogInformation($"{nameof(Sve)}.{nameof({Isa}.{Method})}({Op1VectorType}<{Op1BaseType}>, {Op2VectorType}<{Op2BaseType}>, {Op1VectorType}<{Op1BaseType}>): {method} failed:"); + TestLibrary.TestFramework.LogInformation($" mask: ({string.Join(", ", mask)})"); + TestLibrary.TestFramework.LogInformation($" input: ({string.Join(", ", data)})"); + TestLibrary.TestFramework.LogInformation($"result: ({string.Join(", ", actualResult)})"); + TestLibrary.TestFramework.LogInformation(string.Empty); + + Succeeded = false; + } + } + + private void ValidateZeroResult({Op1VectorType}<{Op1BaseType}> data, {Op2VectorType}<{Op2BaseType}> address, [CallerMemberName] string method = "") + { + {Op1BaseType}[] dataArray = new {Op1BaseType}[DataElementCount]; + {Op2BaseType}[] addressArray = new {Op2BaseType}[AddressElementCount]; + + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref dataArray[0]), data); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op2BaseType}, byte>(ref addressArray[0]), address); + + ValidateZeroResult(dataArray, addressArray, method); + } + + private void ValidateZeroResult(void* data, void* address, [CallerMemberName] string method = "") + { + {Op1BaseType}[] dataArray = new {Op1BaseType}[DataElementCount]; + {Op2BaseType}[] addressArray = new {Op2BaseType}[AddressElementCount]; + + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref dataArray[0]), ref Unsafe.AsRef(data), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op2BaseType}, byte>(ref addressArray[0]), ref Unsafe.AsRef(address), (uint)(Unsafe.SizeOf<{Op2VectorType}<{Op2BaseType}>>())); + + ValidateZeroResult(dataArray, addressArray, method); + } + + private void ValidateZeroResult({Op1BaseType}[] data, {Op2BaseType}[] address, [CallerMemberName] string method = "") + { + bool succeeded = true; + var actualResult = new {NarrowingType}[DataElementCount]; + for (var i = 0; i < DataElementCount; i++) + { + actualResult[i] = *({NarrowingType}*)(address[i]); + } + + for (var i = 0; i < DataElementCount; i++) + { + if (actualResult[i] != 0) + { + succeeded = false; + break; + } + } + + if (!succeeded) + { + TestLibrary.TestFramework.LogInformation($"{nameof(Sve)}.{nameof({Isa}.{Method})}({Op1VectorType}<{Op1BaseType}>, {Op2VectorType}<{Op2BaseType}>, {Op1VectorType}<{Op1BaseType}>): {method} failed:"); + TestLibrary.TestFramework.LogInformation($" input: ({string.Join(", ", data)})"); + TestLibrary.TestFramework.LogInformation($" result: ({string.Join(", ", actualResult)})"); + TestLibrary.TestFramework.LogInformation(string.Empty); + + Succeeded = false; + } + } + } +} \ No newline at end of file diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveScatterVectorIndices.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveScatterVectorIndices.template new file mode 100644 index 0000000000000..903e79e1f376b --- /dev/null +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveScatterVectorIndices.template @@ -0,0 +1,505 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +/****************************************************************************** + * This file is auto-generated from a template file by the GenerateTests.csx * + * script in tests\src\JIT\HardwareIntrinsics\Arm\Shared. In order to make * + * changes, please update the corresponding template and run according to the * + * directions listed in the file. * + ******************************************************************************/ + +using System; +using System.Collections.Generic; +using System.Numerics; +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.Arm; +using Xunit; + +namespace JIT.HardwareIntrinsics.Arm._Sve +{ + public static partial class Program + { + [Fact] + public static void {TestName}() + { + var test = new SveScatterVectorIndicesTest__{TestName}(); + + if (test.IsSupported) + { + // Validates basic functionality works, using Unsafe.Read + test.RunBasicScenario_UnsafeRead(); + + // Validates basic functionality works + test.RunBasicScenario_Load(); + + // Validates fully masked out load works. + test.RunBasicScenario_FalseMask(); + + // Validates fully masked out load with invalid address works. + test.RunBasicScenario_NonFaulting(); + + // Validates calling via reflection works, using Unsafe.Read + test.RunReflectionScenario_UnsafeRead(); + + // Validates passing a local works, using Unsafe.Read + test.RunLclVarScenario_UnsafeRead(); + + // Validates passing an instance member of a class works + test.RunClassFldScenario(); + + // Validates passing the field of a local struct works + test.RunStructLclFldScenario(); + + // Validates passing an instance member of a struct works + test.RunStructFldScenario(); + } + else + { + // Validates we throw on unsupported hardware + test.RunUnsupportedScenario(); + } + + if (!test.Succeeded) + { + throw new Exception("One or more scenarios did not complete as expected."); + } + } + } + + public sealed unsafe class SveScatterVectorIndicesTest__{TestName} + { + private struct DataTable + { + private byte[] inMask; + private byte[] inData; + private byte[] outArray; + private byte[] inOffset; + + private GCHandle inMaskHandle; + private GCHandle inOffsetHandle; + private GCHandle inDataHandle; + private GCHandle outHandle; + + private ulong alignment; + + public DataTable({Op1BaseType}[] inMask, {Op1BaseType}[] inData, {Op1BaseType}[] outArray, {Op3BaseType}[] inOffset, int alignment) + { + int sizeOfInMask = inMask.Length * Unsafe.SizeOf<{Op1BaseType}>(); + int sizeOfInData = inData.Length * Unsafe.SizeOf<{Op1BaseType}>(); + int sizeOfOutArray = outArray.Length * Unsafe.SizeOf<{Op1BaseType}>(); + int sizeOfInOffset = inOffset.Length * Unsafe.SizeOf<{Op3BaseType}>(); + + if ((alignment != 64 && alignment != 16 && alignment != 8) || (alignment * 2) < sizeOfInMask || (alignment * 2) < sizeOfInOffset || (alignment * 2) < sizeOfInData) + { + throw new ArgumentException($"Invalid value of alignment: {alignment}, sizeOfinArray1: {sizeOfInMask}, sizeOfinArray2: {sizeOfInOffset}, sizeOfInData: {sizeOfInData}"); + } + + this.inMask = new byte[alignment * 2]; + this.inData = new byte[alignment * 2]; + this.outArray = new byte[sizeOfOutArray * 2]; + this.inOffset = new byte[alignment * 2]; + + this.inMaskHandle = GCHandle.Alloc(this.inMask, GCHandleType.Pinned); + this.inOffsetHandle = GCHandle.Alloc(this.inOffset, GCHandleType.Pinned); + this.inDataHandle = GCHandle.Alloc(this.inData, GCHandleType.Pinned); + this.outHandle = GCHandle.Alloc(this.outArray, GCHandleType.Pinned); + + this.alignment = (ulong)alignment; + + Unsafe.CopyBlockUnaligned(ref Unsafe.AsRef(inMaskArrayPtr), ref Unsafe.As<{Op1BaseType}, byte>(ref inMask[0]), (uint)sizeOfInMask); + Unsafe.CopyBlockUnaligned(ref Unsafe.AsRef(inDataArrayPtr), ref Unsafe.As<{Op1BaseType}, byte>(ref inData[0]), (uint)sizeOfInData); + Unsafe.CopyBlockUnaligned(ref Unsafe.AsRef(outArrayPtr), ref Unsafe.As<{Op1BaseType}, byte>(ref outArray[0]), (uint)sizeOfOutArray); + Unsafe.CopyBlockUnaligned(ref Unsafe.AsRef(inOffsetArrayPtr), ref Unsafe.As<{Op3BaseType}, byte>(ref inOffset[0]), (uint)sizeOfInOffset); + } + + public void* inMaskArrayPtr => Align((byte*)(inMaskHandle.AddrOfPinnedObject().ToPointer()), alignment); + public void* inOffsetArrayPtr => Align((byte*)(inOffsetHandle.AddrOfPinnedObject().ToPointer()), alignment); + public void* inDataArrayPtr => Align((byte*)(inDataHandle.AddrOfPinnedObject().ToPointer()), alignment); + public void* outArrayPtr => Align((byte*)(outHandle.AddrOfPinnedObject().ToPointer()), alignment); + + public void Dispose() + { + inMaskHandle.Free(); + inDataHandle.Free(); + inOffsetHandle.Free(); + outHandle.Free(); + } + + private static unsafe void* Align(byte* buffer, ulong expectedAlignment) + { + return (void*)(((ulong)buffer + expectedAlignment - 1) & ~(expectedAlignment - 1)); + } + + public void ResetOutArray() + { + for (int i = 0; i < this.outArray.Length; i++) + { + this.outArray[i] = 0; + } + } + } + + public static void MakeDistinct({Op3BaseType}[] input) + { + HashSet<{Op3BaseType}> dict = new (); + for(var i = 0; i < input.Length;) + { + if(!dict.Add(input[i])) + { + // value already exist + input[i] += 1; + continue; + } + i++; + } + } + + private struct TestStruct + { + public {Op1VectorType}<{Op1BaseType}> _mask; + public {Op1VectorType}<{Op1BaseType}> _data; + public {Op2BaseType}* _baseAddr; + public {Op3{Op1VectorType}Type}<{Op3BaseType}> _offset; + + public static TestStruct Create(DataTable _dataTable) + { + var testStruct = new TestStruct(); + + for (var i = 0; i < MaskElementCount; i++) { _maskArr[i] = {NextValueOp1}; } + for (var i = 0; i < DataElementCount; i++) { _dataArr[i] = {NextValueOp4}; } + _dataTable.ResetOutArray(); + + // Fill full of offsets into the data buffer. + for (var i = 0; i < OffsetElementCount; i++) { _offsetArr[i] = TestLibrary.Generator.Get{Op3BaseType}() % ({Op3BaseType})OutElementCount; } + MakeDistinct(_offsetArr); + + testStruct._baseAddr = ({Op2BaseType}*) _dataTable.outArrayPtr; + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1VectorType}<{Op1BaseType}>, byte>(ref testStruct._mask), ref Unsafe.As<{Op1BaseType}, byte>(ref _maskArr[0]), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1VectorType}<{Op1BaseType}>, byte>(ref testStruct._data), ref Unsafe.As<{Op1BaseType}, byte>(ref _dataArr[0]), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op3VectorType}<{Op3BaseType}>, byte>(ref testStruct._offset), ref Unsafe.As<{Op3BaseType}, byte>(ref _offsetArr[0]), (uint)Unsafe.SizeOf<{Op3VectorType}<{Op3BaseType}>>()); + + return testStruct; + } + + public void RunStructFldScenario(SveScatterVectorIndicesTest__{TestName} testClass) + { + {Isa}.{Method}(_mask, _baseAddr, _offset, _data); + testClass.ValidateResult(_mask, _data, _baseAddr, _offset); + } + } + + private static readonly int LargestVectorSize = 64; + + // A large enough buffer to hold many values. + // Values in address will point to locations within this buffer. + private static readonly int OutElementCount = 1024; + private static readonly int MaskElementCount = Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>() / sizeof({Op1BaseType}); + private static readonly int DataElementCount = Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>() / sizeof({Op1BaseType}); + private static readonly int OffsetElementCount = Unsafe.SizeOf<{Op3VectorType}<{Op3BaseType}>>() / sizeof({Op3BaseType}); + + private static {Op1BaseType}[] _maskArr = new {Op1BaseType}[MaskElementCount]; + private static {Op1BaseType}[] _dataArr = new {Op1BaseType}[DataElementCount]; + private static {Op3BaseType}[] _offsetArr = new {Op3BaseType}[OffsetElementCount]; + private static {Op1BaseType}[] _falseData = new {Op1BaseType}[DataElementCount]; + + private Vector<{Op1BaseType}> _mask; + private Vector<{Op1BaseType}> _data; + private Vector<{Op3BaseType}> _offset; + private Vector<{Op1BaseType}> _falseFld; + + private {Op2BaseType}* _baseAddr; + + private DataTable _dataTable; + + public SveScatterVectorIndicesTest__{TestName}() + { + Succeeded = true; + + for (var i = 0; i < MaskElementCount; i++) { _maskArr[i] = {NextValueOp1}; } + for (var i = 0; i < DataElementCount; i++) { _dataArr[i] = {NextValueOp4}; } + + // Fill full of offsets into the data buffer. They wil be expanded to full pointers inside the DataTable constructor. + for (var i = 0; i < OffsetElementCount; i++) { _offsetArr[i] = TestLibrary.Generator.Get{Op3BaseType}() % ({Op3BaseType})OutElementCount; } + MakeDistinct(_offsetArr); + + for (var i = 0; i < DataElementCount; i++) { _falseData[i] = {NextValueOp4}; } + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1VectorType}<{Op1BaseType}>, byte>(ref _falseFld), ref Unsafe.As<{Op1BaseType}, byte>(ref _falseData[0]), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + + _dataTable = new DataTable(_maskArr, _dataArr, new {Op1BaseType}[OutElementCount], _offsetArr, LargestVectorSize); + _baseAddr = ({Op2BaseType}*) _dataTable.outArrayPtr; + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1VectorType}<{Op1BaseType}>, byte>(ref _mask), ref Unsafe.As<{Op1BaseType}, byte>(ref _maskArr[0]), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1VectorType}<{Op1BaseType}>, byte>(ref _data), ref Unsafe.As<{Op1BaseType}, byte>(ref _dataArr[0]), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op3VectorType}<{Op3BaseType}>, byte>(ref _offset), ref Unsafe.As<{Op3BaseType}, byte>(ref _offsetArr[0]), (uint)Unsafe.SizeOf<{Op3VectorType}<{Op3BaseType}>>()); + } + + public bool IsSupported => {Isa}.IsSupported; + + public bool Succeeded { get; set; } + + public void RunBasicScenario_UnsafeRead() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunBasicScenario_UnsafeRead)); + + _dataTable.ResetOutArray(); + {Isa}.{Method}( + Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inMaskArrayPtr), + ({Op2BaseType}*) _dataTable.outArrayPtr, + Unsafe.Read<{Op3VectorType}<{Op3BaseType}>>(_dataTable.inOffsetArrayPtr), + Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inDataArrayPtr) + ); + + ValidateResult(_dataTable.inMaskArrayPtr, _dataTable.inDataArrayPtr, _dataTable.outArrayPtr, _dataTable.inOffsetArrayPtr); + + } + + public void RunBasicScenario_Load() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunBasicScenario_Load)); + + Vector<{Op1BaseType}> loadMask1 = {Isa}.CreateTrueMask{Op1BaseType}(SveMaskPattern.All); + Vector<{Op3BaseType}> loadMask2 = {Isa}.CreateTrueMask{Op3BaseType}(SveMaskPattern.All); + _dataTable.ResetOutArray(); + + {Isa}.{Method}( + {Isa}.LoadVector(loadMask1, ({Op1BaseType}*)(_dataTable.inMaskArrayPtr)), + ({Op2BaseType}*) _dataTable.outArrayPtr, + {Isa}.LoadVector(loadMask2, ({Op3BaseType}*)(_dataTable.inOffsetArrayPtr)), + {Isa}.LoadVector(loadMask1, ({Op1BaseType}*)(_dataTable.inDataArrayPtr)) + ); + + ValidateResult(_dataTable.inMaskArrayPtr, _dataTable.inDataArrayPtr, _dataTable.outArrayPtr, _dataTable.inOffsetArrayPtr); + } + + public void RunBasicScenario_FalseMask() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunBasicScenario_FalseMask)); + + Vector<{Op1BaseType}> falseMask = {Isa}.CreateFalseMask{Op1BaseType}(); + + _dataTable.ResetOutArray(); + + {Isa}.{Method}( + falseMask, + ({Op2BaseType}*) _dataTable.outArrayPtr, + Unsafe.Read<{Op1VectorType}<{Op3BaseType}>>(_dataTable.inOffsetArrayPtr), + Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inDataArrayPtr) + ); + + ValidateZeroResult(_dataTable.inDataArrayPtr, _dataTable.outArrayPtr, _dataTable.inOffsetArrayPtr); + } + + public void RunBasicScenario_NonFaulting() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunBasicScenario_NonFaulting)); + + Vector<{Op1BaseType}> falseMask = {Isa}.CreateFalseMask{Op1BaseType}(); + _dataTable.ResetOutArray(); + + try + { + {Isa}.{Method}( + falseMask, + ({Op2BaseType}*) _dataTable.outArrayPtr, + Vector<{Op3BaseType}>.Zero, + Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inDataArrayPtr) + ); + + ValidateZeroResult(_dataTable.inDataArrayPtr, _dataTable.outArrayPtr, _dataTable.inOffsetArrayPtr); + } + catch + { + Succeeded = false; + } + } + + public void RunReflectionScenario_UnsafeRead() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunReflectionScenario_UnsafeRead)); + + _dataTable.ResetOutArray(); + + typeof(Sve).GetMethod(nameof({Isa}.{Method}), new Type[] { typeof(Vector<{Op1BaseType}>), typeof({Op2BaseType}*), typeof(Vector<{Op3BaseType}>), typeof(Vector<{Op1BaseType}>) }) + .Invoke(null, new object[] { + Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inMaskArrayPtr), + Pointer.Box(_dataTable.outArrayPtr, typeof({Op2BaseType}*)), + Unsafe.Read<{Op3VectorType}<{Op3BaseType}>>(_dataTable.inOffsetArrayPtr), + Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inDataArrayPtr), + }); + + ValidateResult(_dataTable.inMaskArrayPtr, _dataTable.inDataArrayPtr, _dataTable.outArrayPtr, _dataTable.inOffsetArrayPtr); +} + +public void RunLclVarScenario_UnsafeRead() +{ + TestLibrary.TestFramework.BeginScenario(nameof(RunLclVarScenario_UnsafeRead)); + + var mask = Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inMaskArrayPtr); + var baseAddr = ({Op2BaseType}*) _dataTable.outArrayPtr; + var offset = Unsafe.Read<{Op3VectorType}<{Op3BaseType}>>(_dataTable.inOffsetArrayPtr); + var data = Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inDataArrayPtr); + + _dataTable.ResetOutArray(); + {Isa}.{Method}(mask, baseAddr, offset, data); + + ValidateResult(mask, data, baseAddr, offset); +} + +public void RunClassFldScenario() +{ + TestLibrary.TestFramework.BeginScenario(nameof(RunClassFldScenario)); + _dataTable.ResetOutArray(); + {Isa}.{Method}(_mask, _baseAddr, _offset, _data); + + ValidateResult(_mask, _data, _baseAddr, _offset); +} + +public void RunStructLclFldScenario() +{ + TestLibrary.TestFramework.BeginScenario(nameof(RunStructLclFldScenario)); + var test = TestStruct.Create(_dataTable); + + {Isa}.{Method}(test._mask, test._baseAddr, test._offset, test._data); + + ValidateResult(test._mask, test._data, test._baseAddr, test._offset); +} + +public void RunStructFldScenario() +{ + TestLibrary.TestFramework.BeginScenario(nameof(RunStructFldScenario)); + + var test = TestStruct.Create(_dataTable); + + test.RunStructFldScenario(this); +} + +public void RunUnsupportedScenario() +{ + TestLibrary.TestFramework.BeginScenario(nameof(RunUnsupportedScenario)); + + Succeeded = false; + + try + { + RunBasicScenario_Load(); + } + catch (PlatformNotSupportedException) + { + Succeeded = true; + } +} + +private void ValidateResult(Vector<{Op1BaseType}> mask, Vector<{Op1BaseType}> data, {Op2BaseType}* baseAddr, Vector<{Op3BaseType}> offset, [CallerMemberName] string method = "") +{ + {Op1BaseType}[] maskArray = new {Op1BaseType}[MaskElementCount]; + {Op1BaseType}[] dataArray = new {Op1BaseType}[DataElementCount]; + {Op3BaseType}[] offsetArray = new {Op3BaseType}[OffsetElementCount]; + + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref maskArray[0]), mask); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref dataArray[0]), data); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op3BaseType}, byte>(ref offsetArray[0]), offset); + + ValidateResult(maskArray, dataArray, baseAddr, offsetArray, method); +} + +private void ValidateResult(void* mask, void* data, void* baseAddr, void* offset, [CallerMemberName] string method = "") +{ + {Op1BaseType}[] maskArray = new {Op1BaseType}[MaskElementCount]; + {Op1BaseType}[] dataArray = new {Op1BaseType}[DataElementCount]; + {Op3BaseType}[] offsetArray = new {Op3BaseType}[OffsetElementCount]; + + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref maskArray[0]), ref Unsafe.AsRef(mask), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref dataArray[0]), ref Unsafe.AsRef(data), (uint)(Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>())); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op3BaseType}, byte>(ref offsetArray[0]), ref Unsafe.AsRef(offset), (uint)(Unsafe.SizeOf<{Op3VectorType}<{Op3BaseType}>>())); + + ValidateResult(maskArray, dataArray, ({Op2BaseType}*) baseAddr, offsetArray, method); +} + +private void ValidateResult({Op1BaseType}[] mask, {Op1BaseType}[] data, {Op2BaseType}* baseAddr, {Op3BaseType}[] offset, [CallerMemberName] string method = "") +{ + bool succeeded = true; + + var actualResult = new {Op1BaseType}[DataElementCount]; + for (var i = 0; i < DataElementCount; i++) + { + actualResult[i] = *({Op2BaseType}*)(baseAddr + offset[i]); + } + + for (var i = 0; i < DataElementCount; i++) + { + {Op1BaseType} expectedResult = mask[i] == 0 ? 0 : data[i]; + if (actualResult[i] != expectedResult) + { + succeeded = false; + break; + } + } + + if (!succeeded) + { + TestLibrary.TestFramework.LogInformation($"{nameof(Sve)}.{nameof({Isa}.{Method})}({Op1BaseType}, {Op2BaseType}*, {Op3BaseType}, {Op1BaseType}): {method} failed:"); + TestLibrary.TestFramework.LogInformation($" mask: ({string.Join(", ", mask)})"); + TestLibrary.TestFramework.LogInformation($" input: ({string.Join(", ", data)})"); + TestLibrary.TestFramework.LogInformation($"indices: ({string.Join(", ", offset)})"); + TestLibrary.TestFramework.LogInformation($" result: ({string.Join(", ", actualResult)})"); + TestLibrary.TestFramework.LogInformation(string.Empty); + + Succeeded = false; + } +} + +private void ValidateZeroResult(Vector<{Op1BaseType}> data, {Op2BaseType}* baseAddr, Vector<{Op3BaseType}> offset, [CallerMemberName] string method = "") +{ + {Op1BaseType}[] dataArray = new {Op1BaseType}[DataElementCount]; + {Op3BaseType}[] offsetArray = new {Op3BaseType}[OffsetElementCount]; + + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref dataArray[0]), data); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op3BaseType}, byte>(ref offsetArray[0]), offset); + + ValidateZeroResult(dataArray, baseAddr, offsetArray, method); +} + +private void ValidateZeroResult(void* data, void* baseAddr, void* offset, [CallerMemberName] string method = "") +{ + {Op1BaseType}[] dataArray = new {Op1BaseType}[DataElementCount]; + {Op3BaseType}[] offsetArray = new {Op3BaseType}[OffsetElementCount]; + + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref dataArray[0]), ref Unsafe.AsRef(data), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op3BaseType}, byte>(ref offsetArray[0]), ref Unsafe.AsRef(offset), (uint)(Unsafe.SizeOf<{Op3VectorType}<{Op3BaseType}>>())); + + ValidateZeroResult(dataArray, ({Op2BaseType}*) baseAddr, offsetArray, method); +} + +private void ValidateZeroResult({Op1BaseType}[] data, {Op2BaseType}* baseAddr, {Op3BaseType}[] offset, [CallerMemberName] string method = "") +{ + bool succeeded = true; + var actualResult = new {Op1BaseType}[DataElementCount]; + for (var i = 0; i < DataElementCount; i++) + { + actualResult[i] = *({Op2BaseType}*)(baseAddr + offset[i]); + } + + for (var i = 0; i < DataElementCount; i++) + { + if (actualResult[i] != 0) + { + succeeded = false; + break; + } + } + + if (!succeeded) + { + TestLibrary.TestFramework.LogInformation($"{nameof(Sve)}.{nameof({Isa}.{Method})}({Op1BaseType}, {Op2BaseType}*, {Op3BaseType}, {Op1BaseType}): {method} failed:"); + TestLibrary.TestFramework.LogInformation($" input: ({string.Join(", ", data)})"); + TestLibrary.TestFramework.LogInformation($"indices: ({string.Join(", ", offset)})"); + TestLibrary.TestFramework.LogInformation($" result: ({string.Join(", ", actualResult)})"); + TestLibrary.TestFramework.LogInformation(string.Empty); + + Succeeded = false; + } +} + } +} \ No newline at end of file