diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 9568aceac47f88..90d6453a018a97 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -3459,6 +3459,12 @@ class Compiler GenTree* gtNewSimdCreateSequenceNode( var_types type, GenTree* op1, GenTree* op2, var_types simdBaseType, unsigned simdSize); + GenTree* gtNewSimdCreateGeometricSequenceNode( + var_types type, GenTree* op1, GenTree* op2, var_types simdBaseType, unsigned simdSize); + + GenTree* gtNewSimdCreateAlternatingSequenceNode( + var_types type, GenTree* op1, GenTree* op2, var_types simdBaseType, unsigned simdSize); + GenTree* gtNewSimdDotProdNode(var_types type, GenTree* op1, GenTree* op2, @@ -3589,6 +3595,30 @@ class Compiler var_types simdBaseType, unsigned simdSize); + GenTree* gtNewSimdConcatNode(var_types type, + GenTree* op1, + GenTree* op2, + var_types simdBaseType, + unsigned simdSize, + bool leftUpper, + bool rightUpper); + + GenTree* gtNewSimdZipNode(var_types type, + GenTree* op1, + GenTree* op2, + var_types simdBaseType, + unsigned simdSize, + bool upper); + + GenTree* gtNewSimdUnzipNode(var_types type, + GenTree* op1, + GenTree* op2, + var_types simdBaseType, + unsigned simdSize, + bool odd); + + GenTree* gtNewSimdReverseNode(var_types type, GenTree* op1, var_types simdBaseType, unsigned simdSize); + GenTree* gtNewSimdRoundNode( var_types type, GenTree* op1, var_types simdBaseType, unsigned simdSize); diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index d91d1961bc2ba4..53a3b494bd196b 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -26768,6 +26768,854 @@ GenTree* Compiler::gtNewSimdNarrowNode( #endif // !TARGET_XARCH && !TARGET_ARM64 } +//---------------------------------------------------------------------------------------------- +// Compiler::gtNewSimdCreateGeometricSequenceNode: Creates a new simd CreateGeometricSequence node +// +// Arguments: +// type - The return type of SIMD node being created +// op1 - The initial value +// op2 - The multiplier value +// simdBaseType - The base type of SIMD type of the intrinsic +// simdSize - The size of the SIMD type of the intrinsic +// +// Returns: +// The created CreateGeometricSequence node +// +GenTree* Compiler::gtNewSimdCreateGeometricSequenceNode( + var_types type, GenTree* op1, GenTree* op2, var_types simdBaseType, unsigned simdSize) +{ + assert(varTypeIsSIMD(type)); + assert(getSIMDTypeForSize(simdSize) == type); + assert(varTypeIsArithmetic(simdBaseType)); + assert(op2->OperIsConst()); + + // op2 is expected to be constant. When op1 is also constant the whole sequence can be folded + // to a constant; otherwise build the constant multiplier vector and leave one broadcast+multiply. + + GenTreeVecCon* vecCon = gtNewVconNode(type); + uint32_t simdCount = getSIMDVectorLength(simdSize, simdBaseType); + bool isPartial = !op1->OperIsConst(); + + switch (simdBaseType) + { + case TYP_BYTE: + case TYP_UBYTE: + { + uint64_t initial = isPartial ? 1 : static_cast(op1->AsIntConCommon()->IntegralValue()); + uint64_t multiplier = static_cast(op2->AsIntConCommon()->IntegralValue()); + + for (uint32_t index = 0; index < simdCount; index++) + { + vecCon->gtSimdVal.u8[index] = static_cast(initial); + initial *= multiplier; + } + break; + } + + case TYP_SHORT: + case TYP_USHORT: + { + uint64_t initial = isPartial ? 1 : static_cast(op1->AsIntConCommon()->IntegralValue()); + uint64_t multiplier = static_cast(op2->AsIntConCommon()->IntegralValue()); + + for (uint32_t index = 0; index < simdCount; index++) + { + vecCon->gtSimdVal.u16[index] = static_cast(initial); + initial *= multiplier; + } + break; + } + + case TYP_INT: + case TYP_UINT: + { + uint64_t initial = isPartial ? 1 : static_cast(op1->AsIntConCommon()->IntegralValue()); + uint64_t multiplier = static_cast(op2->AsIntConCommon()->IntegralValue()); + + for (uint32_t index = 0; index < simdCount; index++) + { + vecCon->gtSimdVal.u32[index] = static_cast(initial); + initial *= multiplier; + } + break; + } + + case TYP_LONG: + case TYP_ULONG: + { + uint64_t initial = isPartial ? 1 : static_cast(op1->AsIntConCommon()->IntegralValue()); + uint64_t multiplier = static_cast(op2->AsIntConCommon()->IntegralValue()); + + for (uint32_t index = 0; index < simdCount; index++) + { + vecCon->gtSimdVal.u64[index] = initial; + initial *= multiplier; + } + break; + } + + case TYP_FLOAT: + { + float initial = isPartial ? 1.0f : static_cast(op1->AsDblCon()->DconValue()); + float multiplier = static_cast(op2->AsDblCon()->DconValue()); + + for (uint32_t index = 0; index < simdCount; index++) + { + vecCon->gtSimdVal.f32[index] = initial; + initial *= multiplier; + } + break; + } + + case TYP_DOUBLE: + { + double initial = isPartial ? 1.0 : op1->AsDblCon()->DconValue(); + double multiplier = op2->AsDblCon()->DconValue(); + + for (uint32_t index = 0; index < simdCount; index++) + { + vecCon->gtSimdVal.f64[index] = initial; + initial *= multiplier; + } + break; + } + + default: + { + unreached(); + } + } + + GenTree* result = vecCon; + + if (isPartial) + { + GenTree* initial = gtNewSimdCreateBroadcastNode(type, op1, simdBaseType, simdSize); + result = gtNewSimdBinOpNode(GT_MUL, type, result, initial, simdBaseType, simdSize); + } + + return result; +} + +//---------------------------------------------------------------------------------------------- +// Compiler::gtNewSimdCreateAlternatingSequenceNode: Creates a new simd CreateAlternatingSequence node +// +// Arguments: +// type - The return type of SIMD node being created +// op1 - The even-indexed value +// op2 - The odd-indexed value +// simdBaseType - The base type of SIMD type of the intrinsic +// simdSize - The size of the SIMD type of the intrinsic +// +// Returns: +// The created CreateAlternatingSequence node +// +GenTree* Compiler::gtNewSimdCreateAlternatingSequenceNode( + var_types type, GenTree* op1, GenTree* op2, var_types simdBaseType, unsigned simdSize) +{ + assert(varTypeIsSIMD(type)); + assert(getSIMDTypeForSize(simdSize) == type); + assert(varTypeIsArithmetic(simdBaseType)); + + // Fold constant pairs directly. Otherwise build two broadcasts and zip them, except where + // the target has a better way to broadcast the two-lane pattern directly. + + uint32_t simdCount = getSIMDVectorLength(simdSize, simdBaseType); + + if (simdCount == 1) + { + // Only the even-indexed value contributes to the result, but op2 still needs to be evaluated for side effects. + GenTree* result = gtNewSimdCreateBroadcastNode(type, op1, simdBaseType, simdSize); + GenTree* resultDup = fgMakeMultiUse(&result); + return gtNewOperNode(GT_COMMA, type, result, gtWrapWithSideEffects(resultDup, op2, GTF_ALL_EFFECT)); + } + + if (op1->OperIsConst() && op2->OperIsConst()) + { + GenTreeVecCon* vecCon = gtNewVconNode(type); + + switch (simdBaseType) + { + case TYP_BYTE: + case TYP_UBYTE: + { + uint64_t even = static_cast(op1->AsIntConCommon()->IntegralValue()); + uint64_t odd = static_cast(op2->AsIntConCommon()->IntegralValue()); + + for (uint32_t index = 0; index < simdCount; index++) + { + vecCon->gtSimdVal.u8[index] = static_cast(((index & 1) == 0) ? even : odd); + } + break; + } + + case TYP_SHORT: + case TYP_USHORT: + { + uint64_t even = static_cast(op1->AsIntConCommon()->IntegralValue()); + uint64_t odd = static_cast(op2->AsIntConCommon()->IntegralValue()); + + for (uint32_t index = 0; index < simdCount; index++) + { + vecCon->gtSimdVal.u16[index] = static_cast(((index & 1) == 0) ? even : odd); + } + break; + } + + case TYP_INT: + case TYP_UINT: + { + uint64_t even = static_cast(op1->AsIntConCommon()->IntegralValue()); + uint64_t odd = static_cast(op2->AsIntConCommon()->IntegralValue()); + + for (uint32_t index = 0; index < simdCount; index++) + { + vecCon->gtSimdVal.u32[index] = static_cast(((index & 1) == 0) ? even : odd); + } + break; + } + + case TYP_LONG: + case TYP_ULONG: + { + uint64_t even = static_cast(op1->AsIntConCommon()->IntegralValue()); + uint64_t odd = static_cast(op2->AsIntConCommon()->IntegralValue()); + + for (uint32_t index = 0; index < simdCount; index++) + { + vecCon->gtSimdVal.u64[index] = ((index & 1) == 0) ? even : odd; + } + break; + } + + case TYP_FLOAT: + { + double even = op1->AsDblCon()->DconValue(); + double odd = op2->AsDblCon()->DconValue(); + + for (uint32_t index = 0; index < simdCount; index++) + { + vecCon->gtSimdVal.f32[index] = static_cast(((index & 1) == 0) ? even : odd); + } + break; + } + + case TYP_DOUBLE: + { + double even = op1->AsDblCon()->DconValue(); + double odd = op2->AsDblCon()->DconValue(); + + for (uint32_t index = 0; index < simdCount; index++) + { + vecCon->gtSimdVal.f64[index] = ((index & 1) == 0) ? even : odd; + } + break; + } + + default: + { + unreached(); + } + } + + return vecCon; + } + +#if defined(TARGET_XARCH) + if (((simdBaseType == TYP_INT) || (simdBaseType == TYP_UINT)) && + ((compOpportunisticallyDependsOn(InstructionSet_AVX2) && ((simdSize == 16) || (simdSize == 32))) || + (compOpportunisticallyDependsOn(InstructionSet_AVX512) && (simdSize == 64)))) + { + // var pattern = Vector128.CreateScalarUnsafe(op1).WithElement(1, op2); + // return Broadcast(pattern.AsInt64()).As(); + + GenTree* pattern = gtNewSimdCreateScalarUnsafeNode(TYP_SIMD16, op1, simdBaseType, 16); + pattern = gtNewSimdWithElementNode(TYP_SIMD16, pattern, gtNewIconNode(1), op2, simdBaseType, 16); + + if (simdSize == 64) + { + return gtNewSimdHWIntrinsicNode(type, pattern, NI_AVX512_BroadcastPairScalarToVector512, simdBaseType, + simdSize); + } + + var_types broadcastBaseType = (simdBaseType == TYP_INT) ? TYP_LONG : TYP_ULONG; + NamedIntrinsic broadcast = + (simdSize == 16) ? NI_AVX2_BroadcastScalarToVector128 : NI_AVX2_BroadcastScalarToVector256; + return gtNewSimdHWIntrinsicNode(type, pattern, broadcast, broadcastBaseType, simdSize); + } +#endif // TARGET_XARCH + + GenTree* even = gtNewSimdCreateBroadcastNode(type, op1, simdBaseType, simdSize); + GenTree* odd = gtNewSimdCreateBroadcastNode(type, op2, simdBaseType, simdSize); + + return gtNewSimdZipNode(type, even, odd, simdBaseType, simdSize, false); +} + +//---------------------------------------------------------------------------------------------- +// Compiler::gtNewSimdConcatNode: Creates a new simd ConcatLowerLower/... node +// +// Arguments: +// type - The return type of SIMD node being created +// op1 - The vector that supplies the lower half +// op2 - The vector that supplies the upper half +// simdBaseType - The base type of SIMD type of the intrinsic +// simdSize - The size of the SIMD type of the intrinsic +// leftUpper - Whether the lower result half comes from the upper half of op1 +// rightUpper - Whether the upper result half comes from the upper half of op2 +// +// Returns: +// The created concat node +// +GenTree* Compiler::gtNewSimdConcatNode(var_types type, + GenTree* op1, + GenTree* op2, + var_types simdBaseType, + unsigned simdSize, + bool leftUpper, + bool rightUpper) +{ + assert(varTypeIsSIMD(type)); + assert(getSIMDTypeForSize(simdSize) == type); + assert(varTypeIsArithmetic(simdBaseType)); + + uint32_t simdCount = getSIMDVectorLength(simdSize, simdBaseType); + uint32_t lowerCount = (simdCount + 1) / 2; + + if (simdCount == 1) + { + GenTree* result = op1; + GenTree* resultDup = fgMakeMultiUse(&result); + return gtNewOperNode(GT_COMMA, type, result, gtWrapWithSideEffects(resultDup, op2, GTF_ALL_EFFECT)); + } + +#if defined(TARGET_ARM64) + if ((simdSize == 8) && (simdCount == 2) && + ((simdBaseType == TYP_INT) || (simdBaseType == TYP_UINT) || (simdBaseType == TYP_FLOAT))) + { + // var result = op1; + // if (leftUpper) + // { + // result = result.WithElement(0, result.GetElement(1)); + // } + // return result.WithElement(1, op2.GetElement(rightUpper ? 1 : 0)); + + GenTree* result = op1; + + if (leftUpper) + { + GenTree* resultDup = fgMakeMultiUse(&result); + result = gtNewSimdHWIntrinsicNode(type, result, gtNewIconNode(0), resultDup, gtNewIconNode(1), + NI_AdvSimd_Arm64_InsertSelectedScalar, simdBaseType, simdSize); + } + + return gtNewSimdHWIntrinsicNode(type, result, gtNewIconNode(1), op2, gtNewIconNode(rightUpper ? 1 : 0), + NI_AdvSimd_Arm64_InsertSelectedScalar, simdBaseType, simdSize); + } + + if (simdSize == 16) + { + // var result = op1.AsUInt64(); + // if (leftUpper) + // { + // result = result.WithElement(0, result.GetElement(1)); + // } + // return result.WithElement(1, op2.AsUInt64().GetElement(rightUpper ? 1 : 0)).As(); + + GenTree* result = op1; + + if (leftUpper) + { + GenTree* resultDup = fgMakeMultiUse(&result); + result = gtNewSimdHWIntrinsicNode(type, result, gtNewIconNode(0), resultDup, gtNewIconNode(1), + NI_AdvSimd_Arm64_InsertSelectedScalar, TYP_ULONG, simdSize); + } + + return gtNewSimdHWIntrinsicNode(type, result, gtNewIconNode(1), op2, gtNewIconNode(rightUpper ? 1 : 0), + NI_AdvSimd_Arm64_InsertSelectedScalar, TYP_ULONG, simdSize); + } +#endif // TARGET_ARM64 + +#if defined(TARGET_XARCH) + if (simdSize == 16) +#elif defined(TARGET_ARM64) + if (simdSize == 8) +#else +#error Unsupported platform +#endif // !TARGET_XARCH && !TARGET_ARM64 + { +#if defined(TARGET_XARCH) + if ((simdBaseType == TYP_INT) || (simdBaseType == TYP_UINT) || (simdBaseType == TYP_FLOAT)) + { + // return Sse.Shuffle(op1.AsSingle(), op2.AsSingle(), immediate).As(); + + uint32_t leftStart = leftUpper ? simdCount - lowerCount : 0; + uint32_t rightStart = rightUpper ? simdCount - lowerCount : 0; + unsigned immediate = 0; + + for (uint32_t index = 0; index < simdCount; index++) + { + uint32_t shuffleIndex = (index < lowerCount) ? leftStart + index : rightStart + index - lowerCount; + immediate |= shuffleIndex << (index * 2); + } + + return gtNewSimdHWIntrinsicNode(type, op1, op2, gtNewIconNode(immediate), NI_X86Base_Shuffle, TYP_FLOAT, + simdSize); + } +#endif // TARGET_XARCH + + // var tmp = op1.ToVectorUnsafe().WithUpper(op2); + // return Shuffle(tmp, indices).GetLower(); + + unsigned wideSimdSize = simdSize * 2; + var_types wideType = getSIMDTypeForSize(wideSimdSize); + GenTreeVecCon* shuffle = gtNewVconNode(wideType); + uint32_t wideCount = getSIMDVectorLength(wideSimdSize, simdBaseType); + uint32_t leftStart = leftUpper ? simdCount - lowerCount : 0; + uint32_t rightStart = rightUpper ? simdCount - lowerCount : 0; + + for (uint32_t index = 0; index < wideCount; index++) + { + uint32_t shuffleIndex = 0; + + if (index < simdCount) + { + shuffleIndex = (index < lowerCount) ? leftStart + index : simdCount + rightStart + index - lowerCount; + } + + switch (simdBaseType) + { + case TYP_BYTE: + case TYP_UBYTE: + shuffle->gtSimdVal.u8[index] = static_cast(shuffleIndex); + break; + + case TYP_SHORT: + case TYP_USHORT: + shuffle->gtSimdVal.u16[index] = static_cast(shuffleIndex); + break; + + case TYP_INT: + case TYP_UINT: + case TYP_FLOAT: + shuffle->gtSimdVal.u32[index] = shuffleIndex; + break; + + case TYP_LONG: + case TYP_ULONG: + case TYP_DOUBLE: + shuffle->gtSimdVal.u64[index] = shuffleIndex; + break; + + default: + unreached(); + } + } + + assert(IsValidForShuffle(shuffle, wideSimdSize, simdBaseType, nullptr, false)); + +#if defined(TARGET_XARCH) + GenTree* result = + gtNewSimdHWIntrinsicNode(wideType, op1, NI_Vector128_ToVector256Unsafe, simdBaseType, simdSize); +#elif defined(TARGET_ARM64) + GenTree* result = + gtNewSimdHWIntrinsicNode(wideType, op1, NI_Vector64_ToVector128Unsafe, simdBaseType, simdSize); +#else +#error Unsupported platform +#endif // !TARGET_XARCH && !TARGET_ARM64 + + result = gtNewSimdWithUpperNode(wideType, result, op2, simdBaseType, wideSimdSize); + result = gtNewSimdShuffleNode(wideType, result, shuffle, simdBaseType, wideSimdSize, false); + + return gtNewSimdGetLowerNode(type, result, simdBaseType, wideSimdSize); + } + + // var lower = leftUpper ? op1.GetUpper() : op1.GetLower(); + // var upper = rightUpper ? op2.GetUpper() : op2.GetLower(); + // return lower.ToVectorUnsafe().WithUpper(upper); + + var_types halfType = getSIMDTypeForSize(simdSize / 2); + GenTree* lower = leftUpper ? gtNewSimdGetUpperNode(halfType, op1, simdBaseType, simdSize) + : gtNewSimdGetLowerNode(halfType, op1, simdBaseType, simdSize); + GenTree* upper = rightUpper ? gtNewSimdGetUpperNode(halfType, op2, simdBaseType, simdSize) + : gtNewSimdGetLowerNode(halfType, op2, simdBaseType, simdSize); + +#if defined(TARGET_XARCH) + GenTree* result = + (simdSize == 32) + ? gtNewSimdHWIntrinsicNode(type, lower, NI_Vector128_ToVector256Unsafe, simdBaseType, simdSize / 2) + : gtNewSimdHWIntrinsicNode(type, lower, NI_Vector256_ToVector512Unsafe, simdBaseType, simdSize / 2); +#elif defined(TARGET_ARM64) + GenTree* result = gtNewSimdHWIntrinsicNode(type, lower, NI_Vector64_ToVector128Unsafe, simdBaseType, simdSize / 2); +#else +#error Unsupported platform +#endif // !TARGET_XARCH && !TARGET_ARM64 + + return gtNewSimdWithUpperNode(type, result, upper, simdBaseType, simdSize); +} + +//---------------------------------------------------------------------------------------------- +// Compiler::gtNewSimdZipNode: Creates a new simd ZipLower/ZipUpper node +// +// Arguments: +// type - The return type of SIMD node being created +// op1 - The vector that supplies even-indexed elements +// op2 - The vector that supplies odd-indexed elements +// simdBaseType - The base type of SIMD type of the intrinsic +// simdSize - The size of the SIMD type of the intrinsic +// upper - Whether to zip the upper halves +// +// Returns: +// The created zip node +// +GenTree* Compiler::gtNewSimdZipNode( + var_types type, GenTree* op1, GenTree* op2, var_types simdBaseType, unsigned simdSize, bool upper) +{ + assert(varTypeIsSIMD(type)); + assert(getSIMDTypeForSize(simdSize) == type); + assert(varTypeIsArithmetic(simdBaseType)); + + if (getSIMDVectorLength(simdSize, simdBaseType) == 1) + { + GenTree* result = op1; + GenTree* resultDup = fgMakeMultiUse(&result); + return gtNewOperNode(GT_COMMA, type, result, gtWrapWithSideEffects(resultDup, op2, GTF_ALL_EFFECT)); + } + +#if defined(TARGET_XARCH) + if (simdSize == 16) + { + NamedIntrinsic intrinsic = upper ? NI_X86Base_UnpackHigh : NI_X86Base_UnpackLow; + return gtNewSimdHWIntrinsicNode(type, op1, op2, intrinsic, simdBaseType, simdSize); + } + + if ((simdSize == 64) && ((simdBaseType == TYP_INT) || (simdBaseType == TYP_UINT)) && + compOpportunisticallyDependsOn(InstructionSet_AVX512)) + { + // var lower = Avx512F.UnpackLow(op1, op2); + // var upper = Avx512F.UnpackHigh(op1, op2); + // return Shuffle4x128(Shuffle4x128(lower, upper, lanes), ..., SHUFFLE_WYZX); + + GenTree* op1Dup = fgMakeMultiUse(&op1); + GenTree* op2Dup = fgMakeMultiUse(&op2); + + GenTree* lower = gtNewSimdHWIntrinsicNode(type, op1, op2, NI_AVX512_UnpackLow, simdBaseType, simdSize); + GenTree* higher = gtNewSimdHWIntrinsicNode(type, op1Dup, op2Dup, NI_AVX512_UnpackHigh, simdBaseType, simdSize); + + unsigned lanes = upper ? 0xEE : 0x44; + GenTree* result = gtNewSimdHWIntrinsicNode(type, lower, higher, gtNewIconNode(lanes), NI_AVX512_Shuffle4x128, + simdBaseType, simdSize); + GenTree* resultDup = fgMakeMultiUse(&result); + + return gtNewSimdHWIntrinsicNode(type, result, resultDup, gtNewIconNode(SHUFFLE_WYZX), NI_AVX512_Shuffle4x128, + simdBaseType, simdSize); + } +#elif defined(TARGET_ARM64) + NamedIntrinsic intrinsic = upper ? NI_AdvSimd_Arm64_ZipHigh : NI_AdvSimd_Arm64_ZipLow; + return gtNewSimdHWIntrinsicNode(type, op1, op2, intrinsic, simdBaseType, simdSize); +#else +#error Unsupported platform +#endif // !TARGET_XARCH && !TARGET_ARM64 + +#if defined(TARGET_XARCH) + // var lower = Zip(left, right, upper: false); + // var upper = Zip(left, right, upper: true); + // return lower.ToVectorUnsafe().WithUpper(upper); + + var_types halfType = getSIMDTypeForSize(simdSize / 2); + GenTree* left = upper ? gtNewSimdGetUpperNode(halfType, op1, simdBaseType, simdSize) + : gtNewSimdGetLowerNode(halfType, op1, simdBaseType, simdSize); + GenTree* right = upper ? gtNewSimdGetUpperNode(halfType, op2, simdBaseType, simdSize) + : gtNewSimdGetLowerNode(halfType, op2, simdBaseType, simdSize); + + GenTree* leftDup = fgMakeMultiUse(&left); + GenTree* rightDup = fgMakeMultiUse(&right); + + GenTree* lower = gtNewSimdZipNode(halfType, left, right, simdBaseType, simdSize / 2, false); + GenTree* higher = gtNewSimdZipNode(halfType, leftDup, rightDup, simdBaseType, simdSize / 2, true); + + GenTree* result = + (simdSize == 32) + ? gtNewSimdHWIntrinsicNode(type, lower, NI_Vector128_ToVector256Unsafe, simdBaseType, simdSize / 2) + : gtNewSimdHWIntrinsicNode(type, lower, NI_Vector256_ToVector512Unsafe, simdBaseType, simdSize / 2); + return gtNewSimdWithUpperNode(type, result, higher, simdBaseType, simdSize); +#endif // TARGET_XARCH +} + +//---------------------------------------------------------------------------------------------- +// Compiler::gtNewSimdUnzipNode: Creates a new simd UnzipEven/UnzipOdd node +// +// Arguments: +// type - The return type of SIMD node being created +// op1 - The vector that supplies the lower half +// op2 - The vector that supplies the upper half +// simdBaseType - The base type of SIMD type of the intrinsic +// simdSize - The size of the SIMD type of the intrinsic +// odd - Whether to unzip odd-indexed elements +// +// Returns: +// The created unzip node +// +GenTree* Compiler::gtNewSimdUnzipNode( + var_types type, GenTree* op1, GenTree* op2, var_types simdBaseType, unsigned simdSize, bool odd) +{ + assert(varTypeIsSIMD(type)); + assert(getSIMDTypeForSize(simdSize) == type); + assert(varTypeIsArithmetic(simdBaseType)); + + uint32_t simdCount = getSIMDVectorLength(simdSize, simdBaseType); + + if (simdCount == 1) + { + if (odd) + { + GenTree* result = gtWrapWithSideEffects(gtNewZeroConNode(type), op2, GTF_ALL_EFFECT); + return gtWrapWithSideEffects(result, op1, GTF_ALL_EFFECT); + } + + GenTree* result = op1; + GenTree* resultDup = fgMakeMultiUse(&result); + return gtNewOperNode(GT_COMMA, type, result, gtWrapWithSideEffects(resultDup, op2, GTF_ALL_EFFECT)); + } + +#if defined(TARGET_ARM64) + // return odd ? AdvSimd.Arm64.UnzipOdd(op1, op2) : AdvSimd.Arm64.UnzipEven(op1, op2); + + NamedIntrinsic intrinsic = odd ? NI_AdvSimd_Arm64_UnzipOdd : NI_AdvSimd_Arm64_UnzipEven; + return gtNewSimdHWIntrinsicNode(type, op1, op2, intrinsic, simdBaseType, simdSize); +#elif defined(TARGET_XARCH) + if (simdSize == 16) + { + if ((simdBaseType == TYP_INT) || (simdBaseType == TYP_UINT) || (simdBaseType == TYP_FLOAT)) + { + // return Sse.Shuffle(op1.AsSingle(), op2.AsSingle(), odd ? 0xDD : 0x88).As(); + + return gtNewSimdHWIntrinsicNode(type, op1, op2, gtNewIconNode(odd ? 0xDD : 0x88), NI_X86Base_Shuffle, + TYP_FLOAT, simdSize); + } + + // var tmp = op1.ToVectorUnsafe().WithUpper(op2); + // return Shuffle(tmp, indices).GetLower(); + + unsigned wideSimdSize = simdSize * 2; + var_types wideType = getSIMDTypeForSize(wideSimdSize); + GenTreeVecCon* shuffle = gtNewVconNode(wideType); + uint32_t wideCount = getSIMDVectorLength(wideSimdSize, simdBaseType); + uint32_t start = odd ? 1 : 0; + uint32_t lowerCount = (simdCount - start + 1) / 2; + + for (uint32_t index = 0; index < wideCount; index++) + { + uint32_t shuffleIndex = 0; + + if (index < simdCount) + { + shuffleIndex = + (index < lowerCount) ? start + (index * 2) : simdCount + start + ((index - lowerCount) * 2); + } + + switch (simdBaseType) + { + case TYP_BYTE: + case TYP_UBYTE: + shuffle->gtSimdVal.u8[index] = static_cast(shuffleIndex); + break; + + case TYP_SHORT: + case TYP_USHORT: + shuffle->gtSimdVal.u16[index] = static_cast(shuffleIndex); + break; + + case TYP_INT: + case TYP_UINT: + case TYP_FLOAT: + shuffle->gtSimdVal.u32[index] = shuffleIndex; + break; + + case TYP_LONG: + case TYP_ULONG: + case TYP_DOUBLE: + shuffle->gtSimdVal.u64[index] = shuffleIndex; + break; + + default: + unreached(); + } + } + + assert(IsValidForShuffle(shuffle, wideSimdSize, simdBaseType, nullptr, false)); + + GenTree* result = + gtNewSimdHWIntrinsicNode(wideType, op1, NI_Vector128_ToVector256Unsafe, simdBaseType, simdSize); + result = gtNewSimdWithUpperNode(wideType, result, op2, simdBaseType, wideSimdSize); + result = gtNewSimdShuffleNode(wideType, result, shuffle, simdBaseType, wideSimdSize, false); + + return gtNewSimdGetLowerNode(type, result, simdBaseType, wideSimdSize); + } + + if ((simdSize == 32) && ((simdBaseType == TYP_INT) || (simdBaseType == TYP_UINT)) && + compOpportunisticallyDependsOn(InstructionSet_AVX2)) + { + // var left = Shuffle(op1, indices); + // var right = Shuffle(op2, indices); + // return left.GetLower().ToVector256Unsafe().WithUpper(right.GetLower()); + + GenTreeVecCon* shuffle = gtNewVconNode(type); + uint32_t start = odd ? 1 : 0; + + for (uint32_t index = 0; index < simdCount; index++) + { + shuffle->gtSimdVal.u32[index] = start + (2 * (index % (simdCount / 2))); + } + + assert(IsValidForShuffle(shuffle, simdSize, simdBaseType, nullptr, false)); + + GenTree* leftShuffle = shuffle; + GenTree* rightShuffle = gtCloneExpr(shuffle); + GenTree* left = gtNewSimdShuffleNode(type, op1, leftShuffle, simdBaseType, simdSize, false); + GenTree* right = gtNewSimdShuffleNode(type, op2, rightShuffle, simdBaseType, simdSize, false); + + var_types halfType = getSIMDTypeForSize(simdSize / 2); + GenTree* lower = gtNewSimdGetLowerNode(halfType, left, simdBaseType, simdSize); + GenTree* upper = gtNewSimdGetLowerNode(halfType, right, simdBaseType, simdSize); + + GenTree* result = + gtNewSimdHWIntrinsicNode(type, lower, NI_Vector128_ToVector256Unsafe, simdBaseType, simdSize / 2); + return gtNewSimdWithUpperNode(type, result, upper, simdBaseType, simdSize); + } + + if ((simdSize == 64) && ((simdBaseType == TYP_INT) || (simdBaseType == TYP_UINT)) && + compOpportunisticallyDependsOn(InstructionSet_AVX512)) + { + // return Avx512F.PermuteVar16x32x2(op1, indices, op2); + + GenTreeVecCon* shuffle = gtNewVconNode(type); + uint32_t start = odd ? 1 : 0; + + for (uint32_t index = 0; index < simdCount; index++) + { + shuffle->gtSimdVal.u32[index] = + (index < (simdCount / 2)) ? start + (2 * index) : simdCount + start + (2 * (index - (simdCount / 2))); + } + + return gtNewSimdHWIntrinsicNode(type, op1, shuffle, op2, NI_AVX512_PermuteVar16x32x2, simdBaseType, simdSize); + } + + // var lower = Unzip(op1.GetLower(), op1.GetUpper()); + // var upper = Unzip(op2.GetLower(), op2.GetUpper()); + // return lower.ToVectorUnsafe().WithUpper(upper); + + GenTree* op1Dup = fgMakeMultiUse(&op1); + GenTree* op2Dup = fgMakeMultiUse(&op2); + + var_types halfType = getSIMDTypeForSize(simdSize / 2); + GenTree* op1Lower = gtNewSimdGetLowerNode(halfType, op1, simdBaseType, simdSize); + GenTree* op1Upper = gtNewSimdGetUpperNode(halfType, op1Dup, simdBaseType, simdSize); + GenTree* op2Lower = gtNewSimdGetLowerNode(halfType, op2, simdBaseType, simdSize); + GenTree* op2Upper = gtNewSimdGetUpperNode(halfType, op2Dup, simdBaseType, simdSize); + + GenTree* lower = gtNewSimdUnzipNode(halfType, op1Lower, op1Upper, simdBaseType, simdSize / 2, odd); + GenTree* higher = gtNewSimdUnzipNode(halfType, op2Lower, op2Upper, simdBaseType, simdSize / 2, odd); + + GenTree* result = + (simdSize == 32) + ? gtNewSimdHWIntrinsicNode(type, lower, NI_Vector128_ToVector256Unsafe, simdBaseType, simdSize / 2) + : gtNewSimdHWIntrinsicNode(type, lower, NI_Vector256_ToVector512Unsafe, simdBaseType, simdSize / 2); + return gtNewSimdWithUpperNode(type, result, higher, simdBaseType, simdSize); +#else +#error Unsupported platform +#endif // !TARGET_XARCH && !TARGET_ARM64 +} + +//---------------------------------------------------------------------------------------------- +// Compiler::gtNewSimdReverseNode: Creates a new simd Reverse node +// +// Arguments: +// type - The return type of SIMD node being created +// op1 - The vector to reverse +// simdBaseType - The base type of SIMD type of the intrinsic +// simdSize - The size of the SIMD type of the intrinsic +// +// Returns: +// The created reverse node +// +GenTree* Compiler::gtNewSimdReverseNode(var_types type, GenTree* op1, var_types simdBaseType, unsigned simdSize) +{ + assert(varTypeIsSIMD(type)); + assert(getSIMDTypeForSize(simdSize) == type); + assert(varTypeIsArithmetic(simdBaseType)); + + uint32_t simdCount = getSIMDVectorLength(simdSize, simdBaseType); + + if (simdCount == 1) + { + return op1; + } + +#if defined(TARGET_XARCH) + if ((simdSize == 32) && ((simdBaseType == TYP_INT) || (simdBaseType == TYP_UINT)) && + compOpportunisticallyDependsOn(InstructionSet_AVX2)) + { + // var tmp = Avx2.Shuffle(op1, SHUFFLE_XYZW); + // return Avx2.Permute2x128(tmp, tmp, 1); + + GenTree* reverseInLane = + gtNewSimdHWIntrinsicNode(type, op1, gtNewIconNode(SHUFFLE_XYZW), NI_AVX2_Shuffle, simdBaseType, simdSize); + GenTree* reverseInLaneDup = fgMakeMultiUse(&reverseInLane); + + return gtNewSimdHWIntrinsicNode(type, reverseInLane, reverseInLaneDup, gtNewIconNode(1), NI_AVX2_Permute2x128, + simdBaseType, simdSize); + } +#elif defined(TARGET_ARM64) + if ((simdSize == 8) && ((simdBaseType == TYP_INT) || (simdBaseType == TYP_UINT) || (simdBaseType == TYP_FLOAT))) + { + // return AdvSimd.ReverseElement32(op1.AsInt64()).As(); + + return gtNewSimdHWIntrinsicNode(type, op1, NI_AdvSimd_ReverseElement32, TYP_LONG, simdSize); + } +#endif // TARGET_XARCH || TARGET_ARM64 + + // return Shuffle(op1, indices); + + GenTreeVecCon* shuffle = gtNewVconNode(type); + + for (uint32_t index = 0; index < simdCount; index++) + { + uint32_t shuffleIndex = simdCount - 1 - index; + + switch (simdBaseType) + { + case TYP_BYTE: + case TYP_UBYTE: + shuffle->gtSimdVal.u8[index] = static_cast(shuffleIndex); + break; + + case TYP_SHORT: + case TYP_USHORT: + shuffle->gtSimdVal.u16[index] = static_cast(shuffleIndex); + break; + + case TYP_INT: + case TYP_UINT: + case TYP_FLOAT: + shuffle->gtSimdVal.u32[index] = shuffleIndex; + break; + + case TYP_LONG: + case TYP_ULONG: + case TYP_DOUBLE: + shuffle->gtSimdVal.u64[index] = shuffleIndex; + break; + + default: + unreached(); + } + } + + assert(IsValidForShuffle(shuffle, simdSize, simdBaseType, nullptr, false)); + + return gtNewSimdShuffleNode(type, op1, shuffle, simdBaseType, simdSize, false); +} + //------------------------------------------------------------------------ // gtNewSimdRoundNode: Creates a new simd Round node // diff --git a/src/coreclr/jit/hwintrinsicarm64.cpp b/src/coreclr/jit/hwintrinsicarm64.cpp index 7275c2ffe4f305..4a897fea6750a3 100644 --- a/src/coreclr/jit/hwintrinsicarm64.cpp +++ b/src/coreclr/jit/hwintrinsicarm64.cpp @@ -1334,6 +1334,47 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, break; } + case NI_Vector64_CreateGeometricSequence: + case NI_Vector128_CreateGeometricSequence: + { + assert(sig->numArgs == 2); + + if (!impStackTop(0).val->OperIsConst()) + { + break; + } + + if (varTypeIsLong(simdBaseType) && !impStackTop(1).val->OperIsConst() && (simdSize != 8)) + { + // TODO-ARM64-CQ: We should support long/ulong multiplication. + break; + } + + impSpillSideEffect(true, stackState.esStackDepth - + 2 DEBUGARG("Spilling op1 side effects for vector CreateGeometricSequence")); + + op2 = impPopStack().val; + op1 = impPopStack().val; + + retNode = gtNewSimdCreateGeometricSequenceNode(retType, op1, op2, simdBaseType, simdSize); + break; + } + + case NI_Vector64_CreateAlternatingSequence: + case NI_Vector128_CreateAlternatingSequence: + { + assert(sig->numArgs == 2); + + impSpillSideEffect(true, stackState.esStackDepth - + 2 DEBUGARG("Spilling op1 side effects for vector CreateAlternatingSequence")); + + op2 = impPopStack().val; + op1 = impPopStack().val; + + retNode = gtNewSimdCreateAlternatingSequenceNode(retType, op1, op2, simdBaseType, simdSize); + break; + } + case NI_Vector64_CreateScalarUnsafe: case NI_Vector128_CreateScalarUnsafe: { @@ -1519,6 +1560,20 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, break; } + case NI_Vector64_get_SignSequence: + case NI_Vector128_get_SignSequence: + { + assert(sig->numArgs == 0); + + var_types scalarType = genActualType(simdBaseType); + GenTree* one = gtNewOneConNode(scalarType); + GenTree* negativeOne = varTypeIsFloating(simdBaseType) ? gtNewDconNode(-1.0, simdBaseType) + : gtNewAllBitsSetConNode(scalarType); + + retNode = gtNewSimdCreateAlternatingSequenceNode(retType, one, negativeOne, simdBaseType, simdSize); + break; + } + case NI_Vector64_get_NaN: case NI_Vector128_get_NaN: { @@ -2844,6 +2899,71 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, break; } + case NI_Vector64_ConcatLowerLower: + case NI_Vector128_ConcatLowerLower: + case NI_Vector64_ConcatLowerUpper: + case NI_Vector128_ConcatLowerUpper: + case NI_Vector64_ConcatUpperLower: + case NI_Vector128_ConcatUpperLower: + case NI_Vector64_ConcatUpperUpper: + case NI_Vector128_ConcatUpperUpper: + { + assert(sig->numArgs == 2); + + op2 = impSIMDPopStack(); + op1 = impSIMDPopStack(); + + bool leftUpper = + (intrinsic == NI_Vector64_ConcatUpperLower) || (intrinsic == NI_Vector128_ConcatUpperLower) || + (intrinsic == NI_Vector64_ConcatUpperUpper) || (intrinsic == NI_Vector128_ConcatUpperUpper); + bool rightUpper = + (intrinsic == NI_Vector64_ConcatLowerUpper) || (intrinsic == NI_Vector128_ConcatLowerUpper) || + (intrinsic == NI_Vector64_ConcatUpperUpper) || (intrinsic == NI_Vector128_ConcatUpperUpper); + + retNode = gtNewSimdConcatNode(retType, op1, op2, simdBaseType, simdSize, leftUpper, rightUpper); + break; + } + + case NI_Vector64_ZipLower: + case NI_Vector128_ZipLower: + case NI_Vector64_ZipUpper: + case NI_Vector128_ZipUpper: + { + assert(sig->numArgs == 2); + + op2 = impSIMDPopStack(); + op1 = impSIMDPopStack(); + + bool upper = (intrinsic == NI_Vector64_ZipUpper) || (intrinsic == NI_Vector128_ZipUpper); + retNode = gtNewSimdZipNode(retType, op1, op2, simdBaseType, simdSize, upper); + break; + } + + case NI_Vector64_UnzipEven: + case NI_Vector128_UnzipEven: + case NI_Vector64_UnzipOdd: + case NI_Vector128_UnzipOdd: + { + assert(sig->numArgs == 2); + + op2 = impSIMDPopStack(); + op1 = impSIMDPopStack(); + + bool odd = (intrinsic == NI_Vector64_UnzipOdd) || (intrinsic == NI_Vector128_UnzipOdd); + retNode = gtNewSimdUnzipNode(retType, op1, op2, simdBaseType, simdSize, odd); + break; + } + + case NI_Vector64_Reverse: + case NI_Vector128_Reverse: + { + assert(sig->numArgs == 1); + + op1 = impSIMDPopStack(); + retNode = gtNewSimdReverseNode(retType, op1, simdBaseType, simdSize); + break; + } + case NI_Vector64_op_ExclusiveOr: case NI_Vector128_op_ExclusiveOr: { diff --git a/src/coreclr/jit/hwintrinsiclistarm64.h b/src/coreclr/jit/hwintrinsiclistarm64.h index b127b61920d4b7..7d261d855a00e8 100644 --- a/src/coreclr/jit/hwintrinsiclistarm64.h +++ b/src/coreclr/jit/hwintrinsiclistarm64.h @@ -33,6 +33,10 @@ HARDWARE_INTRINSIC(Vector64, AsUInt16, HARDWARE_INTRINSIC(Vector64, AsUInt32, 8, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector64, AsUInt64, 8, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector64, Ceiling, 8, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector64, ConcatLowerLower, 8, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector64, ConcatLowerUpper, 8, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector64, ConcatUpperLower, 8, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector64, ConcatUpperUpper, 8, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector64, ConditionalSelect, 8, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector64, ConvertToDouble, 8, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector64, ConvertToInt32, 8, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) @@ -45,6 +49,8 @@ HARDWARE_INTRINSIC(Vector64, ConvertToUInt32Native, HARDWARE_INTRINSIC(Vector64, ConvertToUInt64, 8, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector64, ConvertToUInt64Native, 8, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector64, Create, 8, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen) +HARDWARE_INTRINSIC(Vector64, CreateAlternatingSequence, 8, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector64, CreateGeometricSequence, 8, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector64, CreateScalar, 8, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen) HARDWARE_INTRINSIC(Vector64, CreateScalarUnsafe, 8, 1, {INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_invalid, INS_invalid, INS_fmov, INS_invalid}, HW_Category_SIMD, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_SupportsContainment) HARDWARE_INTRINSIC(Vector64, CreateSequence, 8, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) @@ -96,6 +102,7 @@ HARDWARE_INTRINSIC(Vector64, MinNumber, HARDWARE_INTRINSIC(Vector64, MultiplyAddEstimate, 8, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector64, Narrow, 8, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector64, NarrowWithSaturation, 8, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector64, Reverse, 8, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector64, Round, 8, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector64, ShiftLeft, 8, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector64, Shuffle, 8, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_CanBenefitFromConstantProp) @@ -111,9 +118,13 @@ HARDWARE_INTRINSIC(Vector64, ToScalar, HARDWARE_INTRINSIC(Vector64, ToVector128, 8, 1, {INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov}, HW_Category_SIMD, HW_Flag_SpecialCodeGen) HARDWARE_INTRINSIC(Vector64, ToVector128Unsafe, 8, 1, {INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov}, HW_Category_SIMD, HW_Flag_SpecialCodeGen) HARDWARE_INTRINSIC(Vector64, Truncate, 8, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector64, UnzipEven, 8, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector64, UnzipOdd, 8, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector64, WidenLower, 8, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector64, WidenUpper, 8, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector64, WithElement, 8, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Vector64, ZipLower, 8, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector64, ZipUpper, 8, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector64, get_AllBitsSet, 8, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector64, get_E, 8, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector64, get_Epsilon, 8, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) @@ -125,6 +136,7 @@ HARDWARE_INTRINSIC(Vector64, get_NegativeZero, HARDWARE_INTRINSIC(Vector64, get_One, 8, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector64, get_Pi, 8, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector64, get_PositiveInfinity, 8, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector64, get_SignSequence, 8, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector64, get_Tau, 8, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector64, get_Zero, 8, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector64, op_Addition, 8, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) @@ -173,6 +185,10 @@ HARDWARE_INTRINSIC(Vector128, AsVector2, HARDWARE_INTRINSIC(Vector128, AsVector3, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_mov, INS_invalid}, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) HARDWARE_INTRINSIC(Vector128, AsVector4, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector128, Ceiling, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector128, ConcatLowerLower, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector128, ConcatLowerUpper, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector128, ConcatUpperLower, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector128, ConcatUpperUpper, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, ConditionalSelect, 16, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, ConvertToDouble, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector128, ConvertToInt32, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) @@ -185,6 +201,8 @@ HARDWARE_INTRINSIC(Vector128, ConvertToUInt32Native, HARDWARE_INTRINSIC(Vector128, ConvertToUInt64, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector128, ConvertToUInt64Native, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector128, Create, 16, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen) +HARDWARE_INTRINSIC(Vector128, CreateAlternatingSequence, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector128, CreateGeometricSequence, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, CreateScalar, 16, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen) HARDWARE_INTRINSIC(Vector128, CreateScalarUnsafe, 16, 1, {INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_fmov, INS_fmov}, HW_Category_SIMD, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_SupportsContainment) HARDWARE_INTRINSIC(Vector128, CreateSequence, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) @@ -238,6 +256,7 @@ HARDWARE_INTRINSIC(Vector128, MinNumber, HARDWARE_INTRINSIC(Vector128, MultiplyAddEstimate, 16, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, Narrow, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, NarrowWithSaturation, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector128, Reverse, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, Round, 16, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, ShiftLeft, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, Shuffle, 16, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_CanBenefitFromConstantProp) @@ -251,11 +270,15 @@ HARDWARE_INTRINSIC(Vector128, SubtractSaturate, HARDWARE_INTRINSIC(Vector128, Sum, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector128, ToScalar, 16, 1, {INS_smov, INS_umov, INS_smov, INS_umov, INS_smov, INS_umov, INS_umov, INS_umov, INS_dup, INS_dup}, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen) HARDWARE_INTRINSIC(Vector128, Truncate, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector128, UnzipEven, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector128, UnzipOdd, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, WidenLower, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector128, WidenUpper, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector128, WithElement, 16, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport) HARDWARE_INTRINSIC(Vector128, WithLower, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen) HARDWARE_INTRINSIC(Vector128, WithUpper, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen) +HARDWARE_INTRINSIC(Vector128, ZipLower, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector128, ZipUpper, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, get_AllBitsSet, 16, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, get_E, 16, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, get_Epsilon, 16, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) @@ -267,6 +290,7 @@ HARDWARE_INTRINSIC(Vector128, get_NegativeZero, HARDWARE_INTRINSIC(Vector128, get_One, 16, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, get_Pi, 16, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, get_PositiveInfinity, 16, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector128, get_SignSequence, 16, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, get_Tau, 16, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, get_Zero, 16, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, op_Addition, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) diff --git a/src/coreclr/jit/hwintrinsiclistxarch.h b/src/coreclr/jit/hwintrinsiclistxarch.h index de2322872c60fc..bcbc4a0b0927f1 100644 --- a/src/coreclr/jit/hwintrinsiclistxarch.h +++ b/src/coreclr/jit/hwintrinsiclistxarch.h @@ -51,6 +51,10 @@ HARDWARE_INTRINSIC(Vector128, AsVector2, HARDWARE_INTRINSIC(Vector128, AsVector3, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movups, INS_invalid}, -1, 1, HW_Category_SimpleSIMD, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) HARDWARE_INTRINSIC(Vector128, AsVector4, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector128, Ceiling, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector128, ConcatLowerLower, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector128, ConcatLowerUpper, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector128, ConcatUpperLower, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector128, ConcatUpperUpper, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, ConditionalSelect, 16, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen) HARDWARE_INTRINSIC(Vector128, ConvertToDouble, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector128, ConvertToInt32, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) @@ -63,6 +67,8 @@ HARDWARE_INTRINSIC(Vector128, ConvertToUInt32Native, HARDWARE_INTRINSIC(Vector128, ConvertToUInt64, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector128, ConvertToUInt64Native, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector128, Create, 16, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen) +HARDWARE_INTRINSIC(Vector128, CreateAlternatingSequence, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector128, CreateGeometricSequence, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, CreateScalar, 16, 1, {INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd64, INS_movd64, INS_movss, INS_movsd_simd}, -1, -1, HW_Category_SIMDScalar, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_NoRMWSemantics) HARDWARE_INTRINSIC(Vector128, CreateScalarUnsafe, 16, 1, {INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd64, INS_movd64, INS_movss, INS_movsd_simd}, -1, -1, HW_Category_SIMDScalar, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_NoRMWSemantics) HARDWARE_INTRINSIC(Vector128, CreateSequence, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) @@ -114,6 +120,7 @@ HARDWARE_INTRINSIC(Vector128, MinNumber, HARDWARE_INTRINSIC(Vector128, MultiplyAddEstimate, 16, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, Narrow, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, NarrowWithSaturation, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Vector128, Reverse, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, Round, 16, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, ShiftLeft, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, Shuffle, 16, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_CanBenefitFromConstantProp) @@ -130,9 +137,13 @@ HARDWARE_INTRINSIC(Vector128, ToVector256, HARDWARE_INTRINSIC(Vector128, ToVector256Unsafe, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqu32, INS_movdqu32, INS_movdqu32, INS_movdqu32, INS_movups, INS_movupd}, -1, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_NormalizeSmallTypeToInt) HARDWARE_INTRINSIC(Vector128, ToVector512, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqu32, INS_movdqu32, INS_vmovdqu64, INS_vmovdqu64, INS_movups, INS_movupd}, -1, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_NormalizeSmallTypeToInt) HARDWARE_INTRINSIC(Vector128, Truncate, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector128, UnzipEven, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector128, UnzipOdd, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, WidenLower, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector128, WidenUpper, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector128, WithElement, 16, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoContainment|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Vector128, ZipLower, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector128, ZipUpper, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, get_AllBitsSet, 16, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, get_E, 16, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, get_Epsilon, 16, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) @@ -144,6 +155,7 @@ HARDWARE_INTRINSIC(Vector128, get_NegativeZero, HARDWARE_INTRINSIC(Vector128, get_One, 16, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, get_Pi, 16, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, get_PositiveInfinity, 16, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector128, get_SignSequence, 16, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, get_Tau, 16, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, get_Zero, 16, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, op_Addition, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) @@ -188,6 +200,10 @@ HARDWARE_INTRINSIC(Vector256, AsUInt64, HARDWARE_INTRINSIC(Vector256, AsVector, 32, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) HARDWARE_INTRINSIC(Vector256, AsVector256, 32, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) HARDWARE_INTRINSIC(Vector256, Ceiling, 32, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector256, ConcatLowerLower, 32, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector256, ConcatLowerUpper, 32, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector256, ConcatUpperLower, 32, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector256, ConcatUpperUpper, 32, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) HARDWARE_INTRINSIC(Vector256, ConditionalSelect, 32, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_AvxOnlyCompatible) HARDWARE_INTRINSIC(Vector256, ConvertToDouble, 32, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector256, ConvertToInt32, 32, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) @@ -200,6 +216,8 @@ HARDWARE_INTRINSIC(Vector256, ConvertToUInt32Native, HARDWARE_INTRINSIC(Vector256, ConvertToUInt64, 32, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector256, ConvertToUInt64Native, 32, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector256, Create, 32, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector256, CreateAlternatingSequence, 32, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector256, CreateGeometricSequence, 32, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) HARDWARE_INTRINSIC(Vector256, CreateScalar, 32, 1, {INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd64, INS_movd64, INS_movss, INS_movsd_simd}, -1, -1, HW_Category_SIMDScalar, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_AvxOnlyCompatible) HARDWARE_INTRINSIC(Vector256, CreateScalarUnsafe, 32, 1, {INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd64, INS_movd64, INS_movss, INS_movsd_simd}, -1, -1, HW_Category_SIMDScalar, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_AvxOnlyCompatible) HARDWARE_INTRINSIC(Vector256, CreateSequence, 32, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) @@ -253,6 +271,7 @@ HARDWARE_INTRINSIC(Vector256, MinNumber, HARDWARE_INTRINSIC(Vector256, MultiplyAddEstimate, 32, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector256, Narrow, 32, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector256, NarrowWithSaturation, 32, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Vector256, Reverse, 32, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector256, Round, 32, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector256, ShiftLeft, 32, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector256, Shuffle, 32, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_CanBenefitFromConstantProp) @@ -268,11 +287,15 @@ HARDWARE_INTRINSIC(Vector256, ToScalar, HARDWARE_INTRINSIC(Vector256, ToVector512, 32, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqu32, INS_movdqu32, INS_vmovdqu64, INS_vmovdqu64, INS_movups, INS_movupd}, -1, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NormalizeSmallTypeToInt) HARDWARE_INTRINSIC(Vector256, ToVector512Unsafe, 32, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqu32, INS_movdqu32, INS_vmovdqu64, INS_vmovdqu64, INS_movups, INS_movupd}, -1, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NormalizeSmallTypeToInt) HARDWARE_INTRINSIC(Vector256, Truncate, 32, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector256, UnzipEven, 32, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector256, UnzipOdd, 32, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector256, WidenLower, 32, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector256, WidenUpper, 32, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector256, WithElement, 32, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoContainment|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) HARDWARE_INTRINSIC(Vector256, WithLower, 32, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_AvxOnlyCompatible) HARDWARE_INTRINSIC(Vector256, WithUpper, 32, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector256, ZipLower, 32, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector256, ZipUpper, 32, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector256, get_AllBitsSet, 32, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector256, get_E, 32, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector256, get_Epsilon, 32, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) @@ -284,6 +307,7 @@ HARDWARE_INTRINSIC(Vector256, get_NegativeZero, HARDWARE_INTRINSIC(Vector256, get_One, 32, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector256, get_Pi, 32, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector256, get_PositiveInfinity, 32, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector256, get_SignSequence, 32, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector256, get_Tau, 32, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector256, get_Zero, 32, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector256, op_Addition, 32, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) @@ -328,6 +352,10 @@ HARDWARE_INTRINSIC(Vector512, AsUInt64, HARDWARE_INTRINSIC(Vector512, AsVector, 64, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector512, AsVector512, 64, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector512, Ceiling, 64, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector512, ConcatLowerLower, 64, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector512, ConcatLowerUpper, 64, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector512, ConcatUpperLower, 64, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector512, ConcatUpperUpper, 64, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector512, ConditionalSelect, 64, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen) HARDWARE_INTRINSIC(Vector512, ConvertToDouble, 64, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector512, ConvertToInt32, 64, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) @@ -340,6 +368,8 @@ HARDWARE_INTRINSIC(Vector512, ConvertToUInt32Native, HARDWARE_INTRINSIC(Vector512, ConvertToUInt64, 64, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector512, ConvertToUInt64Native, 64, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector512, Create, 64, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen) +HARDWARE_INTRINSIC(Vector512, CreateAlternatingSequence, 64, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector512, CreateGeometricSequence, 64, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector512, CreateScalar, 64, 1, {INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd64, INS_movd64, INS_movss, INS_movsd_simd}, -1, -1, HW_Category_SIMDScalar, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen) HARDWARE_INTRINSIC(Vector512, CreateScalarUnsafe, 64, 1, {INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd64, INS_movd64, INS_movss, INS_movsd_simd}, -1, -1, HW_Category_SIMDScalar, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen) HARDWARE_INTRINSIC(Vector512, CreateSequence, 64, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) @@ -394,6 +424,7 @@ HARDWARE_INTRINSIC(Vector512, MinNumber, HARDWARE_INTRINSIC(Vector512, MultiplyAddEstimate, 64, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector512, Narrow, 64, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector512, NarrowWithSaturation, 64, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Vector512, Reverse, 64, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector512, Round, 64, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector512, ShiftLeft, 64, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector512, Shuffle, 64, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_CanBenefitFromConstantProp) @@ -407,11 +438,15 @@ HARDWARE_INTRINSIC(Vector512, SubtractSaturate, HARDWARE_INTRINSIC(Vector512, Sum, 64, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector512, ToScalar, 64, 1, {INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd64, INS_movd64, INS_movss, INS_movsd_simd}, -1, -1, HW_Category_SIMDScalar, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector512, Truncate, 64, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector512, UnzipEven, 64, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector512, UnzipOdd, 64, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector512, WidenLower, 64, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector512, WidenUpper, 64, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector512, WithElement, 64, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoContainment|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector512, WithLower, 64, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen) HARDWARE_INTRINSIC(Vector512, WithUpper, 64, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen) +HARDWARE_INTRINSIC(Vector512, ZipLower, 64, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector512, ZipUpper, 64, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector512, get_AllBitsSet, 64, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector512, get_E, 64, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector512, get_Epsilon, 64, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) @@ -423,6 +458,7 @@ HARDWARE_INTRINSIC(Vector512, get_NegativeZero, HARDWARE_INTRINSIC(Vector512, get_One, 64, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector512, get_Pi, 64, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector512, get_PositiveInfinity, 64, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector512, get_SignSequence, 64, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector512, get_Tau, 64, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector512, get_Zero, 64, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector512, op_Addition, 64, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) diff --git a/src/coreclr/jit/hwintrinsicxarch.cpp b/src/coreclr/jit/hwintrinsicxarch.cpp index ef52964d430eac..9f392260f9f0ec 100644 --- a/src/coreclr/jit/hwintrinsicxarch.cpp +++ b/src/coreclr/jit/hwintrinsicxarch.cpp @@ -2258,6 +2258,57 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, break; } + case NI_Vector128_CreateGeometricSequence: + case NI_Vector256_CreateGeometricSequence: + case NI_Vector512_CreateGeometricSequence: + { + assert(sig->numArgs == 2); + + if (!impStackTop(0).val->OperIsConst()) + { + break; + } + + if (!impStackTop(1).val->OperIsConst() && (simdSize == 32) && varTypeIsIntegral(simdBaseType) && + !compOpportunisticallyDependsOn(InstructionSet_AVX2)) + { + // We can't deal with TYP_SIMD32 for integral types if the compiler doesn't support AVX2 + break; + } + + impSpillSideEffect(true, stackState.esStackDepth - + 2 DEBUGARG("Spilling op1 side effects for vector CreateGeometricSequence")); + + op2 = impPopStack().val; + op1 = impPopStack().val; + + retNode = gtNewSimdCreateGeometricSequenceNode(retType, op1, op2, simdBaseType, simdSize); + break; + } + + case NI_Vector128_CreateAlternatingSequence: + case NI_Vector256_CreateAlternatingSequence: + case NI_Vector512_CreateAlternatingSequence: + { + assert(sig->numArgs == 2); + + if ((!impStackTop(1).val->OperIsConst() || !impStackTop(0).val->OperIsConst()) && (simdSize == 32) && + varTypeIsIntegral(simdBaseType) && !compOpportunisticallyDependsOn(InstructionSet_AVX2)) + { + // We can't deal with TYP_SIMD32 for integral types if the compiler doesn't support AVX2 + break; + } + + impSpillSideEffect(true, stackState.esStackDepth - + 2 DEBUGARG("Spilling op1 side effects for vector CreateAlternatingSequence")); + + op2 = impPopStack().val; + op1 = impPopStack().val; + + retNode = gtNewSimdCreateAlternatingSequenceNode(retType, op1, op2, simdBaseType, simdSize); + break; + } + case NI_Vector128_op_Division: case NI_Vector256_op_Division: case NI_Vector512_op_Division: @@ -2545,6 +2596,21 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, break; } + case NI_Vector128_get_SignSequence: + case NI_Vector256_get_SignSequence: + case NI_Vector512_get_SignSequence: + { + assert(sig->numArgs == 0); + + var_types scalarType = genActualType(simdBaseType); + GenTree* one = gtNewOneConNode(scalarType); + GenTree* negativeOne = varTypeIsFloating(simdBaseType) ? gtNewDconNode(-1.0, simdBaseType) + : gtNewAllBitsSetConNode(scalarType); + + retNode = gtNewSimdCreateAlternatingSequenceNode(retType, one, negativeOne, simdBaseType, simdSize); + break; + } + case NI_Vector128_get_NaN: case NI_Vector256_get_NaN: case NI_Vector512_get_NaN: @@ -4167,6 +4233,135 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, break; } + case NI_Vector128_ConcatLowerLower: + case NI_Vector256_ConcatLowerLower: + case NI_Vector512_ConcatLowerLower: + case NI_Vector128_ConcatLowerUpper: + case NI_Vector256_ConcatLowerUpper: + case NI_Vector512_ConcatLowerUpper: + case NI_Vector128_ConcatUpperLower: + case NI_Vector256_ConcatUpperLower: + case NI_Vector512_ConcatUpperLower: + case NI_Vector128_ConcatUpperUpper: + case NI_Vector256_ConcatUpperUpper: + case NI_Vector512_ConcatUpperUpper: + { + assert(sig->numArgs == 2); + + if (simdSize == 16) + { + bool supportsX86BaseShuffle = + (simdBaseType == TYP_INT) || (simdBaseType == TYP_UINT) || (simdBaseType == TYP_FLOAT); + + if (!supportsX86BaseShuffle && !compOpportunisticallyDependsOn(InstructionSet_AVX2)) + { + break; + } + } + + op2 = impSIMDPopStack(); + op1 = impSIMDPopStack(); + + bool leftUpper = + (intrinsic == NI_Vector128_ConcatUpperLower) || (intrinsic == NI_Vector256_ConcatUpperLower) || + (intrinsic == NI_Vector512_ConcatUpperLower) || (intrinsic == NI_Vector128_ConcatUpperUpper) || + (intrinsic == NI_Vector256_ConcatUpperUpper) || (intrinsic == NI_Vector512_ConcatUpperUpper); + bool rightUpper = + (intrinsic == NI_Vector128_ConcatLowerUpper) || (intrinsic == NI_Vector256_ConcatLowerUpper) || + (intrinsic == NI_Vector512_ConcatLowerUpper) || (intrinsic == NI_Vector128_ConcatUpperUpper) || + (intrinsic == NI_Vector256_ConcatUpperUpper) || (intrinsic == NI_Vector512_ConcatUpperUpper); + + retNode = gtNewSimdConcatNode(retType, op1, op2, simdBaseType, simdSize, leftUpper, rightUpper); + break; + } + + case NI_Vector128_ZipLower: + case NI_Vector256_ZipLower: + case NI_Vector512_ZipLower: + case NI_Vector128_ZipUpper: + case NI_Vector256_ZipUpper: + case NI_Vector512_ZipUpper: + { + assert(sig->numArgs == 2); + + if ((simdSize == 32) && varTypeIsIntegral(simdBaseType) && + !compOpportunisticallyDependsOn(InstructionSet_AVX2)) + { + break; + } + + op2 = impSIMDPopStack(); + op1 = impSIMDPopStack(); + + bool upper = (intrinsic == NI_Vector128_ZipUpper) || (intrinsic == NI_Vector256_ZipUpper) || + (intrinsic == NI_Vector512_ZipUpper); + retNode = gtNewSimdZipNode(retType, op1, op2, simdBaseType, simdSize, upper); + break; + } + + case NI_Vector128_UnzipEven: + case NI_Vector256_UnzipEven: + case NI_Vector512_UnzipEven: + case NI_Vector128_UnzipOdd: + case NI_Vector256_UnzipOdd: + case NI_Vector512_UnzipOdd: + { + assert(sig->numArgs == 2); + + if (simdSize == 16) + { + bool supportsX86BaseShuffle = + (simdBaseType == TYP_INT) || (simdBaseType == TYP_UINT) || (simdBaseType == TYP_FLOAT); + + if (!supportsX86BaseShuffle && !compOpportunisticallyDependsOn(InstructionSet_AVX2)) + { + break; + } + } + else if (simdSize > 16) + { + if (!compOpportunisticallyDependsOn(varTypeIsFloating(simdBaseType) ? InstructionSet_AVX + : InstructionSet_AVX2)) + { + break; + } + } + + op2 = impSIMDPopStack(); + op1 = impSIMDPopStack(); + + bool odd = (intrinsic == NI_Vector128_UnzipOdd) || (intrinsic == NI_Vector256_UnzipOdd) || + (intrinsic == NI_Vector512_UnzipOdd); + retNode = gtNewSimdUnzipNode(retType, op1, op2, simdBaseType, simdSize, odd); + break; + } + + case NI_Vector128_Reverse: + case NI_Vector256_Reverse: + case NI_Vector512_Reverse: + { + assert(sig->numArgs == 1); + + if (simdSize == 32) + { + if (!compOpportunisticallyDependsOn(varTypeIsFloating(simdBaseType) ? InstructionSet_AVX + : InstructionSet_AVX2)) + { + break; + } + } + + if ((simdSize == 64) && varTypeIsByte(simdBaseType) && + !compOpportunisticallyDependsOn(InstructionSet_AVX512v2)) + { + break; + } + + op1 = impSIMDPopStack(); + retNode = gtNewSimdReverseNode(retType, op1, simdBaseType, simdSize); + break; + } + case NI_Vector128_op_ExclusiveOr: case NI_Vector256_op_ExclusiveOr: case NI_Vector512_op_ExclusiveOr: diff --git a/src/libraries/System.Numerics.Vectors/ref/System.Numerics.Vectors.cs b/src/libraries/System.Numerics.Vectors/ref/System.Numerics.Vectors.cs index 350862a4af099a..86f95e57f1f981 100644 --- a/src/libraries/System.Numerics.Vectors/ref/System.Numerics.Vectors.cs +++ b/src/libraries/System.Numerics.Vectors/ref/System.Numerics.Vectors.cs @@ -312,6 +312,20 @@ public static partial class Vector public static System.Numerics.Vector CreateScalar(T value) { throw null; } public static System.Numerics.Vector CreateScalarUnsafe(T value) { throw null; } public static System.Numerics.Vector CreateSequence(T start, T step) { throw null; } + public static System.Numerics.Vector CreateGeometricSequence(T initial, [System.Diagnostics.CodeAnalysis.ConstantExpected] T multiplier) { throw null; } + public static System.Numerics.Vector CreateAlternatingSequence(T even, T odd) { throw null; } + public static System.Numerics.Vector CreateHarmonicSequence(T start, T step) { throw null; } + public static System.Numerics.Vector ZipLower(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector ZipUpper(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static (System.Numerics.Vector Lower, System.Numerics.Vector Upper) Zip(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector UnzipEven(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector UnzipOdd(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static (System.Numerics.Vector Even, System.Numerics.Vector Odd) Unzip(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector ConcatLowerLower(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector ConcatUpperLower(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector ConcatUpperUpper(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector ConcatLowerUpper(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector Reverse(System.Numerics.Vector vector) { throw null; } public static System.Numerics.Vector DegreesToRadians(System.Numerics.Vector degrees) { throw null; } public static System.Numerics.Vector DegreesToRadians(System.Numerics.Vector degrees) { throw null; } public static System.Numerics.Vector Divide(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } @@ -590,6 +604,7 @@ public static partial class Vector extension(Vector) where T : System.Numerics.ISignedNumber { public static System.Numerics.Vector NegativeOne { get { throw null; } } + public static System.Numerics.Vector SignSequence { get { throw null; } } } } public partial struct Vector2 : System.IEquatable, System.IFormattable diff --git a/src/libraries/System.Numerics.Vectors/tests/GenericVectorTests.cs b/src/libraries/System.Numerics.Vectors/tests/GenericVectorTests.cs index f18f9ff9278419..51d03a502c9011 100644 --- a/src/libraries/System.Numerics.Vectors/tests/GenericVectorTests.cs +++ b/src/libraries/System.Numerics.Vectors/tests/GenericVectorTests.cs @@ -4531,6 +4531,222 @@ private static void TestCreateSequence(T start, T step) } } + [Fact] + public void CreateGeometricSequenceInt32Test() + { + Vector sequence = Vector.CreateGeometricSequence(1, 2); + int expected = 1; + + for (int index = 0; index < Vector.Count; index++) + { + Assert.Equal(expected, sequence.GetElement(index)); + expected *= 2; + } + } + + [Fact] + public void CreateGeometricSequenceByteWrapsTest() + { + Vector sequence = Vector.CreateGeometricSequence((byte)200, (byte)2); + byte expected = 200; + + for (int index = 0; index < Vector.Count; index++) + { + Assert.Equal(expected, sequence.GetElement(index)); + expected = unchecked((byte)(expected * 2)); + } + } + + [Fact] + public void CreateGeometricSequenceSingleNonConstantInitialTest() + { + const float multiplier = 1.0064822f; + float initial = GetNonConstant(1.0059024f); + Vector sequence = Vector.CreateGeometricSequence(initial, multiplier); + float expected = initial; + + for (int index = 0; index < Vector.Count; index++) + { + AssertExtensions.Equal(expected, sequence.GetElement(index), 1e-6f); + expected *= multiplier; + } + } + + [Fact] + public void CreateGeometricSequenceDoubleNonConstantInitialTest() + { + const double multiplier = 1e-50; + double initial = GetNonConstant(1e-154); + Vector sequence = Vector.CreateGeometricSequence(initial, multiplier); + double expected = initial; + + for (int index = 0; index < Vector.Count; index++) + { + AssertExtensions.Equal(expected, sequence.GetElement(index), 1e-12); + expected *= multiplier; + } + } + + [Fact] + public void CreateAlternatingSequenceInt32Test() + { + Vector sequence = Vector.CreateAlternatingSequence(5, -5); + + for (int index = 0; index < Vector.Count; index++) + { + Assert.Equal(((index & 1) == 0) ? 5 : -5, sequence.GetElement(index)); + } + } + + [Fact] + public void CreateAlternatingSequenceUInt32Test() + { + Vector sequence = Vector.CreateAlternatingSequence(5u, uint.MaxValue - 1u); + + for (int index = 0; index < Vector.Count; index++) + { + Assert.Equal(((index & 1) == 0) ? 5u : uint.MaxValue - 1u, sequence.GetElement(index)); + } + } + + [Fact] + public void CreateAlternatingSequenceDoubleTest() + { + Vector sequence = Vector.CreateAlternatingSequence(1.5, -2.5); + + for (int index = 0; index < Vector.Count; index++) + { + Assert.Equal(((index & 1) == 0) ? 1.5 : -2.5, sequence.GetElement(index)); + } + } + + [Fact] + public void CreateHarmonicSequenceInt32Test() + { + Vector sequence = Vector.CreateHarmonicSequence(1, 1); + int expected = 1; + + for (int index = 0; index < Vector.Count; index++) + { + Assert.Equal(1 / expected, sequence.GetElement(index)); + expected += 1; + } + } + + [Fact] + public void CreateHarmonicSequenceSingleTest() + { + Vector sequence = Vector.CreateHarmonicSequence(1.0f, 1.0f); + float expected = 1.0f; + + for (int index = 0; index < Vector.Count; index++) + { + AssertExtensions.Equal(1.0f / expected, sequence.GetElement(index), 1e-6f); + expected += 1.0f; + } + } + + [Fact] + public void CreateHarmonicSequenceDoubleTest() + { + Vector sequence = Vector.CreateHarmonicSequence(1.0, 1.0); + double expected = 1.0; + + for (int index = 0; index < Vector.Count; index++) + { + AssertExtensions.Equal(1.0 / expected, sequence.GetElement(index), 1e-15); + expected += 1.0; + } + } + + [Fact] + public void SignSequenceInt32Test() + { + Vector sequence = Vector.SignSequence; + + for (int index = 0; index < Vector.Count; index++) + { + Assert.Equal(((index & 1) == 0) ? 1 : -1, sequence.GetElement(index)); + } + } + + [Fact] + public void SignSequenceSingleTest() + { + Vector sequence = Vector.SignSequence; + + for (int index = 0; index < Vector.Count; index++) + { + Assert.Equal(((index & 1) == 0) ? 1.0f : -1.0f, sequence.GetElement(index)); + } + } + + [Fact] + public void SignSequenceDoubleTest() + { + Vector sequence = Vector.SignSequence; + + for (int index = 0; index < Vector.Count; index++) + { + Assert.Equal(((index & 1) == 0) ? 1.0 : -1.0, sequence.GetElement(index)); + } + } + + [Fact] + public void LaneOperationsInt32Test() + { + Vector left = Vector.CreateSequence(0, 1); + Vector right = Vector.CreateSequence(100, 1); + int count = Vector.Count; + int lowerCount = (count + 1) / 2; + int upperStart = count - lowerCount; + + AssertVectorEqual(CreateVector(index => ((index & 1) == 0) ? left.GetElement(index / 2) : right.GetElement(index / 2)), Vector.ZipLower(left, right)); + AssertVectorEqual(CreateVector(index => ((index & 1) == 0) ? left.GetElement(upperStart + (index / 2)) : right.GetElement(upperStart + (index / 2))), Vector.ZipUpper(left, right)); + + (Vector lower, Vector upper) = Vector.Zip(left, right); + AssertVectorEqual(Vector.ZipLower(left, right), lower); + AssertVectorEqual(Vector.ZipUpper(left, right), upper); + + AssertVectorEqual(left, Vector.UnzipEven(lower, upper)); + AssertVectorEqual(right, Vector.UnzipOdd(lower, upper)); + + (Vector even, Vector odd) = Vector.Unzip(lower, upper); + AssertVectorEqual(left, even); + AssertVectorEqual(right, odd); + + AssertVectorEqual(CreateVector(index => (index < lowerCount) ? left.GetElement(index) : right.GetElement(index - lowerCount)), Vector.ConcatLowerLower(left, right)); + AssertVectorEqual(CreateVector(index => (index < lowerCount) ? left.GetElement(upperStart + index) : right.GetElement(index - lowerCount)), Vector.ConcatUpperLower(left, right)); + AssertVectorEqual(CreateVector(index => (index < lowerCount) ? left.GetElement(upperStart + index) : right.GetElement(upperStart + index - lowerCount)), Vector.ConcatUpperUpper(left, right)); + AssertVectorEqual(CreateVector(index => (index < lowerCount) ? left.GetElement(index) : right.GetElement(upperStart + index - lowerCount)), Vector.ConcatLowerUpper(left, right)); + + AssertVectorEqual(CreateVector(index => left.GetElement(count - 1 - index)), Vector.Reverse(left)); + } + + private static Vector CreateVector(Func elementSelector) + { + int[] values = new int[Vector.Count]; + + for (int index = 0; index < values.Length; index++) + { + values[index] = elementSelector(index); + } + + return new Vector(values); + } + + private static void AssertVectorEqual(Vector expected, Vector actual) + where T : struct + { + for (int index = 0; index < Vector.Count; index++) + { + Assert.Equal(expected.GetElement(index), actual.GetElement(index)); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static T GetNonConstant(T value) => value; + [Theory] [MemberData(nameof(GenericMathTestMemberData.CosDouble), MemberType = typeof(GenericMathTestMemberData))] public void CosDoubleTest(double value, double expectedResult, double variance) diff --git a/src/libraries/System.Private.CoreLib/src/System/Numerics/Vector.cs b/src/libraries/System.Private.CoreLib/src/System/Numerics/Vector.cs index ae25ce85d2431b..608449f9832774 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Numerics/Vector.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Numerics/Vector.cs @@ -100,6 +100,14 @@ public static Vector NegativeOne [Intrinsic] get => Create(T.NegativeOne); } + + /// + public static Vector SignSequence + { + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => CreateAlternatingSequence(T.One, T.NegativeOne); + } } /// Computes the absolute value of each element in a vector. @@ -879,6 +887,295 @@ public static Vector CreateScalarUnsafe(T value) [MethodImpl(MethodImplOptions.AggressiveInlining)] public static Vector CreateSequence(T start, T step) => (Vector.Indices * step) + Create(start); + /// Creates a new instance where the elements begin at a specified value and are multiplied by another specified value. + /// The type of the elements in the vector. + /// The value that element 0 will be initialized to. + /// The value that indicates how each element should be scaled from the previous. + /// A new instance with the first element initialized to and each subsequent element initialized to the value of the previous element multiplied by . + /// The type of and () is not supported. + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector CreateGeometricSequence(T initial, [ConstantExpected] T multiplier) + { + if (Vector.Count == Vector512.Count) + { + return Vector512.CreateGeometricSequence(initial, multiplier).AsVector(); + } + else if (Vector.Count == Vector256.Count) + { + return Vector256.CreateGeometricSequence(initial, multiplier).AsVector(); + } + else + { + Debug.Assert(Vector.Count == Vector128.Count); + return Vector128.CreateGeometricSequence(initial, multiplier).AsVector(); + } + } + + /// Creates a new instance whose elements alternate between two specified values. + /// The type of the elements in the vector. + /// The value assigned to even-indexed elements. + /// The value assigned to odd-indexed elements. + /// A new instance whose even-indexed elements are initialized to and odd-indexed elements are initialized to . + /// The type of and () is not supported. + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector CreateAlternatingSequence(T even, T odd) + { + Unsafe.SkipInit(out Vector result); + + for (int index = 0; index < Vector.Count; index += 2) + { + result.SetElementUnsafe(index, even); + result.SetElementUnsafe(index + 1, odd); + } + + return result; + } + + /// Creates a new instance whose elements are the reciprocal of an arithmetic sequence. + /// The type of the elements in the vector. + /// The value that element 0 of the arithmetic sequence will be initialized to. + /// The value that indicates how far apart each element of the arithmetic sequence should be from the previous. + /// A new instance whose elements are initialized to one divided by the corresponding element of the arithmetic sequence. + /// The type of and () is not supported. + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector CreateHarmonicSequence(T start, T step) => Vector.One / CreateSequence(start, step); + + /// Creates a new vector by concatenating the lower halves of two vectors. + /// The type of the elements in the vector. + /// The vector that provides the lower half of the result. + /// The vector that provides the upper half of the result. + /// A new vector whose lower half comes from the lower half of and whose upper half comes from the lower half of . + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector ConcatLowerLower(Vector left, Vector right) + { + if (Vector.Count == Vector512.Count) + { + return Vector512.ConcatLowerLower(left.AsVector512(), right.AsVector512()).AsVector(); + } + else if (Vector.Count == Vector256.Count) + { + return Vector256.ConcatLowerLower(left.AsVector256(), right.AsVector256()).AsVector(); + } + else + { + Debug.Assert(Vector.Count == Vector128.Count); + return Vector128.ConcatLowerLower(left.AsVector128(), right.AsVector128()).AsVector(); + } + } + + /// Creates a new vector by concatenating the upper half of one vector and the lower half of another vector. + /// The type of the elements in the vector. + /// The vector that provides the lower half of the result. + /// The vector that provides the upper half of the result. + /// A new vector whose lower half comes from the upper half of and whose upper half comes from the lower half of . + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector ConcatUpperLower(Vector left, Vector right) + { + if (Vector.Count == Vector512.Count) + { + return Vector512.ConcatUpperLower(left.AsVector512(), right.AsVector512()).AsVector(); + } + else if (Vector.Count == Vector256.Count) + { + return Vector256.ConcatUpperLower(left.AsVector256(), right.AsVector256()).AsVector(); + } + else + { + Debug.Assert(Vector.Count == Vector128.Count); + return Vector128.ConcatUpperLower(left.AsVector128(), right.AsVector128()).AsVector(); + } + } + + /// Creates a new vector by concatenating the upper halves of two vectors. + /// The type of the elements in the vector. + /// The vector that provides the lower half of the result. + /// The vector that provides the upper half of the result. + /// A new vector whose lower half comes from the upper half of and whose upper half comes from the upper half of . + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector ConcatUpperUpper(Vector left, Vector right) + { + if (Vector.Count == Vector512.Count) + { + return Vector512.ConcatUpperUpper(left.AsVector512(), right.AsVector512()).AsVector(); + } + else if (Vector.Count == Vector256.Count) + { + return Vector256.ConcatUpperUpper(left.AsVector256(), right.AsVector256()).AsVector(); + } + else + { + Debug.Assert(Vector.Count == Vector128.Count); + return Vector128.ConcatUpperUpper(left.AsVector128(), right.AsVector128()).AsVector(); + } + } + + /// Creates a new vector by concatenating the lower half of one vector and the upper half of another vector. + /// The type of the elements in the vector. + /// The vector that provides the lower half of the result. + /// The vector that provides the upper half of the result. + /// A new vector whose lower half comes from the lower half of and whose upper half comes from the upper half of . + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector ConcatLowerUpper(Vector left, Vector right) + { + if (Vector.Count == Vector512.Count) + { + return Vector512.ConcatLowerUpper(left.AsVector512(), right.AsVector512()).AsVector(); + } + else if (Vector.Count == Vector256.Count) + { + return Vector256.ConcatLowerUpper(left.AsVector256(), right.AsVector256()).AsVector(); + } + else + { + Debug.Assert(Vector.Count == Vector128.Count); + return Vector128.ConcatLowerUpper(left.AsVector128(), right.AsVector128()).AsVector(); + } + } + + /// Interleaves the lower halves of two vectors. + /// The type of the elements in the vector. + /// The vector that provides the even-indexed elements. + /// The vector that provides the odd-indexed elements. + /// A new vector containing interleaved elements from the lower halves of and . + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector ZipLower(Vector left, Vector right) + { + if (Vector.Count == Vector512.Count) + { + return Vector512.ZipLower(left.AsVector512(), right.AsVector512()).AsVector(); + } + else if (Vector.Count == Vector256.Count) + { + return Vector256.ZipLower(left.AsVector256(), right.AsVector256()).AsVector(); + } + else + { + Debug.Assert(Vector.Count == Vector128.Count); + return Vector128.ZipLower(left.AsVector128(), right.AsVector128()).AsVector(); + } + } + + /// Interleaves the upper halves of two vectors. + /// The type of the elements in the vector. + /// The vector that provides the even-indexed elements. + /// The vector that provides the odd-indexed elements. + /// A new vector containing interleaved elements from the upper halves of and . + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector ZipUpper(Vector left, Vector right) + { + if (Vector.Count == Vector512.Count) + { + return Vector512.ZipUpper(left.AsVector512(), right.AsVector512()).AsVector(); + } + else if (Vector.Count == Vector256.Count) + { + return Vector256.ZipUpper(left.AsVector256(), right.AsVector256()).AsVector(); + } + else + { + Debug.Assert(Vector.Count == Vector128.Count); + return Vector128.ZipUpper(left.AsVector128(), right.AsVector128()).AsVector(); + } + } + + /// Interleaves two vectors into their lower and upper halves. + /// The type of the elements in the vector. + /// The vector that provides the even-indexed elements. + /// The vector that provides the odd-indexed elements. + /// A pair of vectors containing interleaved elements from the lower and upper halves of and . + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static (Vector Lower, Vector Upper) Zip(Vector left, Vector right) => (ZipLower(left, right), ZipUpper(left, right)); + + /// De-interleaves the even-indexed elements from two vectors. + /// The type of the elements in the vector. + /// The vector that provides the lower half of the result. + /// The vector that provides the upper half of the result. + /// A new vector containing the even-indexed elements from followed by the even-indexed elements from . + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector UnzipEven(Vector left, Vector right) + { + if (Vector.Count == Vector512.Count) + { + return Vector512.UnzipEven(left.AsVector512(), right.AsVector512()).AsVector(); + } + else if (Vector.Count == Vector256.Count) + { + return Vector256.UnzipEven(left.AsVector256(), right.AsVector256()).AsVector(); + } + else + { + Debug.Assert(Vector.Count == Vector128.Count); + return Vector128.UnzipEven(left.AsVector128(), right.AsVector128()).AsVector(); + } + } + + /// De-interleaves the odd-indexed elements from two vectors. + /// The type of the elements in the vector. + /// The vector that provides the lower half of the result. + /// The vector that provides the upper half of the result. + /// A new vector containing the odd-indexed elements from followed by the odd-indexed elements from . + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector UnzipOdd(Vector left, Vector right) + { + if (Vector.Count == Vector512.Count) + { + return Vector512.UnzipOdd(left.AsVector512(), right.AsVector512()).AsVector(); + } + else if (Vector.Count == Vector256.Count) + { + return Vector256.UnzipOdd(left.AsVector256(), right.AsVector256()).AsVector(); + } + else + { + Debug.Assert(Vector.Count == Vector128.Count); + return Vector128.UnzipOdd(left.AsVector128(), right.AsVector128()).AsVector(); + } + } + + /// De-interleaves two vectors into their even-indexed and odd-indexed elements. + /// The type of the elements in the vector. + /// The vector that provides the lower half of each result. + /// The vector that provides the upper half of each result. + /// A pair of vectors containing the even-indexed and odd-indexed elements from and . + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static (Vector Even, Vector Odd) Unzip(Vector left, Vector right) => (UnzipEven(left, right), UnzipOdd(left, right)); + + /// Creates a new vector with the elements of a specified vector in reverse order. + /// The type of the elements in the vector. + /// The vector whose elements will be reversed. + /// A new vector containing the elements of in reverse order. + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector Reverse(Vector vector) + { + if (Vector.Count == Vector512.Count) + { + return Vector512.Reverse(vector.AsVector512()).AsVector(); + } + else if (Vector.Count == Vector256.Count) + { + return Vector256.Reverse(vector.AsVector256()).AsVector(); + } + else + { + Debug.Assert(Vector.Count == Vector128.Count); + return Vector128.Reverse(vector.AsVector128()).AsVector(); + } + } + internal static Vector DegreesToRadians(Vector degrees) where T : ITrigonometricFunctions { @@ -2966,6 +3263,7 @@ public static (Vector Sin, Vector Cos) SinCos(Vector vector /// The vector whose square root is to be computed. /// The type of the elements in the vector. /// A vector whose elements are the square root of the corresponding elements in . + /// The type of () is not supported. [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] public static Vector SquareRoot(Vector value) diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector128.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector128.cs index a7722fa8beffe4..4d5297522a1b54 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector128.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector128.cs @@ -135,6 +135,15 @@ public static Vector128 NegativeOne [Intrinsic] get => Create(T.NegativeOne); } + + /// Gets a new vector with elements that alternate between and , starting with one. + /// The type of the vector () is not supported. + public static Vector128 SignSequence + { + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => CreateAlternatingSequence(T.One, T.NegativeOne); + } } /// Computes the absolute value of each element in a vector. @@ -1578,6 +1587,239 @@ public static Vector128 CreateScalarUnsafe(T value) [MethodImpl(MethodImplOptions.AggressiveInlining)] public static Vector128 CreateSequence(T start, T step) => (Vector128.Indices * step) + Create(start); + /// Creates a new instance where the elements begin at a specified value and are multiplied by another specified value. + /// The type of the elements in the vector. + /// The value that element 0 will be initialized to. + /// The value that indicates how each element should be scaled from the previous. + /// A new instance with the first element initialized to and each subsequent element initialized to the value of the previous element multiplied by . + /// The type of and () is not supported. + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 CreateGeometricSequence(T initial, [ConstantExpected] T multiplier) + { + var lower = Vector64.CreateGeometricSequence(initial, multiplier); + T upperMultiplier = multiplier; + + if (Vector128.Count >= 4) + { + upperMultiplier = Scalar.Multiply(upperMultiplier, upperMultiplier); + } + + if (Vector128.Count >= 8) + { + upperMultiplier = Scalar.Multiply(upperMultiplier, upperMultiplier); + } + + if (Vector128.Count >= 16) + { + upperMultiplier = Scalar.Multiply(upperMultiplier, upperMultiplier); + } + + var upper = Vector64.CreateGeometricSequence(Scalar.Multiply(initial, upperMultiplier), multiplier); + return Create(lower, upper); + } + + /// Creates a new instance whose elements alternate between two specified values. + /// The type of the elements in the vector. + /// The value assigned to even-indexed elements. + /// The value assigned to odd-indexed elements. + /// A new instance whose even-indexed elements are initialized to and odd-indexed elements are initialized to . + /// The type of and () is not supported. + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 CreateAlternatingSequence(T even, T odd) + { + if (Vector64.Count == 1) + { + return Create(Vector64.Create(even), Vector64.Create(odd)); + } + + Vector64 sequence = Vector64.CreateAlternatingSequence(even, odd); + return Create(sequence, sequence); + } + + /// Creates a new instance whose elements are the reciprocal of an arithmetic sequence. + /// The type of the elements in the vector. + /// The value that element 0 of the arithmetic sequence will be initialized to. + /// The value that indicates how far apart each element of the arithmetic sequence should be from the previous. + /// A new instance whose elements are initialized to one divided by the corresponding element of the arithmetic sequence. + /// The type of and () is not supported. + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 CreateHarmonicSequence(T start, T step) => Vector128.One / CreateSequence(start, step); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 ConcatLowerLower(Vector128 left, Vector128 right) => Create(left.GetLower(), right.GetLower()); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 ConcatUpperLower(Vector128 left, Vector128 right) => Create(left.GetUpper(), right.GetLower()); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 ConcatUpperUpper(Vector128 left, Vector128 right) => Create(left.GetUpper(), right.GetUpper()); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 ConcatLowerUpper(Vector128 left, Vector128 right) => Create(left.GetLower(), right.GetUpper()); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 ZipLower(Vector128 left, Vector128 right) + { + if (Vector64.Count == 1) + { + return Create(left.GetLower(), right.GetLower()); + } + + return Create( + Vector64.ZipLower(left.GetLower(), right.GetLower()), + Vector64.ZipUpper(left.GetLower(), right.GetLower()) + ); + } + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 ZipUpper(Vector128 left, Vector128 right) + { + if (Vector64.Count == 1) + { + return Create(left.GetUpper(), right.GetUpper()); + } + + return Create( + Vector64.ZipLower(left.GetUpper(), right.GetUpper()), + Vector64.ZipUpper(left.GetUpper(), right.GetUpper()) + ); + } + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static (Vector128 Lower, Vector128 Upper) Zip(Vector128 left, Vector128 right) => (ZipLower(left, right), ZipUpper(left, right)); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 UnzipEven(Vector128 left, Vector128 right) => Create( + Vector64.UnzipEven(left.GetLower(), left.GetUpper()), + Vector64.UnzipEven(right.GetLower(), right.GetUpper()) + ); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 UnzipOdd(Vector128 left, Vector128 right) + { + if (Vector64.Count == 1) + { + return Create(left.GetUpper(), right.GetUpper()); + } + + return Create( + Vector64.UnzipOdd(left.GetLower(), left.GetUpper()), + Vector64.UnzipOdd(right.GetLower(), right.GetUpper()) + ); + } + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static (Vector128 Even, Vector128 Odd) Unzip(Vector128 left, Vector128 right) => (UnzipEven(left, right), UnzipOdd(left, right)); + + private static Vector128 Unzip(Vector128 left, Vector128 right, bool odd) + { + int start = odd ? 1 : 0; + int lowerCount = (Vector128.Count - start + 1) / 2; + + if (lowerCount == 0) + { + return Vector128.Zero; + } + + Unsafe.SkipInit(out Vector128 result); + + for (int index = 0; index < Vector128.Count; index++) + { + T value = (index < lowerCount) + ? left.GetElementUnsafe(start + (index * 2)) + : right.GetElementUnsafe(start + ((index - lowerCount) * 2)); + + result.SetElementUnsafe(index, value); + } + + return result; + } + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Reverse(Vector128 vector) + { + if (typeof(T) == typeof(byte)) + { + return Shuffle(vector.As(), CreateSequence((byte)(Vector128.Count - 1), byte.MaxValue)).As(); + } + if (typeof(T) == typeof(double)) + { + return Shuffle(vector.As(), CreateSequence(Vector128.Count - 1, -1L)).As(); + } + if (typeof(T) == typeof(short)) + { + return Shuffle(vector.As(), CreateSequence((short)(Vector128.Count - 1), (short)-1)).As(); + } + if (typeof(T) == typeof(int)) + { + return Shuffle(vector.As(), CreateSequence(Vector128.Count - 1, -1)).As(); + } + if (typeof(T) == typeof(long)) + { + return Shuffle(vector.As(), CreateSequence(Vector128.Count - 1, -1L)).As(); + } + if (typeof(T) == typeof(nint)) + { + return Unsafe.SizeOf() == sizeof(long) + ? Shuffle(vector.As(), CreateSequence(Vector128.Count - 1, -1L)).As() + : Shuffle(vector.As(), CreateSequence(Vector128.Count - 1, -1)).As(); + } + if (typeof(T) == typeof(sbyte)) + { + return Shuffle(vector.As(), CreateSequence((sbyte)(Vector128.Count - 1), (sbyte)-1)).As(); + } + if (typeof(T) == typeof(float)) + { + return Shuffle(vector.As(), CreateSequence(Vector128.Count - 1, -1)).As(); + } + if (typeof(T) == typeof(ushort)) + { + return Shuffle(vector.As(), CreateSequence((ushort)(Vector128.Count - 1), ushort.MaxValue)).As(); + } + if (typeof(T) == typeof(uint)) + { + return Shuffle(vector.As(), CreateSequence((uint)(Vector128.Count - 1), uint.MaxValue)).As(); + } + if (typeof(T) == typeof(ulong)) + { + return Shuffle(vector.As(), CreateSequence((ulong)(Vector128.Count - 1), ulong.MaxValue)).As(); + } + if (typeof(T) == typeof(nuint)) + { + return Unsafe.SizeOf() == sizeof(ulong) + ? Shuffle(vector.As(), CreateSequence((ulong)(Vector128.Count - 1), ulong.MaxValue)).As() + : Shuffle(vector.As(), CreateSequence((uint)(Vector128.Count - 1), uint.MaxValue)).As(); + } + + ThrowHelper.ThrowForUnsupportedIntrinsicsVector128BaseType(); + return default; + } + /// [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector256.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector256.cs index da0b8bc061f5ec..aab372a8b0d4ac 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector256.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector256.cs @@ -129,6 +129,14 @@ public static Vector256 NegativeOne [Intrinsic] get => Create(T.NegativeOne); } + + /// + public static Vector256 SignSequence + { + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => CreateAlternatingSequence(T.One, T.NegativeOne); + } } /// Computes the absolute value of each element in a vector. @@ -1658,6 +1666,235 @@ public static Vector256 CreateScalarUnsafe(T value) [MethodImpl(MethodImplOptions.AggressiveInlining)] public static Vector256 CreateSequence(T start, T step) => (Vector256.Indices * step) + Create(start); + /// Creates a new instance where the elements begin at a specified value and are multiplied by another specified value. + /// The type of the elements in the vector. + /// The value that element 0 will be initialized to. + /// The value that indicates how each element should be scaled from the previous. + /// A new instance with the first element initialized to and each subsequent element initialized to the value of the previous element multiplied by . + /// The type of and () is not supported. + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 CreateGeometricSequence(T initial, [ConstantExpected] T multiplier) + { + var lower = Vector128.CreateGeometricSequence(initial, multiplier); + T upperMultiplier = multiplier; + + if (Vector256.Count >= 4) + { + upperMultiplier = Scalar.Multiply(upperMultiplier, upperMultiplier); + } + + if (Vector256.Count >= 8) + { + upperMultiplier = Scalar.Multiply(upperMultiplier, upperMultiplier); + } + + if (Vector256.Count >= 16) + { + upperMultiplier = Scalar.Multiply(upperMultiplier, upperMultiplier); + } + + if (Vector256.Count >= 32) + { + upperMultiplier = Scalar.Multiply(upperMultiplier, upperMultiplier); + } + + var upper = Vector128.CreateGeometricSequence(Scalar.Multiply(initial, upperMultiplier), multiplier); + return Create(lower, upper); + } + + /// Creates a new instance whose elements alternate between two specified values. + /// The type of the elements in the vector. + /// The value assigned to even-indexed elements. + /// The value assigned to odd-indexed elements. + /// A new instance whose even-indexed elements are initialized to and odd-indexed elements are initialized to . + /// The type of and () is not supported. + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 CreateAlternatingSequence(T even, T odd) + { + Vector128 sequence = Vector128.CreateAlternatingSequence(even, odd); + return Create(sequence, sequence); + } + + /// Creates a new instance whose elements are the reciprocal of an arithmetic sequence. + /// The type of the elements in the vector. + /// The value that element 0 of the arithmetic sequence will be initialized to. + /// The value that indicates how far apart each element of the arithmetic sequence should be from the previous. + /// A new instance whose elements are initialized to one divided by the corresponding element of the arithmetic sequence. + /// The type of and () is not supported. + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 CreateHarmonicSequence(T start, T step) + { + if (IsHardwareAccelerated) + { + return Vector256.One / CreateSequence(start, step); + } + + T upperStart = Scalar.Add(start, Scalar.Multiply(Scalar.Convert(Vector128.Count), step)); + + return Create( + Vector128.CreateHarmonicSequence(start, step), + Vector128.CreateHarmonicSequence(upperStart, step) + ); + } + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 ConcatLowerLower(Vector256 left, Vector256 right) => Create(left.GetLower(), right.GetLower()); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 ConcatUpperLower(Vector256 left, Vector256 right) => Create(left.GetUpper(), right.GetLower()); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 ConcatUpperUpper(Vector256 left, Vector256 right) => Create(left.GetUpper(), right.GetUpper()); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 ConcatLowerUpper(Vector256 left, Vector256 right) => Create(left.GetLower(), right.GetUpper()); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 ZipLower(Vector256 left, Vector256 right) => Create( + Vector128.ZipLower(left.GetLower(), right.GetLower()), + Vector128.ZipUpper(left.GetLower(), right.GetLower()) + ); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 ZipUpper(Vector256 left, Vector256 right) => Create( + Vector128.ZipLower(left.GetUpper(), right.GetUpper()), + Vector128.ZipUpper(left.GetUpper(), right.GetUpper()) + ); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static (Vector256 Lower, Vector256 Upper) Zip(Vector256 left, Vector256 right) + { + if (Avx2.IsSupported && ((typeof(T) == typeof(int)) || (typeof(T) == typeof(uint)))) + { + Vector256 lower = Avx2.UnpackLow(left.AsInt32(), right.AsInt32()); + Vector256 upper = Avx2.UnpackHigh(left.AsInt32(), right.AsInt32()); + + return ( + Avx2.Permute2x128(lower, upper, 0x20).As(), + Avx2.Permute2x128(lower, upper, 0x31).As() + ); + } + + return (ZipLower(left, right), ZipUpper(left, right)); + } + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 UnzipEven(Vector256 left, Vector256 right) => Create( + Vector128.UnzipEven(left.GetLower(), left.GetUpper()), + Vector128.UnzipEven(right.GetLower(), right.GetUpper()) + ); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 UnzipOdd(Vector256 left, Vector256 right) => Create( + Vector128.UnzipOdd(left.GetLower(), left.GetUpper()), + Vector128.UnzipOdd(right.GetLower(), right.GetUpper()) + ); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static (Vector256 Even, Vector256 Odd) Unzip(Vector256 left, Vector256 right) + { + if (Avx2.IsSupported && ((typeof(T) == typeof(int)) || (typeof(T) == typeof(uint)))) + { + Vector256 leftUnzip = Avx2.Shuffle(left.AsInt32(), 0xD8); + leftUnzip = Avx2.Permute4x64(leftUnzip.AsInt64(), 0xD8).AsInt32(); + + Vector256 rightUnzip = Avx2.Shuffle(right.AsInt32(), 0xD8); + rightUnzip = Avx2.Permute4x64(rightUnzip.AsInt64(), 0xD8).AsInt32(); + + return ( + Avx2.Permute2x128(leftUnzip, rightUnzip, 0x20).As(), + Avx2.Permute2x128(leftUnzip, rightUnzip, 0x31).As() + ); + } + + return (UnzipEven(left, right), UnzipOdd(left, right)); + } + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Reverse(Vector256 vector) + { + if (typeof(T) == typeof(byte)) + { + return Shuffle(vector.As(), CreateSequence((byte)(Vector256.Count - 1), byte.MaxValue)).As(); + } + if (typeof(T) == typeof(double)) + { + return Shuffle(vector.As(), CreateSequence(Vector256.Count - 1, -1L)).As(); + } + if (typeof(T) == typeof(short)) + { + return Shuffle(vector.As(), CreateSequence((short)(Vector256.Count - 1), (short)-1)).As(); + } + if (typeof(T) == typeof(int)) + { + return Shuffle(vector.As(), CreateSequence(Vector256.Count - 1, -1)).As(); + } + if (typeof(T) == typeof(long)) + { + return Shuffle(vector.As(), CreateSequence(Vector256.Count - 1, -1L)).As(); + } + if (typeof(T) == typeof(nint)) + { + return Unsafe.SizeOf() == sizeof(long) + ? Shuffle(vector.As(), CreateSequence(Vector256.Count - 1, -1L)).As() + : Shuffle(vector.As(), CreateSequence(Vector256.Count - 1, -1)).As(); + } + if (typeof(T) == typeof(sbyte)) + { + return Shuffle(vector.As(), CreateSequence((sbyte)(Vector256.Count - 1), (sbyte)-1)).As(); + } + if (typeof(T) == typeof(float)) + { + return Shuffle(vector.As(), CreateSequence(Vector256.Count - 1, -1)).As(); + } + if (typeof(T) == typeof(ushort)) + { + return Shuffle(vector.As(), CreateSequence((ushort)(Vector256.Count - 1), ushort.MaxValue)).As(); + } + if (typeof(T) == typeof(uint)) + { + return Shuffle(vector.As(), CreateSequence((uint)(Vector256.Count - 1), uint.MaxValue)).As(); + } + if (typeof(T) == typeof(ulong)) + { + return Shuffle(vector.As(), CreateSequence((ulong)(Vector256.Count - 1), ulong.MaxValue)).As(); + } + if (typeof(T) == typeof(nuint)) + { + return Unsafe.SizeOf() == sizeof(ulong) + ? Shuffle(vector.As(), CreateSequence((ulong)(Vector256.Count - 1), ulong.MaxValue)).As() + : Shuffle(vector.As(), CreateSequence((uint)(Vector256.Count - 1), uint.MaxValue)).As(); + } + + ThrowHelper.ThrowForUnsupportedIntrinsicsVector256BaseType(); + return default; + } + /// [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector512.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector512.cs index c3b4e2ee371f55..20b8f1f101b119 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector512.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector512.cs @@ -6,6 +6,7 @@ using System.Numerics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using System.Runtime.Intrinsics.X86; namespace System.Runtime.Intrinsics { @@ -128,6 +129,14 @@ public static Vector512 NegativeOne [Intrinsic] get => Create(T.NegativeOne); } + + /// + public static Vector512 SignSequence + { + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => CreateAlternatingSequence(T.One, T.NegativeOne); + } } /// Computes the absolute value of each element in a vector. @@ -1690,6 +1699,262 @@ public static Vector512 CreateScalarUnsafe(T value) [MethodImpl(MethodImplOptions.AggressiveInlining)] public static Vector512 CreateSequence(T start, T step) => (Vector512.Indices * step) + Create(start); + /// Creates a new instance where the elements begin at a specified value and are multiplied by another specified value. + /// The type of the elements in the vector. + /// The value that element 0 will be initialized to. + /// The value that indicates how each element should be scaled from the previous. + /// A new instance with the first element initialized to and each subsequent element initialized to the value of the previous element multiplied by . + /// The type of and () is not supported. + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 CreateGeometricSequence(T initial, [ConstantExpected] T multiplier) + { + var lower = Vector256.CreateGeometricSequence(initial, multiplier); + T upperMultiplier = multiplier; + + if (Vector512.Count >= 4) + { + upperMultiplier = Scalar.Multiply(upperMultiplier, upperMultiplier); + } + + if (Vector512.Count >= 8) + { + upperMultiplier = Scalar.Multiply(upperMultiplier, upperMultiplier); + } + + if (Vector512.Count >= 16) + { + upperMultiplier = Scalar.Multiply(upperMultiplier, upperMultiplier); + } + + if (Vector512.Count >= 32) + { + upperMultiplier = Scalar.Multiply(upperMultiplier, upperMultiplier); + } + + if (Vector512.Count >= 64) + { + upperMultiplier = Scalar.Multiply(upperMultiplier, upperMultiplier); + } + + var upper = Vector256.CreateGeometricSequence(Scalar.Multiply(initial, upperMultiplier), multiplier); + return Create(lower, upper); + } + + /// Creates a new instance whose elements alternate between two specified values. + /// The type of the elements in the vector. + /// The value assigned to even-indexed elements. + /// The value assigned to odd-indexed elements. + /// A new instance whose even-indexed elements are initialized to and odd-indexed elements are initialized to . + /// The type of and () is not supported. + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 CreateAlternatingSequence(T even, T odd) + { + Vector256 sequence = Vector256.CreateAlternatingSequence(even, odd); + return Create(sequence, sequence); + } + + /// Creates a new instance whose elements are the reciprocal of an arithmetic sequence. + /// The type of the elements in the vector. + /// The value that element 0 of the arithmetic sequence will be initialized to. + /// The value that indicates how far apart each element of the arithmetic sequence should be from the previous. + /// A new instance whose elements are initialized to one divided by the corresponding element of the arithmetic sequence. + /// The type of and () is not supported. + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 CreateHarmonicSequence(T start, T step) + { + if (IsHardwareAccelerated) + { + return Vector512.One / CreateSequence(start, step); + } + + T upperStart = Scalar.Add(start, Scalar.Multiply(Scalar.Convert(Vector256.Count), step)); + + return Create( + Vector256.CreateHarmonicSequence(start, step), + Vector256.CreateHarmonicSequence(upperStart, step) + ); + } + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 ConcatLowerLower(Vector512 left, Vector512 right) => Create(left.GetLower(), right.GetLower()); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 ConcatUpperLower(Vector512 left, Vector512 right) => Create(left.GetUpper(), right.GetLower()); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 ConcatUpperUpper(Vector512 left, Vector512 right) => Create(left.GetUpper(), right.GetUpper()); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 ConcatLowerUpper(Vector512 left, Vector512 right) => Create(left.GetLower(), right.GetUpper()); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 ZipLower(Vector512 left, Vector512 right) => Create( + Vector256.ZipLower(left._lower, right._lower), + Vector256.ZipUpper(left._lower, right._lower) + ); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 ZipUpper(Vector512 left, Vector512 right) => Create( + Vector256.ZipLower(left._upper, right._upper), + Vector256.ZipUpper(left._upper, right._upper) + ); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static (Vector512 Lower, Vector512 Upper) Zip(Vector512 left, Vector512 right) + { + if (Avx512F.IsSupported && ((typeof(T) == typeof(int)) || (typeof(T) == typeof(uint)))) + { + Vector512 lower = Avx512F.UnpackLow(left.AsInt32(), right.AsInt32()); + Vector512 upper = Avx512F.UnpackHigh(left.AsInt32(), right.AsInt32()); + + Vector512 lowerResult = Avx512F.Shuffle4x128(lower, upper, 0x44); + lowerResult = Avx512F.Shuffle4x128(lowerResult, lowerResult, 0xD8); + + Vector512 upperResult = Avx512F.Shuffle4x128(lower, upper, 0xEE); + upperResult = Avx512F.Shuffle4x128(upperResult, upperResult, 0xD8); + + return (lowerResult.As(), upperResult.As()); + } + + if (IsHardwareAccelerated) + { + return (ZipLower(left, right), ZipUpper(left, right)); + } + + (Vector256 lower0, Vector256 upper0) = Vector256.Zip(left.GetLower(), right.GetLower()); + (Vector256 lower1, Vector256 upper1) = Vector256.Zip(left.GetUpper(), right.GetUpper()); + + return (Create(lower0, upper0), Create(lower1, upper1)); + } + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 UnzipEven(Vector512 left, Vector512 right) => Create( + Vector256.UnzipEven(left.GetLower(), left.GetUpper()), + Vector256.UnzipEven(right.GetLower(), right.GetUpper()) + ); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 UnzipOdd(Vector512 left, Vector512 right) => Create( + Vector256.UnzipOdd(left.GetLower(), left.GetUpper()), + Vector256.UnzipOdd(right.GetLower(), right.GetUpper()) + ); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static (Vector512 Even, Vector512 Odd) Unzip(Vector512 left, Vector512 right) + { + if (Avx512F.IsSupported && ((typeof(T) == typeof(int)) || (typeof(T) == typeof(uint)))) + { + Vector512 evenIndices = Create( + 0, 2, 4, 6, 8, 10, 12, 14, + 16, 18, 20, 22, 24, 26, 28, 30 + ); + Vector512 oddIndices = Create( + 1, 3, 5, 7, 9, 11, 13, 15, + 17, 19, 21, 23, 25, 27, 29, 31 + ); + + Vector512 even = Avx512F.PermuteVar16x32x2(left.AsInt32(), evenIndices, right.AsInt32()); + Vector512 odd = Avx512F.PermuteVar16x32x2(left.AsInt32(), oddIndices, right.AsInt32()); + + return (even.As(), odd.As()); + } + + if (IsHardwareAccelerated) + { + return (UnzipEven(left, right), UnzipOdd(left, right)); + } + + (Vector256 even0, Vector256 odd0) = Vector256.Unzip(left._lower, left._upper); + (Vector256 even1, Vector256 odd1) = Vector256.Unzip(right._lower, right._upper); + + return (Create(even0, even1), Create(odd0, odd1)); + } + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Reverse(Vector512 vector) + { + if (typeof(T) == typeof(byte)) + { + return Shuffle(vector.As(), CreateSequence((byte)(Vector512.Count - 1), byte.MaxValue)).As(); + } + if (typeof(T) == typeof(double)) + { + return Shuffle(vector.As(), CreateSequence(Vector512.Count - 1, -1L)).As(); + } + if (typeof(T) == typeof(short)) + { + return Shuffle(vector.As(), CreateSequence((short)(Vector512.Count - 1), (short)-1)).As(); + } + if (typeof(T) == typeof(int)) + { + return Shuffle(vector.As(), CreateSequence(Vector512.Count - 1, -1)).As(); + } + if (typeof(T) == typeof(long)) + { + return Shuffle(vector.As(), CreateSequence(Vector512.Count - 1, -1L)).As(); + } + if (typeof(T) == typeof(nint)) + { + return Unsafe.SizeOf() == sizeof(long) + ? Shuffle(vector.As(), CreateSequence(Vector512.Count - 1, -1L)).As() + : Shuffle(vector.As(), CreateSequence(Vector512.Count - 1, -1)).As(); + } + if (typeof(T) == typeof(sbyte)) + { + return Shuffle(vector.As(), CreateSequence((sbyte)(Vector512.Count - 1), (sbyte)-1)).As(); + } + if (typeof(T) == typeof(float)) + { + return Shuffle(vector.As(), CreateSequence(Vector512.Count - 1, -1)).As(); + } + if (typeof(T) == typeof(ushort)) + { + return Shuffle(vector.As(), CreateSequence((ushort)(Vector512.Count - 1), ushort.MaxValue)).As(); + } + if (typeof(T) == typeof(uint)) + { + return Shuffle(vector.As(), CreateSequence((uint)(Vector512.Count - 1), uint.MaxValue)).As(); + } + if (typeof(T) == typeof(ulong)) + { + return Shuffle(vector.As(), CreateSequence((ulong)(Vector512.Count - 1), ulong.MaxValue)).As(); + } + if (typeof(T) == typeof(nuint)) + { + return Unsafe.SizeOf() == sizeof(ulong) + ? Shuffle(vector.As(), CreateSequence((ulong)(Vector512.Count - 1), ulong.MaxValue)).As() + : Shuffle(vector.As(), CreateSequence((uint)(Vector512.Count - 1), uint.MaxValue)).As(); + } + + ThrowHelper.ThrowForUnsupportedIntrinsicsVector512BaseType(); + return default; + } + /// [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector64.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector64.cs index 625c96d231e7af..54228364f3b2cf 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector64.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector64.cs @@ -102,6 +102,14 @@ public static Vector64 NegativeOne [Intrinsic] get => Create(T.NegativeOne); } + + /// + public static Vector64 SignSequence + { + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => CreateAlternatingSequence(T.One, T.NegativeOne); + } } /// Computes the absolute value of each element in a vector. @@ -1386,6 +1394,331 @@ public static Vector64 CreateScalarUnsafe(T value) [MethodImpl(MethodImplOptions.AggressiveInlining)] public static Vector64 CreateSequence(T start, T step) => (Vector64.Indices * step) + Create(start); + /// Creates a new instance where the elements begin at a specified value and are multiplied by another specified value. + /// The type of the elements in the vector. + /// The value that element 0 will be initialized to. + /// The value that indicates how each element should be scaled from the previous. + /// A new instance with the first element initialized to and each subsequent element initialized to the value of the previous element multiplied by . + /// The type of and () is not supported. + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector64 CreateGeometricSequence(T initial, [ConstantExpected] T multiplier) + { + Unsafe.SkipInit(out Vector64 result); + + T value = initial; + result.SetElementUnsafe(0, value); + + if (Vector64.Count >= 2) + { + value = Scalar.Multiply(value, multiplier); + result.SetElementUnsafe(1, value); + } + + if (Vector64.Count >= 4) + { + value = Scalar.Multiply(value, multiplier); + result.SetElementUnsafe(2, value); + + value = Scalar.Multiply(value, multiplier); + result.SetElementUnsafe(3, value); + } + + if (Vector64.Count >= 8) + { + value = Scalar.Multiply(value, multiplier); + result.SetElementUnsafe(4, value); + + value = Scalar.Multiply(value, multiplier); + result.SetElementUnsafe(5, value); + + value = Scalar.Multiply(value, multiplier); + result.SetElementUnsafe(6, value); + + value = Scalar.Multiply(value, multiplier); + result.SetElementUnsafe(7, value); + } + + return result; + } + + /// Creates a new instance whose elements alternate between two specified values. + /// The type of the elements in the vector. + /// The value assigned to even-indexed elements. + /// The value assigned to odd-indexed elements. + /// A new instance whose even-indexed elements are initialized to and odd-indexed elements are initialized to . + /// The type of and () is not supported. + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector64 CreateAlternatingSequence(T even, T odd) + { + Unsafe.SkipInit(out Vector64 result); + + if (Vector64.Count == 1) + { + result.SetElementUnsafe(0, even); + return result; + } + + for (int index = 0; index < Vector64.Count; index += 2) + { + result.SetElementUnsafe(index, even); + result.SetElementUnsafe(index + 1, odd); + } + + return result; + } + + /// Creates a new instance whose elements are the reciprocal of an arithmetic sequence. + /// The type of the elements in the vector. + /// The value that element 0 of the arithmetic sequence will be initialized to. + /// The value that indicates how far apart each element of the arithmetic sequence should be from the previous. + /// A new instance whose elements are initialized to one divided by the corresponding element of the arithmetic sequence. + /// The type of and () is not supported. + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector64 CreateHarmonicSequence(T start, T step) => Vector64.One / CreateSequence(start, step); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector64 ConcatLowerLower(Vector64 left, Vector64 right) + { + if (Vector64.Count == 1) + { + return left; + } + + Unsafe.SkipInit(out Vector64 result); + + for (int index = 0; index < Vector64.Count / 2; index++) + { + result.SetElementUnsafe(index, left.GetElementUnsafe(index)); + result.SetElementUnsafe(index + (Vector64.Count / 2), right.GetElementUnsafe(index)); + } + + return result; + } + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector64 ConcatUpperLower(Vector64 left, Vector64 right) + { + if (Vector64.Count == 1) + { + return left; + } + + Unsafe.SkipInit(out Vector64 result); + + for (int index = 0; index < Vector64.Count / 2; index++) + { + result.SetElementUnsafe(index, left.GetElementUnsafe(index + (Vector64.Count / 2))); + result.SetElementUnsafe(index + (Vector64.Count / 2), right.GetElementUnsafe(index)); + } + + return result; + } + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector64 ConcatUpperUpper(Vector64 left, Vector64 right) + { + if (Vector64.Count == 1) + { + return left; + } + + Unsafe.SkipInit(out Vector64 result); + + for (int index = 0; index < Vector64.Count / 2; index++) + { + result.SetElementUnsafe(index, left.GetElementUnsafe(index + (Vector64.Count / 2))); + result.SetElementUnsafe(index + (Vector64.Count / 2), right.GetElementUnsafe(index + (Vector64.Count / 2))); + } + + return result; + } + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector64 ConcatLowerUpper(Vector64 left, Vector64 right) + { + if (Vector64.Count == 1) + { + return left; + } + + Unsafe.SkipInit(out Vector64 result); + + for (int index = 0; index < Vector64.Count / 2; index++) + { + result.SetElementUnsafe(index, left.GetElementUnsafe(index)); + result.SetElementUnsafe(index + (Vector64.Count / 2), right.GetElementUnsafe(index + (Vector64.Count / 2))); + } + + return result; + } + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector64 ZipLower(Vector64 left, Vector64 right) + { + if (Vector64.Count == 1) + { + return left; + } + + Unsafe.SkipInit(out Vector64 result); + + for (int index = 0; index < Vector64.Count; index += 2) + { + result.SetElementUnsafe(index, left.GetElementUnsafe(index / 2)); + result.SetElementUnsafe(index + 1, right.GetElementUnsafe(index / 2)); + } + + return result; + } + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector64 ZipUpper(Vector64 left, Vector64 right) + { + if (Vector64.Count == 1) + { + return left; + } + + Unsafe.SkipInit(out Vector64 result); + + for (int index = 0; index < Vector64.Count; index += 2) + { + result.SetElementUnsafe(index, left.GetElementUnsafe(Vector64.Count / 2 + index / 2)); + result.SetElementUnsafe(index + 1, right.GetElementUnsafe(Vector64.Count / 2 + index / 2)); + } + + return result; + } + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static (Vector64 Lower, Vector64 Upper) Zip(Vector64 left, Vector64 right) => (ZipLower(left, right), ZipUpper(left, right)); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector64 UnzipEven(Vector64 left, Vector64 right) + { + if (Vector64.Count == 1) + { + return left; + } + + Unsafe.SkipInit(out Vector64 result); + + for (int index = 0; index < Vector64.Count / 2; index++) + { + result.SetElementUnsafe(index, left.GetElementUnsafe(index * 2)); + result.SetElementUnsafe(index + (Vector64.Count / 2), right.GetElementUnsafe(index * 2)); + } + + return result; + } + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector64 UnzipOdd(Vector64 left, Vector64 right) + { + if (Vector64.Count == 1) + { + return Vector64.Zero; + } + + Unsafe.SkipInit(out Vector64 result); + + for (int index = 0; index < Vector64.Count / 2; index++) + { + result.SetElementUnsafe(index, left.GetElementUnsafe((index * 2) + 1)); + result.SetElementUnsafe(index + (Vector64.Count / 2), right.GetElementUnsafe((index * 2) + 1)); + } + + return result; + } + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static (Vector64 Even, Vector64 Odd) Unzip(Vector64 left, Vector64 right) => (UnzipEven(left, right), UnzipOdd(left, right)); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector64 Reverse(Vector64 vector) + { + if (typeof(T) == typeof(byte)) + { + return Shuffle(vector.As(), CreateSequence((byte)(Vector64.Count - 1), byte.MaxValue)).As(); + } + if (typeof(T) == typeof(double)) + { + return vector; + } + if (typeof(T) == typeof(short)) + { + return Shuffle(vector.As(), CreateSequence((short)(Vector64.Count - 1), (short)-1)).As(); + } + if (typeof(T) == typeof(int)) + { + return Shuffle(vector.As(), CreateSequence(Vector64.Count - 1, -1)).As(); + } + if (typeof(T) == typeof(long)) + { + return vector; + } + if (typeof(T) == typeof(nint)) + { + return Unsafe.SizeOf() == sizeof(long) + ? vector + : Shuffle(vector.As(), CreateSequence(Vector64.Count - 1, -1)).As(); + } + if (typeof(T) == typeof(sbyte)) + { + return Shuffle(vector.As(), CreateSequence((sbyte)(Vector64.Count - 1), (sbyte)-1)).As(); + } + if (typeof(T) == typeof(float)) + { + return Shuffle(vector.As(), CreateSequence(Vector64.Count - 1, -1)).As(); + } + if (typeof(T) == typeof(ushort)) + { + return Shuffle(vector.As(), CreateSequence((ushort)(Vector64.Count - 1), ushort.MaxValue)).As(); + } + if (typeof(T) == typeof(uint)) + { + return Shuffle(vector.As(), CreateSequence((uint)(Vector64.Count - 1), uint.MaxValue)).As(); + } + if (typeof(T) == typeof(ulong)) + { + return vector; + } + if (typeof(T) == typeof(nuint)) + { + return Unsafe.SizeOf() == sizeof(ulong) + ? vector + : Shuffle(vector.As(), CreateSequence((uint)(Vector64.Count - 1), uint.MaxValue)).As(); + } + + ThrowHelper.ThrowForUnsupportedIntrinsicsVector64BaseType(); + return default; + } + internal static Vector64 DegreesToRadians(Vector64 degrees) where T : ITrigonometricFunctions { diff --git a/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs b/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs index 855363fbe75383..d8ef29f0ee2423 100644 --- a/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs +++ b/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs @@ -171,6 +171,20 @@ public static void CopyTo(this System.Runtime.Intrinsics.Vector128 vector, public static System.Runtime.Intrinsics.Vector128 CreateScalarUnsafe(ulong value) { throw null; } public static System.Runtime.Intrinsics.Vector128 CreateScalarUnsafe(T value) { throw null; } public static System.Runtime.Intrinsics.Vector128 CreateSequence(T start, T step) { throw null; } + public static System.Runtime.Intrinsics.Vector128 CreateGeometricSequence(T initial, [System.Diagnostics.CodeAnalysis.ConstantExpected] T multiplier) { throw null; } + public static System.Runtime.Intrinsics.Vector128 CreateAlternatingSequence(T even, T odd) { throw null; } + public static System.Runtime.Intrinsics.Vector128 CreateHarmonicSequence(T start, T step) { throw null; } + public static System.Runtime.Intrinsics.Vector128 ZipLower(System.Runtime.Intrinsics.Vector128 left, System.Runtime.Intrinsics.Vector128 right) { throw null; } + public static System.Runtime.Intrinsics.Vector128 ZipUpper(System.Runtime.Intrinsics.Vector128 left, System.Runtime.Intrinsics.Vector128 right) { throw null; } + public static (System.Runtime.Intrinsics.Vector128 Lower, System.Runtime.Intrinsics.Vector128 Upper) Zip(System.Runtime.Intrinsics.Vector128 left, System.Runtime.Intrinsics.Vector128 right) { throw null; } + public static System.Runtime.Intrinsics.Vector128 UnzipEven(System.Runtime.Intrinsics.Vector128 left, System.Runtime.Intrinsics.Vector128 right) { throw null; } + public static System.Runtime.Intrinsics.Vector128 UnzipOdd(System.Runtime.Intrinsics.Vector128 left, System.Runtime.Intrinsics.Vector128 right) { throw null; } + public static (System.Runtime.Intrinsics.Vector128 Even, System.Runtime.Intrinsics.Vector128 Odd) Unzip(System.Runtime.Intrinsics.Vector128 left, System.Runtime.Intrinsics.Vector128 right) { throw null; } + public static System.Runtime.Intrinsics.Vector128 ConcatLowerLower(System.Runtime.Intrinsics.Vector128 left, System.Runtime.Intrinsics.Vector128 right) { throw null; } + public static System.Runtime.Intrinsics.Vector128 ConcatUpperLower(System.Runtime.Intrinsics.Vector128 left, System.Runtime.Intrinsics.Vector128 right) { throw null; } + public static System.Runtime.Intrinsics.Vector128 ConcatUpperUpper(System.Runtime.Intrinsics.Vector128 left, System.Runtime.Intrinsics.Vector128 right) { throw null; } + public static System.Runtime.Intrinsics.Vector128 ConcatLowerUpper(System.Runtime.Intrinsics.Vector128 left, System.Runtime.Intrinsics.Vector128 right) { throw null; } + public static System.Runtime.Intrinsics.Vector128 Reverse(System.Runtime.Intrinsics.Vector128 vector) { throw null; } public static System.Runtime.Intrinsics.Vector128 Create(System.ReadOnlySpan values) { throw null; } public static System.Runtime.Intrinsics.Vector128 Create(System.Runtime.Intrinsics.Vector64 value) { throw null; } public static System.Runtime.Intrinsics.Vector128 Create(System.Runtime.Intrinsics.Vector64 lower, System.Runtime.Intrinsics.Vector64 upper) { throw null; } @@ -430,6 +444,7 @@ public static void CopyTo(this System.Runtime.Intrinsics.Vector128 vector, extension(System.Runtime.Intrinsics.Vector128) where T : System.Numerics.ISignedNumber { public static System.Runtime.Intrinsics.Vector128 NegativeOne { get { throw null; } } + public static System.Runtime.Intrinsics.Vector128 SignSequence { get { throw null; } } } } public readonly partial struct Vector128 : System.IEquatable> @@ -616,6 +631,20 @@ public static void CopyTo(this System.Runtime.Intrinsics.Vector256 vector, public static System.Runtime.Intrinsics.Vector256 CreateScalarUnsafe(ulong value) { throw null; } public static System.Runtime.Intrinsics.Vector256 CreateScalarUnsafe(T value) { throw null; } public static System.Runtime.Intrinsics.Vector256 CreateSequence(T start, T step) { throw null; } + public static System.Runtime.Intrinsics.Vector256 CreateGeometricSequence(T initial, [System.Diagnostics.CodeAnalysis.ConstantExpected] T multiplier) { throw null; } + public static System.Runtime.Intrinsics.Vector256 CreateAlternatingSequence(T even, T odd) { throw null; } + public static System.Runtime.Intrinsics.Vector256 CreateHarmonicSequence(T start, T step) { throw null; } + public static System.Runtime.Intrinsics.Vector256 ZipLower(System.Runtime.Intrinsics.Vector256 left, System.Runtime.Intrinsics.Vector256 right) { throw null; } + public static System.Runtime.Intrinsics.Vector256 ZipUpper(System.Runtime.Intrinsics.Vector256 left, System.Runtime.Intrinsics.Vector256 right) { throw null; } + public static (System.Runtime.Intrinsics.Vector256 Lower, System.Runtime.Intrinsics.Vector256 Upper) Zip(System.Runtime.Intrinsics.Vector256 left, System.Runtime.Intrinsics.Vector256 right) { throw null; } + public static System.Runtime.Intrinsics.Vector256 UnzipEven(System.Runtime.Intrinsics.Vector256 left, System.Runtime.Intrinsics.Vector256 right) { throw null; } + public static System.Runtime.Intrinsics.Vector256 UnzipOdd(System.Runtime.Intrinsics.Vector256 left, System.Runtime.Intrinsics.Vector256 right) { throw null; } + public static (System.Runtime.Intrinsics.Vector256 Even, System.Runtime.Intrinsics.Vector256 Odd) Unzip(System.Runtime.Intrinsics.Vector256 left, System.Runtime.Intrinsics.Vector256 right) { throw null; } + public static System.Runtime.Intrinsics.Vector256 ConcatLowerLower(System.Runtime.Intrinsics.Vector256 left, System.Runtime.Intrinsics.Vector256 right) { throw null; } + public static System.Runtime.Intrinsics.Vector256 ConcatUpperLower(System.Runtime.Intrinsics.Vector256 left, System.Runtime.Intrinsics.Vector256 right) { throw null; } + public static System.Runtime.Intrinsics.Vector256 ConcatUpperUpper(System.Runtime.Intrinsics.Vector256 left, System.Runtime.Intrinsics.Vector256 right) { throw null; } + public static System.Runtime.Intrinsics.Vector256 ConcatLowerUpper(System.Runtime.Intrinsics.Vector256 left, System.Runtime.Intrinsics.Vector256 right) { throw null; } + public static System.Runtime.Intrinsics.Vector256 Reverse(System.Runtime.Intrinsics.Vector256 vector) { throw null; } public static System.Runtime.Intrinsics.Vector256 Create(System.ReadOnlySpan values) { throw null; } public static System.Runtime.Intrinsics.Vector256 Create(System.Runtime.Intrinsics.Vector64 value) { throw null; } public static System.Runtime.Intrinsics.Vector256 Create(System.Runtime.Intrinsics.Vector128 value) { throw null; } @@ -876,6 +905,7 @@ public static void CopyTo(this System.Runtime.Intrinsics.Vector256 vector, extension(System.Runtime.Intrinsics.Vector256) where T : System.Numerics.ISignedNumber { public static System.Runtime.Intrinsics.Vector256 NegativeOne { get { throw null; } } + public static System.Runtime.Intrinsics.Vector256 SignSequence { get { throw null; } } } } public readonly partial struct Vector256 : System.IEquatable> @@ -1062,6 +1092,20 @@ public static void CopyTo(this System.Runtime.Intrinsics.Vector512 vector, public static System.Runtime.Intrinsics.Vector512 CreateScalarUnsafe(ulong value) { throw null; } public static System.Runtime.Intrinsics.Vector512 CreateScalarUnsafe(T value) { throw null; } public static System.Runtime.Intrinsics.Vector512 CreateSequence(T start, T step) { throw null; } + public static System.Runtime.Intrinsics.Vector512 CreateGeometricSequence(T initial, [System.Diagnostics.CodeAnalysis.ConstantExpected] T multiplier) { throw null; } + public static System.Runtime.Intrinsics.Vector512 CreateAlternatingSequence(T even, T odd) { throw null; } + public static System.Runtime.Intrinsics.Vector512 CreateHarmonicSequence(T start, T step) { throw null; } + public static System.Runtime.Intrinsics.Vector512 ZipLower(System.Runtime.Intrinsics.Vector512 left, System.Runtime.Intrinsics.Vector512 right) { throw null; } + public static System.Runtime.Intrinsics.Vector512 ZipUpper(System.Runtime.Intrinsics.Vector512 left, System.Runtime.Intrinsics.Vector512 right) { throw null; } + public static (System.Runtime.Intrinsics.Vector512 Lower, System.Runtime.Intrinsics.Vector512 Upper) Zip(System.Runtime.Intrinsics.Vector512 left, System.Runtime.Intrinsics.Vector512 right) { throw null; } + public static System.Runtime.Intrinsics.Vector512 UnzipEven(System.Runtime.Intrinsics.Vector512 left, System.Runtime.Intrinsics.Vector512 right) { throw null; } + public static System.Runtime.Intrinsics.Vector512 UnzipOdd(System.Runtime.Intrinsics.Vector512 left, System.Runtime.Intrinsics.Vector512 right) { throw null; } + public static (System.Runtime.Intrinsics.Vector512 Even, System.Runtime.Intrinsics.Vector512 Odd) Unzip(System.Runtime.Intrinsics.Vector512 left, System.Runtime.Intrinsics.Vector512 right) { throw null; } + public static System.Runtime.Intrinsics.Vector512 ConcatLowerLower(System.Runtime.Intrinsics.Vector512 left, System.Runtime.Intrinsics.Vector512 right) { throw null; } + public static System.Runtime.Intrinsics.Vector512 ConcatUpperLower(System.Runtime.Intrinsics.Vector512 left, System.Runtime.Intrinsics.Vector512 right) { throw null; } + public static System.Runtime.Intrinsics.Vector512 ConcatUpperUpper(System.Runtime.Intrinsics.Vector512 left, System.Runtime.Intrinsics.Vector512 right) { throw null; } + public static System.Runtime.Intrinsics.Vector512 ConcatLowerUpper(System.Runtime.Intrinsics.Vector512 left, System.Runtime.Intrinsics.Vector512 right) { throw null; } + public static System.Runtime.Intrinsics.Vector512 Reverse(System.Runtime.Intrinsics.Vector512 vector) { throw null; } public static System.Runtime.Intrinsics.Vector512 Create(System.ReadOnlySpan values) { throw null; } public static System.Runtime.Intrinsics.Vector512 Create(System.Runtime.Intrinsics.Vector64 value) { throw null; } public static System.Runtime.Intrinsics.Vector512 Create(System.Runtime.Intrinsics.Vector128 value) { throw null; } @@ -1321,6 +1365,7 @@ public static void CopyTo(this System.Runtime.Intrinsics.Vector512 vector, extension(System.Runtime.Intrinsics.Vector512) where T : System.Numerics.ISignedNumber { public static System.Runtime.Intrinsics.Vector512 NegativeOne { get { throw null; } } + public static System.Runtime.Intrinsics.Vector512 SignSequence { get { throw null; } } } } public readonly partial struct Vector512 : System.IEquatable> @@ -1484,6 +1529,20 @@ public static void CopyTo(this System.Runtime.Intrinsics.Vector64 vector, public static System.Runtime.Intrinsics.Vector64 CreateScalarUnsafe(ulong value) { throw null; } public static System.Runtime.Intrinsics.Vector64 CreateScalarUnsafe(T value) { throw null; } public static System.Runtime.Intrinsics.Vector64 CreateSequence(T start, T step) { throw null; } + public static System.Runtime.Intrinsics.Vector64 CreateGeometricSequence(T initial, [System.Diagnostics.CodeAnalysis.ConstantExpected] T multiplier) { throw null; } + public static System.Runtime.Intrinsics.Vector64 CreateAlternatingSequence(T even, T odd) { throw null; } + public static System.Runtime.Intrinsics.Vector64 CreateHarmonicSequence(T start, T step) { throw null; } + public static System.Runtime.Intrinsics.Vector64 ZipLower(System.Runtime.Intrinsics.Vector64 left, System.Runtime.Intrinsics.Vector64 right) { throw null; } + public static System.Runtime.Intrinsics.Vector64 ZipUpper(System.Runtime.Intrinsics.Vector64 left, System.Runtime.Intrinsics.Vector64 right) { throw null; } + public static (System.Runtime.Intrinsics.Vector64 Lower, System.Runtime.Intrinsics.Vector64 Upper) Zip(System.Runtime.Intrinsics.Vector64 left, System.Runtime.Intrinsics.Vector64 right) { throw null; } + public static System.Runtime.Intrinsics.Vector64 UnzipEven(System.Runtime.Intrinsics.Vector64 left, System.Runtime.Intrinsics.Vector64 right) { throw null; } + public static System.Runtime.Intrinsics.Vector64 UnzipOdd(System.Runtime.Intrinsics.Vector64 left, System.Runtime.Intrinsics.Vector64 right) { throw null; } + public static (System.Runtime.Intrinsics.Vector64 Even, System.Runtime.Intrinsics.Vector64 Odd) Unzip(System.Runtime.Intrinsics.Vector64 left, System.Runtime.Intrinsics.Vector64 right) { throw null; } + public static System.Runtime.Intrinsics.Vector64 ConcatLowerLower(System.Runtime.Intrinsics.Vector64 left, System.Runtime.Intrinsics.Vector64 right) { throw null; } + public static System.Runtime.Intrinsics.Vector64 ConcatUpperLower(System.Runtime.Intrinsics.Vector64 left, System.Runtime.Intrinsics.Vector64 right) { throw null; } + public static System.Runtime.Intrinsics.Vector64 ConcatUpperUpper(System.Runtime.Intrinsics.Vector64 left, System.Runtime.Intrinsics.Vector64 right) { throw null; } + public static System.Runtime.Intrinsics.Vector64 ConcatLowerUpper(System.Runtime.Intrinsics.Vector64 left, System.Runtime.Intrinsics.Vector64 right) { throw null; } + public static System.Runtime.Intrinsics.Vector64 Reverse(System.Runtime.Intrinsics.Vector64 vector) { throw null; } public static System.Runtime.Intrinsics.Vector64 Create(System.ReadOnlySpan values) { throw null; } public static System.Runtime.Intrinsics.Vector64 Create(T value) { throw null; } public static System.Runtime.Intrinsics.Vector64 Create(T[] values) { throw null; } @@ -1729,6 +1788,7 @@ public static void CopyTo(this System.Runtime.Intrinsics.Vector64 vector, extension(System.Runtime.Intrinsics.Vector64) where T : System.Numerics.ISignedNumber { public static System.Runtime.Intrinsics.Vector64 NegativeOne { get { throw null; } } + public static System.Runtime.Intrinsics.Vector64 SignSequence { get { throw null; } } } } public readonly partial struct Vector64 : System.IEquatable> diff --git a/src/libraries/System.Runtime.Intrinsics/tests/Vectors/Vector128Tests.cs b/src/libraries/System.Runtime.Intrinsics/tests/Vectors/Vector128Tests.cs index 6246c1231c32d2..0c2a439d395d6b 100644 --- a/src/libraries/System.Runtime.Intrinsics/tests/Vectors/Vector128Tests.cs +++ b/src/libraries/System.Runtime.Intrinsics/tests/Vectors/Vector128Tests.cs @@ -5305,6 +5305,250 @@ private static void TestCreateSequence(T start, T step) } } + [Fact] + public void CreateGeometricSequenceInt32Test() + { + Vector128 sequence = Vector128.CreateGeometricSequence(1, 2); + int expected = 1; + + for (int index = 0; index < Vector128.Count; index++) + { + Assert.Equal(expected, sequence.GetElement(index)); + expected *= 2; + } + } + + [Fact] + public void CreateGeometricSequenceByteWrapsTest() + { + Vector128 sequence = Vector128.CreateGeometricSequence((byte)200, (byte)2); + byte expected = 200; + + for (int index = 0; index < Vector128.Count; index++) + { + Assert.Equal(expected, sequence.GetElement(index)); + expected = unchecked((byte)(expected * 2)); + } + } + + [Fact] + public void CreateGeometricSequenceSingleNonConstantInitialTest() + { + const float multiplier = 1.0064822f; + float initial = GetNonConstant(1.0059024f); + Vector128 sequence = Vector128.CreateGeometricSequence(initial, multiplier); + float expected = initial; + + for (int index = 0; index < Vector128.Count; index++) + { + AssertExtensions.Equal(expected, sequence.GetElement(index), 1e-6f); + expected *= multiplier; + } + } + + [Fact] + public void CreateGeometricSequenceDoubleNonConstantInitialTest() + { + const double multiplier = 1e-50; + double initial = GetNonConstant(1e-154); + Vector128 sequence = Vector128.CreateGeometricSequence(initial, multiplier); + double expected = initial; + + for (int index = 0; index < Vector128.Count; index++) + { + AssertExtensions.Equal(expected, sequence.GetElement(index), 1e-12); + expected *= multiplier; + } + } + + [Fact] + public void CreateAlternatingSequenceInt32Test() + { + Vector128 sequence = Vector128.CreateAlternatingSequence(5, -5); + + for (int index = 0; index < Vector128.Count; index++) + { + Assert.Equal(((index & 1) == 0) ? 5 : -5, sequence.GetElement(index)); + } + } + + [Fact] + public void CreateAlternatingSequenceUInt32Test() + { + Vector128 sequence = Vector128.CreateAlternatingSequence(5u, uint.MaxValue - 1u); + + for (int index = 0; index < Vector128.Count; index++) + { + Assert.Equal(((index & 1) == 0) ? 5u : uint.MaxValue - 1u, sequence.GetElement(index)); + } + } + + [Fact] + public void CreateAlternatingSequenceDoubleTest() + { + Vector128 sequence = Vector128.CreateAlternatingSequence(1.5, -2.5); + + for (int index = 0; index < Vector128.Count; index++) + { + Assert.Equal(((index & 1) == 0) ? 1.5 : -2.5, sequence.GetElement(index)); + } + } + + [Fact] + public void CreateHarmonicSequenceInt32Test() + { + Vector128 sequence = Vector128.CreateHarmonicSequence(1, 1); + int expected = 1; + + for (int index = 0; index < Vector128.Count; index++) + { + Assert.Equal(1 / expected, sequence.GetElement(index)); + expected += 1; + } + } + + [Fact] + public void CreateHarmonicSequenceSingleTest() + { + Vector128 sequence = Vector128.CreateHarmonicSequence(1.0f, 1.0f); + float expected = 1.0f; + + for (int index = 0; index < Vector128.Count; index++) + { + AssertExtensions.Equal(1.0f / expected, sequence.GetElement(index), 1e-6f); + expected += 1.0f; + } + } + + [Fact] + public void CreateHarmonicSequenceDoubleTest() + { + Vector128 sequence = Vector128.CreateHarmonicSequence(1.0, 1.0); + double expected = 1.0; + + for (int index = 0; index < Vector128.Count; index++) + { + AssertExtensions.Equal(1.0 / expected, sequence.GetElement(index), 1e-15); + expected += 1.0; + } + } + + [Fact] + public void SignSequenceInt32Test() + { + Vector128 sequence = Vector128.SignSequence; + + for (int index = 0; index < Vector128.Count; index++) + { + Assert.Equal(((index & 1) == 0) ? 1 : -1, sequence.GetElement(index)); + } + } + + [Fact] + public void SignSequenceSingleTest() + { + Vector128 sequence = Vector128.SignSequence; + + for (int index = 0; index < Vector128.Count; index++) + { + Assert.Equal(((index & 1) == 0) ? 1.0f : -1.0f, sequence.GetElement(index)); + } + } + + [Fact] + public void SignSequenceDoubleTest() + { + Vector128 sequence = Vector128.SignSequence; + + for (int index = 0; index < Vector128.Count; index++) + { + Assert.Equal(((index & 1) == 0) ? 1.0 : -1.0, sequence.GetElement(index)); + } + } + + [Fact] + public void LaneOperationsInt32Test() + { + Vector128 left = Vector128.CreateSequence(0, 1); + Vector128 right = Vector128.CreateSequence(100, 1); + int count = Vector128.Count; + int lowerCount = (count + 1) / 2; + int upperStart = count - lowerCount; + + AssertVectorEqual(CreateVector128(index => ((index & 1) == 0) ? left.GetElement(index / 2) : right.GetElement(index / 2)), Vector128.ZipLower(left, right)); + AssertVectorEqual(CreateVector128(index => ((index & 1) == 0) ? left.GetElement(upperStart + (index / 2)) : right.GetElement(upperStart + (index / 2))), Vector128.ZipUpper(left, right)); + + (Vector128 lower, Vector128 upper) = Vector128.Zip(left, right); + AssertVectorEqual(Vector128.ZipLower(left, right), lower); + AssertVectorEqual(Vector128.ZipUpper(left, right), upper); + + AssertVectorEqual(left, Vector128.UnzipEven(lower, upper)); + AssertVectorEqual(right, Vector128.UnzipOdd(lower, upper)); + + (Vector128 even, Vector128 odd) = Vector128.Unzip(lower, upper); + AssertVectorEqual(left, even); + AssertVectorEqual(right, odd); + + AssertVectorEqual(CreateVector128(index => (index < lowerCount) ? left.GetElement(index) : right.GetElement(index - lowerCount)), Vector128.ConcatLowerLower(left, right)); + AssertVectorEqual(CreateVector128(index => (index < lowerCount) ? left.GetElement(upperStart + index) : right.GetElement(index - lowerCount)), Vector128.ConcatUpperLower(left, right)); + AssertVectorEqual(CreateVector128(index => (index < lowerCount) ? left.GetElement(upperStart + index) : right.GetElement(upperStart + index - lowerCount)), Vector128.ConcatUpperUpper(left, right)); + AssertVectorEqual(CreateVector128(index => (index < lowerCount) ? left.GetElement(index) : right.GetElement(upperStart + index - lowerCount)), Vector128.ConcatLowerUpper(left, right)); + + AssertVectorEqual(CreateVector128(index => left.GetElement(count - 1 - index)), Vector128.Reverse(left)); + } + + [Fact] + public void LaneOperationsDoubleTest() + { + Vector128 left = Vector128.Create(0.0, 1.0); + Vector128 right = Vector128.Create(100.0, 101.0); + + AssertVectorEqual(Vector128.Create(0.0, 100.0), Vector128.ZipLower(left, right)); + AssertVectorEqual(Vector128.Create(1.0, 101.0), Vector128.ZipUpper(left, right)); + + (Vector128 lower, Vector128 upper) = Vector128.Zip(left, right); + AssertVectorEqual(Vector128.ZipLower(left, right), lower); + AssertVectorEqual(Vector128.ZipUpper(left, right), upper); + + AssertVectorEqual(left, Vector128.UnzipEven(lower, upper)); + AssertVectorEqual(right, Vector128.UnzipOdd(lower, upper)); + + (Vector128 even, Vector128 odd) = Vector128.Unzip(lower, upper); + AssertVectorEqual(left, even); + AssertVectorEqual(right, odd); + + AssertVectorEqual(Vector128.Create(0.0, 100.0), Vector128.ConcatLowerLower(left, right)); + AssertVectorEqual(Vector128.Create(1.0, 100.0), Vector128.ConcatUpperLower(left, right)); + AssertVectorEqual(Vector128.Create(1.0, 101.0), Vector128.ConcatUpperUpper(left, right)); + AssertVectorEqual(Vector128.Create(0.0, 101.0), Vector128.ConcatLowerUpper(left, right)); + + AssertVectorEqual(Vector128.Create(1.0, 0.0), Vector128.Reverse(left)); + } + + private static Vector128 CreateVector128(Func elementSelector) + { + int[] values = new int[Vector128.Count]; + + for (int index = 0; index < values.Length; index++) + { + values[index] = elementSelector(index); + } + + return Vector128.Create(values); + } + + private static void AssertVectorEqual(Vector128 expected, Vector128 actual) + where T : struct + { + for (int index = 0; index < Vector128.Count; index++) + { + Assert.Equal(expected.GetElement(index), actual.GetElement(index)); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static T GetNonConstant(T value) => value; + [Theory] [MemberData(nameof(GenericMathTestMemberData.AsinDouble), MemberType = typeof(GenericMathTestMemberData))] public void AsinDoubleTest(double value, double expectedResult, double variance) diff --git a/src/libraries/System.Runtime.Intrinsics/tests/Vectors/Vector256Tests.cs b/src/libraries/System.Runtime.Intrinsics/tests/Vectors/Vector256Tests.cs index 728a87900f314a..d23307f28a42c9 100644 --- a/src/libraries/System.Runtime.Intrinsics/tests/Vectors/Vector256Tests.cs +++ b/src/libraries/System.Runtime.Intrinsics/tests/Vectors/Vector256Tests.cs @@ -6481,6 +6481,222 @@ private static void TestCreateSequence(T start, T step) } } + [Fact] + public void CreateGeometricSequenceInt32Test() + { + Vector256 sequence = Vector256.CreateGeometricSequence(1, 2); + int expected = 1; + + for (int index = 0; index < Vector256.Count; index++) + { + Assert.Equal(expected, sequence.GetElement(index)); + expected *= 2; + } + } + + [Fact] + public void CreateGeometricSequenceByteWrapsTest() + { + Vector256 sequence = Vector256.CreateGeometricSequence((byte)200, (byte)2); + byte expected = 200; + + for (int index = 0; index < Vector256.Count; index++) + { + Assert.Equal(expected, sequence.GetElement(index)); + expected = unchecked((byte)(expected * 2)); + } + } + + [Fact] + public void CreateGeometricSequenceSingleNonConstantInitialTest() + { + const float multiplier = 1.0064822f; + float initial = GetNonConstant(1.0059024f); + Vector256 sequence = Vector256.CreateGeometricSequence(initial, multiplier); + float expected = initial; + + for (int index = 0; index < Vector256.Count; index++) + { + AssertExtensions.Equal(expected, sequence.GetElement(index), 1e-6f); + expected *= multiplier; + } + } + + [Fact] + public void CreateGeometricSequenceDoubleNonConstantInitialTest() + { + const double multiplier = 1e-50; + double initial = GetNonConstant(1e-154); + Vector256 sequence = Vector256.CreateGeometricSequence(initial, multiplier); + double expected = initial; + + for (int index = 0; index < Vector256.Count; index++) + { + AssertExtensions.Equal(expected, sequence.GetElement(index), 1e-12); + expected *= multiplier; + } + } + + [Fact] + public void CreateAlternatingSequenceInt32Test() + { + Vector256 sequence = Vector256.CreateAlternatingSequence(5, -5); + + for (int index = 0; index < Vector256.Count; index++) + { + Assert.Equal(((index & 1) == 0) ? 5 : -5, sequence.GetElement(index)); + } + } + + [Fact] + public void CreateAlternatingSequenceUInt32Test() + { + Vector256 sequence = Vector256.CreateAlternatingSequence(5u, uint.MaxValue - 1u); + + for (int index = 0; index < Vector256.Count; index++) + { + Assert.Equal(((index & 1) == 0) ? 5u : uint.MaxValue - 1u, sequence.GetElement(index)); + } + } + + [Fact] + public void CreateAlternatingSequenceDoubleTest() + { + Vector256 sequence = Vector256.CreateAlternatingSequence(1.5, -2.5); + + for (int index = 0; index < Vector256.Count; index++) + { + Assert.Equal(((index & 1) == 0) ? 1.5 : -2.5, sequence.GetElement(index)); + } + } + + [Fact] + public void CreateHarmonicSequenceInt32Test() + { + Vector256 sequence = Vector256.CreateHarmonicSequence(1, 1); + int expected = 1; + + for (int index = 0; index < Vector256.Count; index++) + { + Assert.Equal(1 / expected, sequence.GetElement(index)); + expected += 1; + } + } + + [Fact] + public void CreateHarmonicSequenceSingleTest() + { + Vector256 sequence = Vector256.CreateHarmonicSequence(1.0f, 1.0f); + float expected = 1.0f; + + for (int index = 0; index < Vector256.Count; index++) + { + AssertExtensions.Equal(1.0f / expected, sequence.GetElement(index), 1e-6f); + expected += 1.0f; + } + } + + [Fact] + public void CreateHarmonicSequenceDoubleTest() + { + Vector256 sequence = Vector256.CreateHarmonicSequence(1.0, 1.0); + double expected = 1.0; + + for (int index = 0; index < Vector256.Count; index++) + { + AssertExtensions.Equal(1.0 / expected, sequence.GetElement(index), 1e-15); + expected += 1.0; + } + } + + [Fact] + public void SignSequenceInt32Test() + { + Vector256 sequence = Vector256.SignSequence; + + for (int index = 0; index < Vector256.Count; index++) + { + Assert.Equal(((index & 1) == 0) ? 1 : -1, sequence.GetElement(index)); + } + } + + [Fact] + public void SignSequenceSingleTest() + { + Vector256 sequence = Vector256.SignSequence; + + for (int index = 0; index < Vector256.Count; index++) + { + Assert.Equal(((index & 1) == 0) ? 1.0f : -1.0f, sequence.GetElement(index)); + } + } + + [Fact] + public void SignSequenceDoubleTest() + { + Vector256 sequence = Vector256.SignSequence; + + for (int index = 0; index < Vector256.Count; index++) + { + Assert.Equal(((index & 1) == 0) ? 1.0 : -1.0, sequence.GetElement(index)); + } + } + + [Fact] + public void LaneOperationsInt32Test() + { + Vector256 left = Vector256.CreateSequence(0, 1); + Vector256 right = Vector256.CreateSequence(100, 1); + int count = Vector256.Count; + int lowerCount = (count + 1) / 2; + int upperStart = count - lowerCount; + + AssertVectorEqual(CreateVector256(index => ((index & 1) == 0) ? left.GetElement(index / 2) : right.GetElement(index / 2)), Vector256.ZipLower(left, right)); + AssertVectorEqual(CreateVector256(index => ((index & 1) == 0) ? left.GetElement(upperStart + (index / 2)) : right.GetElement(upperStart + (index / 2))), Vector256.ZipUpper(left, right)); + + (Vector256 lower, Vector256 upper) = Vector256.Zip(left, right); + AssertVectorEqual(Vector256.ZipLower(left, right), lower); + AssertVectorEqual(Vector256.ZipUpper(left, right), upper); + + AssertVectorEqual(left, Vector256.UnzipEven(lower, upper)); + AssertVectorEqual(right, Vector256.UnzipOdd(lower, upper)); + + (Vector256 even, Vector256 odd) = Vector256.Unzip(lower, upper); + AssertVectorEqual(left, even); + AssertVectorEqual(right, odd); + + AssertVectorEqual(CreateVector256(index => (index < lowerCount) ? left.GetElement(index) : right.GetElement(index - lowerCount)), Vector256.ConcatLowerLower(left, right)); + AssertVectorEqual(CreateVector256(index => (index < lowerCount) ? left.GetElement(upperStart + index) : right.GetElement(index - lowerCount)), Vector256.ConcatUpperLower(left, right)); + AssertVectorEqual(CreateVector256(index => (index < lowerCount) ? left.GetElement(upperStart + index) : right.GetElement(upperStart + index - lowerCount)), Vector256.ConcatUpperUpper(left, right)); + AssertVectorEqual(CreateVector256(index => (index < lowerCount) ? left.GetElement(index) : right.GetElement(upperStart + index - lowerCount)), Vector256.ConcatLowerUpper(left, right)); + + AssertVectorEqual(CreateVector256(index => left.GetElement(count - 1 - index)), Vector256.Reverse(left)); + } + + private static Vector256 CreateVector256(Func elementSelector) + { + int[] values = new int[Vector256.Count]; + + for (int index = 0; index < values.Length; index++) + { + values[index] = elementSelector(index); + } + + return Vector256.Create(values); + } + + private static void AssertVectorEqual(Vector256 expected, Vector256 actual) + where T : struct + { + for (int index = 0; index < Vector256.Count; index++) + { + Assert.Equal(expected.GetElement(index), actual.GetElement(index)); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static T GetNonConstant(T value) => value; + [Theory] [MemberData(nameof(GenericMathTestMemberData.AsinDouble), MemberType = typeof(GenericMathTestMemberData))] public void AsinDoubleTest(double value, double expectedResult, double variance) diff --git a/src/libraries/System.Runtime.Intrinsics/tests/Vectors/Vector512Tests.cs b/src/libraries/System.Runtime.Intrinsics/tests/Vectors/Vector512Tests.cs index d3c430020b5225..1de53d7b93de1c 100644 --- a/src/libraries/System.Runtime.Intrinsics/tests/Vectors/Vector512Tests.cs +++ b/src/libraries/System.Runtime.Intrinsics/tests/Vectors/Vector512Tests.cs @@ -6264,6 +6264,222 @@ private static void TestCreateSequence(T start, T step) } } + [Fact] + public void CreateGeometricSequenceInt32Test() + { + Vector512 sequence = Vector512.CreateGeometricSequence(1, 2); + int expected = 1; + + for (int index = 0; index < Vector512.Count; index++) + { + Assert.Equal(expected, sequence.GetElement(index)); + expected *= 2; + } + } + + [Fact] + public void CreateGeometricSequenceByteWrapsTest() + { + Vector512 sequence = Vector512.CreateGeometricSequence((byte)200, (byte)2); + byte expected = 200; + + for (int index = 0; index < Vector512.Count; index++) + { + Assert.Equal(expected, sequence.GetElement(index)); + expected = unchecked((byte)(expected * 2)); + } + } + + [Fact] + public void CreateGeometricSequenceSingleNonConstantInitialTest() + { + const float multiplier = 1.0064822f; + float initial = GetNonConstant(1.0059024f); + Vector512 sequence = Vector512.CreateGeometricSequence(initial, multiplier); + float expected = initial; + + for (int index = 0; index < Vector512.Count; index++) + { + AssertExtensions.Equal(expected, sequence.GetElement(index), 1e-6f); + expected *= multiplier; + } + } + + [Fact] + public void CreateGeometricSequenceDoubleNonConstantInitialTest() + { + const double multiplier = 1e-50; + double initial = GetNonConstant(1e-154); + Vector512 sequence = Vector512.CreateGeometricSequence(initial, multiplier); + double expected = initial; + + for (int index = 0; index < Vector512.Count; index++) + { + AssertExtensions.Equal(expected, sequence.GetElement(index), 1e-12); + expected *= multiplier; + } + } + + [Fact] + public void CreateAlternatingSequenceInt32Test() + { + Vector512 sequence = Vector512.CreateAlternatingSequence(5, -5); + + for (int index = 0; index < Vector512.Count; index++) + { + Assert.Equal(((index & 1) == 0) ? 5 : -5, sequence.GetElement(index)); + } + } + + [Fact] + public void CreateAlternatingSequenceUInt32Test() + { + Vector512 sequence = Vector512.CreateAlternatingSequence(5u, uint.MaxValue - 1u); + + for (int index = 0; index < Vector512.Count; index++) + { + Assert.Equal(((index & 1) == 0) ? 5u : uint.MaxValue - 1u, sequence.GetElement(index)); + } + } + + [Fact] + public void CreateAlternatingSequenceDoubleTest() + { + Vector512 sequence = Vector512.CreateAlternatingSequence(1.5, -2.5); + + for (int index = 0; index < Vector512.Count; index++) + { + Assert.Equal(((index & 1) == 0) ? 1.5 : -2.5, sequence.GetElement(index)); + } + } + + [Fact] + public void CreateHarmonicSequenceInt32Test() + { + Vector512 sequence = Vector512.CreateHarmonicSequence(1, 1); + int expected = 1; + + for (int index = 0; index < Vector512.Count; index++) + { + Assert.Equal(1 / expected, sequence.GetElement(index)); + expected += 1; + } + } + + [Fact] + public void CreateHarmonicSequenceSingleTest() + { + Vector512 sequence = Vector512.CreateHarmonicSequence(1.0f, 1.0f); + float expected = 1.0f; + + for (int index = 0; index < Vector512.Count; index++) + { + AssertExtensions.Equal(1.0f / expected, sequence.GetElement(index), 1e-6f); + expected += 1.0f; + } + } + + [Fact] + public void CreateHarmonicSequenceDoubleTest() + { + Vector512 sequence = Vector512.CreateHarmonicSequence(1.0, 1.0); + double expected = 1.0; + + for (int index = 0; index < Vector512.Count; index++) + { + AssertExtensions.Equal(1.0 / expected, sequence.GetElement(index), 1e-15); + expected += 1.0; + } + } + + [Fact] + public void SignSequenceInt32Test() + { + Vector512 sequence = Vector512.SignSequence; + + for (int index = 0; index < Vector512.Count; index++) + { + Assert.Equal(((index & 1) == 0) ? 1 : -1, sequence.GetElement(index)); + } + } + + [Fact] + public void SignSequenceSingleTest() + { + Vector512 sequence = Vector512.SignSequence; + + for (int index = 0; index < Vector512.Count; index++) + { + Assert.Equal(((index & 1) == 0) ? 1.0f : -1.0f, sequence.GetElement(index)); + } + } + + [Fact] + public void SignSequenceDoubleTest() + { + Vector512 sequence = Vector512.SignSequence; + + for (int index = 0; index < Vector512.Count; index++) + { + Assert.Equal(((index & 1) == 0) ? 1.0 : -1.0, sequence.GetElement(index)); + } + } + + [Fact] + public void LaneOperationsInt32Test() + { + Vector512 left = Vector512.CreateSequence(0, 1); + Vector512 right = Vector512.CreateSequence(100, 1); + int count = Vector512.Count; + int lowerCount = (count + 1) / 2; + int upperStart = count - lowerCount; + + AssertVectorEqual(CreateVector512(index => ((index & 1) == 0) ? left.GetElement(index / 2) : right.GetElement(index / 2)), Vector512.ZipLower(left, right)); + AssertVectorEqual(CreateVector512(index => ((index & 1) == 0) ? left.GetElement(upperStart + (index / 2)) : right.GetElement(upperStart + (index / 2))), Vector512.ZipUpper(left, right)); + + (Vector512 lower, Vector512 upper) = Vector512.Zip(left, right); + AssertVectorEqual(Vector512.ZipLower(left, right), lower); + AssertVectorEqual(Vector512.ZipUpper(left, right), upper); + + AssertVectorEqual(left, Vector512.UnzipEven(lower, upper)); + AssertVectorEqual(right, Vector512.UnzipOdd(lower, upper)); + + (Vector512 even, Vector512 odd) = Vector512.Unzip(lower, upper); + AssertVectorEqual(left, even); + AssertVectorEqual(right, odd); + + AssertVectorEqual(CreateVector512(index => (index < lowerCount) ? left.GetElement(index) : right.GetElement(index - lowerCount)), Vector512.ConcatLowerLower(left, right)); + AssertVectorEqual(CreateVector512(index => (index < lowerCount) ? left.GetElement(upperStart + index) : right.GetElement(index - lowerCount)), Vector512.ConcatUpperLower(left, right)); + AssertVectorEqual(CreateVector512(index => (index < lowerCount) ? left.GetElement(upperStart + index) : right.GetElement(upperStart + index - lowerCount)), Vector512.ConcatUpperUpper(left, right)); + AssertVectorEqual(CreateVector512(index => (index < lowerCount) ? left.GetElement(index) : right.GetElement(upperStart + index - lowerCount)), Vector512.ConcatLowerUpper(left, right)); + + AssertVectorEqual(CreateVector512(index => left.GetElement(count - 1 - index)), Vector512.Reverse(left)); + } + + private static Vector512 CreateVector512(Func elementSelector) + { + int[] values = new int[Vector512.Count]; + + for (int index = 0; index < values.Length; index++) + { + values[index] = elementSelector(index); + } + + return Vector512.Create(values); + } + + private static void AssertVectorEqual(Vector512 expected, Vector512 actual) + where T : struct + { + for (int index = 0; index < Vector512.Count; index++) + { + Assert.Equal(expected.GetElement(index), actual.GetElement(index)); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static T GetNonConstant(T value) => value; + [Theory] [MemberData(nameof(GenericMathTestMemberData.AsinDouble), MemberType = typeof(GenericMathTestMemberData))] public void AsinDoubleTest(double value, double expectedResult, double variance) diff --git a/src/libraries/System.Runtime.Intrinsics/tests/Vectors/Vector64Tests.cs b/src/libraries/System.Runtime.Intrinsics/tests/Vectors/Vector64Tests.cs index be91c3325549fb..635a3bdb41f39d 100644 --- a/src/libraries/System.Runtime.Intrinsics/tests/Vectors/Vector64Tests.cs +++ b/src/libraries/System.Runtime.Intrinsics/tests/Vectors/Vector64Tests.cs @@ -4579,6 +4579,293 @@ private static void TestCreateSequence(T start, T step) } } + [Fact] + public void CreateGeometricSequenceInt32Test() + { + Vector64 sequence = Vector64.CreateGeometricSequence(1, 2); + int expected = 1; + + for (int index = 0; index < Vector64.Count; index++) + { + Assert.Equal(expected, sequence.GetElement(index)); + expected *= 2; + } + } + + [Fact] + public void CreateGeometricSequenceByteWrapsTest() + { + Vector64 sequence = Vector64.CreateGeometricSequence((byte)200, (byte)2); + byte expected = 200; + + for (int index = 0; index < Vector64.Count; index++) + { + Assert.Equal(expected, sequence.GetElement(index)); + expected = unchecked((byte)(expected * 2)); + } + } + + [Fact] + public void CreateGeometricSequenceSingleNonConstantInitialTest() + { + const float multiplier = 1.0064822f; + float initial = GetNonConstant(1.0059024f); + Vector64 sequence = Vector64.CreateGeometricSequence(initial, multiplier); + float expected = initial; + + for (int index = 0; index < Vector64.Count; index++) + { + AssertExtensions.Equal(expected, sequence.GetElement(index), 1e-6f); + expected *= multiplier; + } + } + + [Fact] + public void CreateGeometricSequenceDoubleNonConstantInitialTest() + { + const double multiplier = 1e-50; + double initial = GetNonConstant(1e-154); + Vector64 sequence = Vector64.CreateGeometricSequence(initial, multiplier); + double expected = initial; + + for (int index = 0; index < Vector64.Count; index++) + { + AssertExtensions.Equal(expected, sequence.GetElement(index), 1e-12); + expected *= multiplier; + } + } + + [Fact] + public void CreateAlternatingSequenceInt32Test() + { + Vector64 sequence = Vector64.CreateAlternatingSequence(5, -5); + + for (int index = 0; index < Vector64.Count; index++) + { + Assert.Equal(((index & 1) == 0) ? 5 : -5, sequence.GetElement(index)); + } + } + + [Fact] + public void CreateAlternatingSequenceUInt32Test() + { + Vector64 sequence = Vector64.CreateAlternatingSequence(5u, uint.MaxValue - 1u); + + for (int index = 0; index < Vector64.Count; index++) + { + Assert.Equal(((index & 1) == 0) ? 5u : uint.MaxValue - 1u, sequence.GetElement(index)); + } + } + + [Fact] + public void CreateAlternatingSequenceDoubleTest() + { + Vector64 sequence = Vector64.CreateAlternatingSequence(1.5, -2.5); + + for (int index = 0; index < Vector64.Count; index++) + { + Assert.Equal(((index & 1) == 0) ? 1.5 : -2.5, sequence.GetElement(index)); + } + } + + [Fact] + public void CreateAlternatingSequenceDoubleCountOneEvaluatesOddTest() + { + int sideEffects = 0; + Vector64 sequence = Vector64.CreateAlternatingSequence( + 1.0, + GetValueWithSideEffect(-2.0, ref sideEffects)); + + Assert.Equal(1, sideEffects); + Assert.Equal(1.0, sequence.GetElement(0)); + } + + [Fact] + public void CreateHarmonicSequenceInt32Test() + { + Vector64 sequence = Vector64.CreateHarmonicSequence(1, 1); + int expected = 1; + + for (int index = 0; index < Vector64.Count; index++) + { + Assert.Equal(1 / expected, sequence.GetElement(index)); + expected += 1; + } + } + + [Fact] + public void CreateHarmonicSequenceSingleTest() + { + Vector64 sequence = Vector64.CreateHarmonicSequence(1.0f, 1.0f); + float expected = 1.0f; + + for (int index = 0; index < Vector64.Count; index++) + { + AssertExtensions.Equal(1.0f / expected, sequence.GetElement(index), 1e-6f); + expected += 1.0f; + } + } + + [Fact] + public void CreateHarmonicSequenceDoubleTest() + { + Vector64 sequence = Vector64.CreateHarmonicSequence(1.0, 1.0); + double expected = 1.0; + + for (int index = 0; index < Vector64.Count; index++) + { + AssertExtensions.Equal(1.0 / expected, sequence.GetElement(index), 1e-15); + expected += 1.0; + } + } + + [Fact] + public void SignSequenceInt32Test() + { + Vector64 sequence = Vector64.SignSequence; + + for (int index = 0; index < Vector64.Count; index++) + { + Assert.Equal(((index & 1) == 0) ? 1 : -1, sequence.GetElement(index)); + } + } + + [Fact] + public void SignSequenceSingleTest() + { + Vector64 sequence = Vector64.SignSequence; + + for (int index = 0; index < Vector64.Count; index++) + { + Assert.Equal(((index & 1) == 0) ? 1.0f : -1.0f, sequence.GetElement(index)); + } + } + + [Fact] + public void SignSequenceDoubleTest() + { + Vector64 sequence = Vector64.SignSequence; + + for (int index = 0; index < Vector64.Count; index++) + { + Assert.Equal(((index & 1) == 0) ? 1.0 : -1.0, sequence.GetElement(index)); + } + } + + [Fact] + public void LaneOperationsInt32Test() + { + Vector64 left = Vector64.CreateSequence(0, 1); + Vector64 right = Vector64.CreateSequence(100, 1); + int count = Vector64.Count; + int lowerCount = (count + 1) / 2; + int upperStart = count - lowerCount; + + AssertVectorEqual(CreateVector64(index => ((index & 1) == 0) ? left.GetElement(index / 2) : right.GetElement(index / 2)), Vector64.ZipLower(left, right)); + AssertVectorEqual(CreateVector64(index => ((index & 1) == 0) ? left.GetElement(upperStart + (index / 2)) : right.GetElement(upperStart + (index / 2))), Vector64.ZipUpper(left, right)); + + (Vector64 lower, Vector64 upper) = Vector64.Zip(left, right); + AssertVectorEqual(Vector64.ZipLower(left, right), lower); + AssertVectorEqual(Vector64.ZipUpper(left, right), upper); + + AssertVectorEqual(left, Vector64.UnzipEven(lower, upper)); + AssertVectorEqual(right, Vector64.UnzipOdd(lower, upper)); + + (Vector64 even, Vector64 odd) = Vector64.Unzip(lower, upper); + AssertVectorEqual(left, even); + AssertVectorEqual(right, odd); + + AssertVectorEqual(CreateVector64(index => (index < lowerCount) ? left.GetElement(index) : right.GetElement(index - lowerCount)), Vector64.ConcatLowerLower(left, right)); + AssertVectorEqual(CreateVector64(index => (index < lowerCount) ? left.GetElement(upperStart + index) : right.GetElement(index - lowerCount)), Vector64.ConcatUpperLower(left, right)); + AssertVectorEqual(CreateVector64(index => (index < lowerCount) ? left.GetElement(upperStart + index) : right.GetElement(upperStart + index - lowerCount)), Vector64.ConcatUpperUpper(left, right)); + AssertVectorEqual(CreateVector64(index => (index < lowerCount) ? left.GetElement(index) : right.GetElement(upperStart + index - lowerCount)), Vector64.ConcatLowerUpper(left, right)); + + AssertVectorEqual(CreateVector64(index => left.GetElement(count - 1 - index)), Vector64.Reverse(left)); + } + + [Fact] + public void LaneOperationsInt64CountOneTest() + { + Vector64 left = Vector64.Create(10L); + Vector64 right = Vector64.Create(20L); + + AssertVectorEqual(left, Vector64.ZipLower(left, right)); + AssertVectorEqual(left, Vector64.ZipUpper(left, right)); + + (Vector64 lower, Vector64 upper) = Vector64.Zip(left, right); + AssertVectorEqual(left, lower); + AssertVectorEqual(left, upper); + + AssertVectorEqual(left, Vector64.UnzipEven(left, right)); + AssertVectorEqual(Vector64.Zero, Vector64.UnzipOdd(left, right)); + + (Vector64 even, Vector64 odd) = Vector64.Unzip(left, right); + AssertVectorEqual(left, even); + AssertVectorEqual(Vector64.Zero, odd); + + AssertVectorEqual(left, Vector64.ConcatLowerLower(left, right)); + AssertVectorEqual(left, Vector64.ConcatUpperLower(left, right)); + AssertVectorEqual(left, Vector64.ConcatUpperUpper(left, right)); + AssertVectorEqual(left, Vector64.ConcatLowerUpper(left, right)); + } + + [Fact] + public void LaneOperationsDoubleCountOneTest() + { + Vector64 left = Vector64.Create(10.0); + Vector64 right = Vector64.Create(20.0); + + AssertVectorEqual(left, Vector64.ZipLower(left, right)); + AssertVectorEqual(left, Vector64.ZipUpper(left, right)); + + (Vector64 lower, Vector64 upper) = Vector64.Zip(left, right); + AssertVectorEqual(left, lower); + AssertVectorEqual(left, upper); + + AssertVectorEqual(left, Vector64.UnzipEven(left, right)); + AssertVectorEqual(Vector64.Zero, Vector64.UnzipOdd(left, right)); + + (Vector64 even, Vector64 odd) = Vector64.Unzip(left, right); + AssertVectorEqual(left, even); + AssertVectorEqual(Vector64.Zero, odd); + + AssertVectorEqual(left, Vector64.ConcatLowerLower(left, right)); + AssertVectorEqual(left, Vector64.ConcatUpperLower(left, right)); + AssertVectorEqual(left, Vector64.ConcatUpperUpper(left, right)); + AssertVectorEqual(left, Vector64.ConcatLowerUpper(left, right)); + } + + private static Vector64 CreateVector64(Func elementSelector) + { + int[] values = new int[Vector64.Count]; + + for (int index = 0; index < values.Length; index++) + { + values[index] = elementSelector(index); + } + + return Vector64.Create(values); + } + + private static void AssertVectorEqual(Vector64 expected, Vector64 actual) + where T : struct + { + for (int index = 0; index < Vector64.Count; index++) + { + Assert.Equal(expected.GetElement(index), actual.GetElement(index)); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static T GetNonConstant(T value) => value; + + [MethodImpl(MethodImplOptions.NoInlining)] + private static T GetValueWithSideEffect(T value, ref int sideEffects) + { + sideEffects++; + return value; + } + [Theory] [MemberData(nameof(GenericMathTestMemberData.AsinDouble), MemberType = typeof(GenericMathTestMemberData))] public void AsinDoubleTest(double value, double expectedResult, double variance)