Skip to content

Commit

Permalink
JIT ARM64-SVE: Avoid combining conditional select for reduction intri…
Browse files Browse the repository at this point in the history 
…nsics (#107207)
  • Loading branch information
@SwapnilGaikwad
SwapnilGaikwad authored Sep 3, 2024
1 parent eef24a4 commit 46c9a4f
Show file tree
Hide file tree
Showing 11 changed files with 383 additions and 12 deletions.
9 changes: 9 additions & 0 deletions src/coreclr/jit/hwintrinsic.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -229,6 +229,9 @@ enum HWIntrinsicFlag : unsigned int
// (HW_Flag_BaseTypeFrom{First, Second}Arg must also be set to denote the position of the ValueTuple)
HW_Flag_BaseTypeFromValueTupleArg = 0x1000000,

// The intrinsic is a reduce operation.
HW_Flag_ReduceOperation = 0x2000000,

#else
#error Unsupported platform
#endif
Expand Down Expand Up @@ -998,6 +1001,12 @@ struct HWIntrinsicInfo
return (flags & HW_Flag_BaseTypeFromValueTupleArg) != 0;
}

static bool IsReduceOperation(NamedIntrinsic id)
{
const HWIntrinsicFlag flags = lookupFlags(id);
return (flags & HW_Flag_ReduceOperation) != 0;
}

static NamedIntrinsic GetScalarInputVariant(NamedIntrinsic id)
{
assert(HasScalarInputVariant(id));
Expand Down
18 changes: 9 additions & 9 deletions src/coreclr/jit/hwintrinsiclistarm64sve.h
View file

Large diffs are not rendered by default.

5 changes: 3 additions & 2 deletions src/coreclr/jit/lowerarmarch.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4055,9 +4055,10 @@ GenTree* Lowering::LowerHWIntrinsicCndSel(GenTreeHWIntrinsic* cndSelNode)
NamedIntrinsic nestedOp2Id = nestedOp2->AsHWIntrinsic()->GetHWIntrinsicId();

// If the nested op uses Pg/Z, then inactive lanes will result in zeros, so can only transform if
// op3 is all zeros.
// op3 is all zeros. Such a Csel operation is absorbed into the instruction when emitted. Skip this optimisation
// when the nestedOp is a reduce operation.

if (nestedOp1->IsMaskAllBitsSet() &&
if (nestedOp1->IsMaskAllBitsSet() && !HWIntrinsicInfo::IsReduceOperation(nestedOp2Id) &&
(!HWIntrinsicInfo::IsZeroingMaskedOperation(nestedOp2Id) || op3->IsVectorZero()))
{
GenTree* nestedOp2 = nestedCndSel->Op(2);
Expand Down
63 changes: 62 additions & 1 deletion src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -220,6 +220,67 @@
}
}";

const string VecReduceUnOpTest_VectorValidationLogicForCndSel = @"
{
var hasFailed = (mask[0] != 0 ? {ValidateReduceOpResult}: (falseVal[0] != result[0]));

if (hasFailed)
{
succeeded = false;
}
else
{
for (var i = 1; i < RetElementCount; i++)
{
var iterResult = (mask[i] != 0) ? 0 : falseVal[i];
if (mask[i] != 0)
{
// Pick the trueValue
if (iterResult != result[i])
{
succeeded = false;
break;
}
}
else
{
// For false, the values are merged with destination, and we do not know
// those contents would be, so skip verification for them.
}
}
}
}";

const string VecReduceUnOpTest_VectorValidationLogicForCndSel_FalseValue = @"
{
var hasFailed = (mask[0] != 0) ? (trueVal[0] != result[0]): {ValidateReduceOpResult};
if (hasFailed)
{
succeeded = false;
}
else
{
for (var i = 1; i < RetElementCount; i++)
{
var iterResult = (mask[i] != 0) ? trueVal[i] : 0;
if (mask[i] != 0)
{
// Pick the trueValue
if (iterResult != result[i])
{
succeeded = false;
break;
}
}
else
{
// For false, the values are merged with destination, and we do not know
// those contents would be, so skip verification for them.
}
}
}
}";

const string VecReduceOpTest_ValidationLogic = @"if ({ValidateReduceOpResult})
{
succeeded = false;
Expand Down Expand Up @@ -293,7 +354,7 @@
("_SveImmTernOpFirstArgTestTemplate.template", "SveVecImmTernOpFirstArgTest.template", new Dictionary<string, string> { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = SimpleVecOpTest_ValidationLogic, ["TemplateValidationLogicForCndSel"] = SimpleTernVecOpTest_ValidationLogicForCndSel, ["TemplateValidationLogicForCndSel_FalseValue"] = SimpleTernVecOpTest_ValidationLogicForCndSel_FalseValue }),
("_SveScalarTernOpTestTemplate.template", "SveScalarTernOpTest.template", new Dictionary<string, string> { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = SimpleScalarOpTest_ValidationLogic }),
("_SveImm2UnaryOpTestTemplate.template", "SveVecImm2UnOpTest.template", new Dictionary<string, string> { ["TemplateName"] = "Imm", ["TemplateValidationLogic"] = SimpleVecOpTest_ValidationLogic }),
("_SveMinimalUnaryOpTestTemplate.template", "SveVecReduceUnOpTest.template", new Dictionary<string, string> { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = VecReduceOpTest_ValidationLogic }),
("_SveMinimalUnaryOpTestTemplate.template", "SveVecReduceUnOpTest.template", new Dictionary<string, string> { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = VecReduceOpTest_ValidationLogic, ["TemplateValidationLogicForCndSel"] = VecReduceUnOpTest_VectorValidationLogicForCndSel, ["TemplateValidationLogicForCndSel_FalseValue"] = VecReduceUnOpTest_VectorValidationLogicForCndSel_FalseValue }),
("_SveMasklessUnaryOpTestTemplate.template", "SveMasklessSimpleVecOpTest.template", new Dictionary<string, string> { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = SimpleVecOpTest_ValidationLogic }),
("_SveVecAndScalarOpTest.template", "SveVecAndScalarOpTest.template", new Dictionary<string, string> { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = SimpleVecOpTest_VectorValidationLogic }),
("_SveMasklessBinaryOpTestTemplate.template", "SveMasklessVecBinOpTest.template", new Dictionary<string, string> { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = SimpleVecOpTest_ValidationLogic }),
Expand Down
129 changes: 129 additions & 0 deletions src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveMinimalUnaryOpTestTemplate.template
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,12 @@ namespace JIT.HardwareIntrinsics.Arm

// Validates passing an instance member of a struct works
test.RunStructFldScenario();

// Validates executing the test inside conditional, with op3 as falseValue
test.ConditionalSelect_FalseOp();

// Validates executing the test inside conditional, with op3 as zero
test.ConditionalSelect_ZeroOp();
}
else
{
Expand Down Expand Up @@ -139,18 +145,25 @@ namespace JIT.HardwareIntrinsics.Arm
private static readonly int Op1ElementCount = Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>() / sizeof({Op1BaseType});
private static readonly int RetElementCount = Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>() / sizeof({RetBaseType});

private static {RetBaseType}[] _maskData = new {RetBaseType}[RetElementCount];
private static {Op1BaseType}[] _data1 = new {Op1BaseType}[Op1ElementCount];

private {RetVectorType}<{RetBaseType}> _mask;
private {Op1VectorType}<{Op1BaseType}> _fld1;
private {RetVectorType}<{RetBaseType}> _falseFld;

private DataTable _dataTable;

public {TemplateName}UnaryOpTest__{TestName}()
{
Succeeded = true;

for (var i = 0; i < RetElementCount; i++) { _maskData[i] = ({RetBaseType})(TestLibrary.Generator.Get{RetBaseType}() % 2); }
Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetVectorType}<{RetBaseType}>, byte>(ref _mask), ref Unsafe.As<{RetBaseType}, byte>(ref _maskData[0]), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>());

for (var i = 0; i < Op1ElementCount; i++) { _data1[i] = {NextValueOp1}; }
Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1VectorType}<{Op1BaseType}>, byte>(ref _fld1), ref Unsafe.As<{Op1BaseType}, byte>(ref _data1[0]), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>());
Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetVectorType}<{RetBaseType}>, byte>(ref _falseFld), ref Unsafe.As<{Op1BaseType}, byte>(ref _data1[0]), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>());

for (var i = 0; i < Op1ElementCount; i++) { _data1[i] = {NextValueOp1}; }
_dataTable = new DataTable(_data1, new {RetBaseType}[RetElementCount], LargestVectorSize);
Expand Down Expand Up @@ -239,6 +252,66 @@ namespace JIT.HardwareIntrinsics.Arm
test.RunStructFldScenario(this);
}

public void ConditionalSelect_FalseOp()
{
TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_mask - operation in trueValue");
ConditionalSelectScenario_TrueValue(_mask, _fld1, _falseFld);

TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_zero - operation in trueValue");
ConditionalSelectScenario_TrueValue({RetVectorType}<{RetBaseType}>.Zero, _fld1, _falseFld);

TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_all - operation in trueValue");
ConditionalSelectScenario_TrueValue({RetVectorType}<{RetBaseType}>.AllBitsSet, _fld1, _falseFld);

TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_mask - operation in falseValue");
ConditionalSelectScenario_FalseValue(_mask, _fld1, _falseFld);

TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_zero - operation in falseValue");
ConditionalSelectScenario_FalseValue({RetVectorType}<{RetBaseType}>.Zero, _fld1, _falseFld);

TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_all - operation in falseValue");
ConditionalSelectScenario_FalseValue({RetVectorType}<{RetBaseType}>.AllBitsSet, _fld1, _falseFld);
}

public void ConditionalSelect_ZeroOp()
{
TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_mask - operation in trueValue");
ConditionalSelectScenario_TrueValue(_mask, _fld1, {RetVectorType}<{RetBaseType}>.Zero);

TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_zero - operation in trueValue");
ConditionalSelectScenario_TrueValue({RetVectorType}<{RetBaseType}>.Zero, _fld1, {RetVectorType}<{RetBaseType}>.Zero);

TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_all - operation in trueValue");
ConditionalSelectScenario_TrueValue({RetVectorType}<{RetBaseType}>.AllBitsSet, _fld1, {RetVectorType}<{RetBaseType}>.Zero);

TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_mask - operation in falseValue");
ConditionalSelectScenario_FalseValue(_mask, _fld1, {RetVectorType}<{RetBaseType}>.Zero);

TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_zero - operation in falseValue");
ConditionalSelectScenario_FalseValue({RetVectorType}<{RetBaseType}>.Zero, _fld1, {RetVectorType}<{RetBaseType}>.Zero);

TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_all - operation in falseValue");
ConditionalSelectScenario_FalseValue({RetVectorType}<{RetBaseType}>.AllBitsSet, _fld1, {RetVectorType}<{RetBaseType}>.Zero);
}

[method: MethodImpl(MethodImplOptions.AggressiveInlining)]
private void ConditionalSelectScenario_TrueValue({RetVectorType}<{RetBaseType}> mask, {Op1VectorType}<{Op1BaseType}> op1, {RetVectorType}<{RetBaseType}> falseOp)
{
var result = Sve.ConditionalSelect(mask, {Isa}.{Method}(op1), falseOp);

Unsafe.Write(_dataTable.outArrayPtr, result);
ValidateConditionalSelectResult_TrueValue(mask, op1, falseOp, _dataTable.outArrayPtr);
}

[method: MethodImpl(MethodImplOptions.AggressiveInlining)]
private void ConditionalSelectScenario_FalseValue({RetVectorType}<{RetBaseType}> mask, {Op1VectorType}<{Op1BaseType}> op1, {RetVectorType}<{RetBaseType}> trueOp)
{
var result = Sve.ConditionalSelect(mask, trueOp, {Isa}.{Method}(op1));

Unsafe.Write(_dataTable.outArrayPtr, result);
ValidateConditionalSelectResult_FalseValue(mask, op1, trueOp, _dataTable.outArrayPtr);
}

public void RunUnsupportedScenario()
{
TestLibrary.TestFramework.BeginScenario(nameof(RunUnsupportedScenario));
Expand All @@ -260,6 +333,62 @@ namespace JIT.HardwareIntrinsics.Arm
}
}

private void ValidateConditionalSelectResult_TrueValue({RetVectorType}<{RetBaseType}> maskOp, {Op1VectorType}<{Op1BaseType}> leftOp, {RetVectorType}<{RetBaseType}> falseOp, void* output, [CallerMemberName] string method = "")
{
{RetBaseType}[] mask = new {RetBaseType}[RetElementCount];
{Op1BaseType}[] firstOp = new {Op1BaseType}[Op1ElementCount];
{RetBaseType}[] falseVal = new {RetBaseType}[RetElementCount];
{RetBaseType}[] result = new {RetBaseType}[RetElementCount];

Unsafe.WriteUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref mask[0]), maskOp);
Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref firstOp[0]), leftOp);
Unsafe.WriteUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref falseVal[0]), falseOp);
Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref result[0]), ref Unsafe.AsRef<byte>(output), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>());

bool succeeded = true;
{TemplateValidationLogicForCndSel}

if (!succeeded)
{
TestLibrary.TestFramework.LogInformation($"{nameof(Sve)}.{nameof({Isa}.{Method})}<{RetBaseType}>({RetVectorType}<{RetBaseType}>, {RetVectorType}<{RetBaseType}>): {method} failed:");
TestLibrary.TestFramework.LogInformation($" mask: ({string.Join(", ", mask)})");
TestLibrary.TestFramework.LogInformation($" firstOp: ({string.Join(", ", firstOp)})");
TestLibrary.TestFramework.LogInformation($" falseOp: ({string.Join(", ", falseVal)})");
TestLibrary.TestFramework.LogInformation($" result: ({string.Join(", ", result)})");
TestLibrary.TestFramework.LogInformation(string.Empty);

Succeeded = false;
}
}

private void ValidateConditionalSelectResult_FalseValue({RetVectorType}<{RetBaseType}> maskOp, {Op1VectorType}<{Op1BaseType}> leftOp, {RetVectorType}<{RetBaseType}> trueOp, void* output, [CallerMemberName] string method = "")
{
{RetBaseType}[] mask = new {RetBaseType}[RetElementCount];
{Op1BaseType}[] firstOp = new {Op1BaseType}[Op1ElementCount];
{RetBaseType}[] trueVal = new {RetBaseType}[RetElementCount];
{RetBaseType}[] result = new {RetBaseType}[RetElementCount];

Unsafe.WriteUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref mask[0]), maskOp);
Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref firstOp[0]), leftOp);
Unsafe.WriteUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref trueVal[0]), trueOp);
Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref result[0]), ref Unsafe.AsRef<byte>(output), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>());

bool succeeded = true;
{TemplateValidationLogicForCndSel_FalseValue}

if (!succeeded)
{
TestLibrary.TestFramework.LogInformation($"{nameof(Sve)}.{nameof({Isa}.{Method})}<{RetBaseType}>({RetVectorType}<{RetBaseType}>, {RetVectorType}<{RetBaseType}>): {method} failed:");
TestLibrary.TestFramework.LogInformation($" mask: ({string.Join(", ", mask)})");
TestLibrary.TestFramework.LogInformation($"firstOp: ({string.Join(", ", firstOp)})");
TestLibrary.TestFramework.LogInformation($" trueOp: ({string.Join(", ", trueVal)})");
TestLibrary.TestFramework.LogInformation($" result: ({string.Join(", ", result)})");
TestLibrary.TestFramework.LogInformation(string.Empty);

Succeeded = false;
}
}

private void ValidateResult({Op1VectorType}<{Op1BaseType}> op1, void* result, [CallerMemberName] string method = "")
{
{Op1BaseType}[] inArray1 = new {Op1BaseType}[Op1ElementCount];
Expand Down
57 changes: 57 additions & 0 deletions src/tests/JIT/Regression/JitBlue/Runtime_106868/Runtime_106868.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
using System.Runtime.CompilerServices;
using Xunit;

// Generated by Fuzzlyn v2.3 on 2024-08-23 10:17:54
// Run on Arm64 Windows
// Seed: 14752078066107523191-vectort,vector64,vector128,armsve
// Reduced from 52.7 KiB to 0.7 KiB in 00:00:54
// Hits JIT assert in Release:
// Assertion failed '!"Got unexpected instruction format after MOVPRFX"' in 'Program:Main(Fuzzlyn.ExecutionServer.IRuntime)' during 'Emit code' (IL size 54; hash 0xade6b36b; FullOpts)
//
// File: C:\dev\dotnet\runtime2\src\coreclr\jit\emitarm64sve.cpp Line: 18623
//

using System;
using System.Numerics;
using System.Runtime.Intrinsics;
using System.Runtime.Intrinsics.Arm;

public struct S2
{
public Vector<int> F2;
}

public class Runtime_106868
{
public static S2 s_1;

public static int M4()
{
Vector<long> vr17 = default(Vector<long>);
var vr11 = Vector.Create<long>(0);
var vr8 = Sve.SubtractSaturate(vr17, vr11);
return 1;
}

[Fact]
public static void TestEntryPoint()
{
if (Sve.IsSupported)
{
var vr12 = Vector.Create<int>(0);
var vr13 = Vector.Create<int>(0);
var vr14 = M4();
var vr15 = Vector128.CreateScalar(vr14).AsVector();
var vr16 = Sve.AndAcross(vr13);
s_1.F2 = Sve.ConditionalSelect(vr12, vr16, vr15);
Consume(s_1.F2);
}
}

[MethodImpl(MethodImplOptions.NoInlining)]
private static void Consume(Vector<int> v)
{
}
}
9 changes: 9 additions & 0 deletions src/tests/JIT/Regression/JitBlue/Runtime_106868/Runtime_106868.csproj
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<Optimize>True</Optimize>
<NoWarn>$(NoWarn),SYSLIB5003</NoWarn>
</PropertyGroup>
<ItemGroup>
<Compile Include="$(MSBuildProjectName).cs" />
</ItemGroup>
</Project>
Loading

0 comments on commit 46c9a4f

Please sign in to comment.