diff --git a/tests/BenchmarkDotNet.IntegrationTests/Diagnosers/MockInProcessDiagnoser.cs b/tests/BenchmarkDotNet.IntegrationTests/Diagnosers/MockInProcessDiagnoser.cs index a311a26d1e..fb89dcee64 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/Diagnosers/MockInProcessDiagnoser.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/Diagnosers/MockInProcessDiagnoser.cs @@ -11,19 +11,25 @@ namespace BenchmarkDotNet.IntegrationTests.Diagnosers; -public sealed class MockInProcessDiagnoser : IInProcessDiagnoser +public abstract class BaseMockInProcessDiagnoser : IInProcessDiagnoser { + public static Queue s_completedResults = new(); + public Dictionary Results { get; } = []; - public IEnumerable Ids => [nameof(MockInProcessDiagnoser)]; + public abstract string DiagnoserName { get; } + public abstract RunMode DiagnoserRunMode { get; } + public abstract string ExpectedResult { get; } + + public IEnumerable Ids => [DiagnoserName]; public IEnumerable Exporters => []; public IEnumerable Analysers => []; - public void DisplayResults(ILogger logger) => logger.WriteLine($"{nameof(MockInProcessDiagnoser)} results: [{string.Join(", ", Results.Values)}]"); + public void DisplayResults(ILogger logger) => logger.WriteLine($"{DiagnoserName} results: [{string.Join(", ", Results.Values)}]"); - public RunMode GetRunMode(BenchmarkCase benchmarkCase) => RunMode.NoOverhead; + public RunMode GetRunMode(BenchmarkCase benchmarkCase) => DiagnoserRunMode; public void Handle(HostSignal signal, DiagnoserActionParameters parameters) { } @@ -31,20 +37,96 @@ public void Handle(HostSignal signal, DiagnoserActionParameters parameters) { } public IEnumerable Validate(ValidationParameters validationParameters) => []; - public (Type? handlerType, string? serializedConfig) GetSeparateProcessHandlerTypeAndSerializedConfig(BenchmarkCase benchmarkCase) - => (typeof(MockInProcessDiagnoserHandler), null); + public abstract (Type? handlerType, string? serializedConfig) GetSeparateProcessHandlerTypeAndSerializedConfig(BenchmarkCase benchmarkCase); + + public virtual IInProcessDiagnoserHandler? GetSameProcessHandler(BenchmarkCase benchmarkCase) + { + var (handlerType, serializedConfig) = GetSeparateProcessHandlerTypeAndSerializedConfig(benchmarkCase); + if (handlerType == null) + return null; + var handler = (IInProcessDiagnoserHandler)Activator.CreateInstance(handlerType); + handler.Initialize(serializedConfig); + return handler; + } + + public void DeserializeResults(BenchmarkCase benchmarkCase, string results) + { + Results.Add(benchmarkCase, results); + s_completedResults.Enqueue(this); + } +} + +public abstract class BaseMockInProcessDiagnoserHandler : IInProcessDiagnoserHandler +{ + private string _result; - public IInProcessDiagnoserHandler? GetSameProcessHandler(BenchmarkCase benchmarkCase) - => new MockInProcessDiagnoserHandler(); + protected BaseMockInProcessDiagnoserHandler() { } - public void DeserializeResults(BenchmarkCase benchmarkCase, string results) => Results.Add(benchmarkCase, results); + public void Initialize(string? serializedConfig) + { + _result = serializedConfig ?? string.Empty; + } + + public void Handle(BenchmarkSignal signal, InProcessDiagnoserActionArgs args) { } + + public string SerializeResults() => _result; } -public sealed class MockInProcessDiagnoserHandler : IInProcessDiagnoserHandler +public sealed class MockInProcessDiagnoser : BaseMockInProcessDiagnoser { - public void Initialize(string? serializedConfig) { } + public override string DiagnoserName => nameof(MockInProcessDiagnoser); + public override RunMode DiagnoserRunMode => RunMode.NoOverhead; + public override string ExpectedResult => "MockResult"; - public void Handle(BenchmarkSignal signal, InProcessDiagnoserActionArgs args) { } + public override (Type? handlerType, string? serializedConfig) GetSeparateProcessHandlerTypeAndSerializedConfig(BenchmarkCase benchmarkCase) + => (typeof(MockInProcessDiagnoserHandler), ExpectedResult); +} + +public sealed class MockInProcessDiagnoserHandler : BaseMockInProcessDiagnoserHandler +{ +} + +public sealed class MockInProcessDiagnoserExtraRun : BaseMockInProcessDiagnoser +{ + public override string DiagnoserName => nameof(MockInProcessDiagnoserExtraRun); + public override RunMode DiagnoserRunMode => RunMode.ExtraRun; + public override string ExpectedResult => "ExtraRunResult"; + + public override (Type? handlerType, string? serializedConfig) GetSeparateProcessHandlerTypeAndSerializedConfig(BenchmarkCase benchmarkCase) + => (typeof(MockInProcessDiagnoserExtraRunHandler), ExpectedResult); +} - public string SerializeResults() => "MockResult"; +public sealed class MockInProcessDiagnoserExtraRunHandler : BaseMockInProcessDiagnoserHandler +{ +} + +public sealed class MockInProcessDiagnoserNone : BaseMockInProcessDiagnoser +{ + public override string DiagnoserName => nameof(MockInProcessDiagnoserNone); + public override RunMode DiagnoserRunMode => RunMode.None; + public override string ExpectedResult => "NoneResult"; + + public override (Type? handlerType, string? serializedConfig) GetSeparateProcessHandlerTypeAndSerializedConfig(BenchmarkCase benchmarkCase) + => default; // Returns default when RunMode is None + + public override IInProcessDiagnoserHandler? GetSameProcessHandler(BenchmarkCase benchmarkCase) + => null; // Returns null when RunMode is None +} + +public sealed class MockInProcessDiagnoserNoneHandler : BaseMockInProcessDiagnoserHandler +{ +} + +public sealed class MockInProcessDiagnoserSeparateLogic : BaseMockInProcessDiagnoser +{ + public override string DiagnoserName => nameof(MockInProcessDiagnoserSeparateLogic); + public override RunMode DiagnoserRunMode => RunMode.SeparateLogic; + public override string ExpectedResult => "SeparateLogicResult"; + + public override (Type? handlerType, string? serializedConfig) GetSeparateProcessHandlerTypeAndSerializedConfig(BenchmarkCase benchmarkCase) + => (typeof(MockInProcessDiagnoserSeparateLogicHandler), ExpectedResult); +} + +public sealed class MockInProcessDiagnoserSeparateLogicHandler : BaseMockInProcessDiagnoserHandler +{ } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests/InProcessDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/InProcessDiagnoserTests.cs new file mode 100644 index 0000000000..749e3a10be --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests/InProcessDiagnoserTests.cs @@ -0,0 +1,150 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Columns; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.IntegrationTests.Diagnosers; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Tests.Loggers; +using BenchmarkDotNet.Toolchains; +using BenchmarkDotNet.Toolchains.InProcess.Emit; +using BenchmarkDotNet.Toolchains.InProcess.NoEmit; +using Xunit; +using Xunit.Abstractions; +using RunMode = BenchmarkDotNet.Diagnosers.RunMode; + +namespace BenchmarkDotNet.IntegrationTests; + +public class InProcessDiagnoserTests(ITestOutputHelper output) : BenchmarkTestExecutor(output) +{ + // For test explorer since it doesn't handle interfaces well. + public enum ToolchainType + { + Default, + InProcessEmit, + InProcessNoEmit, + } + + private static IEnumerable GetRunModeCombinations(int count) + { + var runModes = (RunMode[]) Enum.GetValues(typeof(RunMode)); + + if (count == 1) + { + foreach (var runMode in runModes) + { + yield return [runMode]; + } + } + else if (count == 3) + { + foreach (var runMode1 in runModes) + { + foreach (var runMode2 in runModes) + { + foreach (var runMode3 in runModes) + { + yield return [runMode1, runMode2, runMode3]; + } + } + } + } + } + + public static IEnumerable GetTestCombinations() + { + var toolchains = (ToolchainType[]) Enum.GetValues(typeof(ToolchainType)); + var counts = new[] { 1, 3 }; + + foreach (var toolchain in toolchains) + { + foreach (var count in counts) + { + foreach (var runModes in GetRunModeCombinations(count)) + { + yield return [runModes, toolchain]; + } + } + } + } + + private static BaseMockInProcessDiagnoser CreateDiagnoser(RunMode runMode) + => runMode switch + { + RunMode.None => new MockInProcessDiagnoserNone(), + RunMode.NoOverhead => new MockInProcessDiagnoser(), + RunMode.ExtraRun => new MockInProcessDiagnoserExtraRun(), + RunMode.SeparateLogic => new MockInProcessDiagnoserSeparateLogic(), + _ => throw new ArgumentException($"Unsupported run mode: {runMode}") + }; + + private ManualConfig CreateConfig(ToolchainType toolchain) + { + var job = toolchain switch + { + ToolchainType.InProcessEmit => Job.Dry.WithToolchain(InProcessEmitToolchain.Instance), + ToolchainType.InProcessNoEmit => Job.Dry.WithToolchain(InProcessNoEmitToolchain.Instance), + _ => Job.Dry + }; + + return new ManualConfig() + .AddLogger(new OutputLogger(Output)) + .AddColumnProvider(DefaultColumnProviders.Instance) + .AddJob(job); + } + + [Theory] + [MemberData(nameof(GetTestCombinations))] + public void MultipleInProcessDiagnosersWork(RunMode[] runModes, ToolchainType toolchain) + { + var diagnosers = runModes.Select(CreateDiagnoser).ToArray(); + var config = CreateConfig(toolchain); + + foreach (var diagnoser in diagnosers) + { + config = config.AddDiagnoser(diagnoser); + } + + var summary = CanExecute(config); + + foreach (var diagnoser in diagnosers) + { + if (diagnoser.DiagnoserRunMode == RunMode.None) + { + Assert.Empty(diagnoser.Results); + } + else + { + Assert.NotEmpty(diagnoser.Results); + Assert.Equal(summary.BenchmarksCases.Length, diagnoser.Results.Count); + Assert.All(diagnoser.Results.Values, result => Assert.Equal(diagnoser.ExpectedResult, result)); + } + } + Assert.Equal( + BaseMockInProcessDiagnoser.s_completedResults, + diagnosers + .Where(d => d.DiagnoserRunMode != RunMode.None) + .OrderBy(d => d.DiagnoserRunMode switch + { + RunMode.NoOverhead => 0, + RunMode.ExtraRun => 1, + RunMode.SeparateLogic => 2, + _ => 3 + }) + ); + BaseMockInProcessDiagnoser.s_completedResults.Clear(); + } + + public class SimpleBenchmark + { + private int counter; + + [Benchmark] + public void BenchmarkMethod() + { + Interlocked.Increment(ref counter); + } + } +} diff --git a/tests/BenchmarkDotNet.IntegrationTests/InProcessEmitTest.cs b/tests/BenchmarkDotNet.IntegrationTests/InProcessEmitTest.cs index bb7ec8cf10..d358712cc0 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/InProcessEmitTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/InProcessEmitTest.cs @@ -131,19 +131,6 @@ public void InProcessBenchmarkEmitsSameIL(Type benchmarkType) Assert.DoesNotContain("No benchmarks found", logger.GetLog()); } - [Fact] - public void InProcessEmitSupportsInProcessDiagnosers() - { - var logger = new OutputLogger(Output); - var diagnoser = new MockInProcessDiagnoser(); - var config = CreateInProcessConfig(logger).AddDiagnoser(diagnoser); - - var summary = CanExecute(config); - - var expected = Enumerable.Repeat("MockResult", summary.BenchmarksCases.Length); - Assert.Equal(expected, diagnoser.Results.Values); - } - [UsedImplicitly(ImplicitUseTargetFlags.WithMembers)] public class BenchmarkAllCases { diff --git a/tests/BenchmarkDotNet.IntegrationTests/InProcessTest.cs b/tests/BenchmarkDotNet.IntegrationTests/InProcessTest.cs index 213ebe4319..e2efa4cd92 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/InProcessTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/InProcessTest.cs @@ -218,19 +218,6 @@ public void InProcessBenchmarkAllCasesSupported() } } - [Fact] - public void InProcessNoEmitSupportsInProcessDiagnosers() - { - var logger = new OutputLogger(Output); - var diagnoser = new MockInProcessDiagnoser(); - var config = CreateInProcessConfig(logger).AddDiagnoser(diagnoser); - - var summary = CanExecute(config); - - var expected = Enumerable.Repeat("MockResult", summary.BenchmarksCases.Length); - Assert.Equal(expected, diagnoser.Results.Values); - } - [UsedImplicitly(ImplicitUseTargetFlags.WithMembers)] public class BenchmarkAllCases { diff --git a/tests/BenchmarkDotNet.IntegrationTests/NativeAotTests.cs b/tests/BenchmarkDotNet.IntegrationTests/NativeAotTests.cs index 9ef4512165..5924a3e7b8 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/NativeAotTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/NativeAotTests.cs @@ -79,7 +79,9 @@ public void NativeAotSupportsInProcessDiagnosers() throw; } - Assert.Equal(["MockResult"], diagnoser.Results.Values); + Assert.Equal([diagnoser.ExpectedResult], diagnoser.Results.Values); + Assert.Equal([diagnoser], BaseMockInProcessDiagnoser.s_completedResults); + BaseMockInProcessDiagnoser.s_completedResults.Clear(); } private static bool GetShouldRunTest() diff --git a/tests/BenchmarkDotNet.IntegrationTests/WasmTests.cs b/tests/BenchmarkDotNet.IntegrationTests/WasmTests.cs index aff5e85f7e..4c4fe9f8d0 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/WasmTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/WasmTests.cs @@ -56,7 +56,9 @@ public void WasmSupportsInProcessDiagnosers() CanExecute(config); - Assert.Equal(["MockResult"], diagnoser.Results.Values); + Assert.Equal([diagnoser.ExpectedResult], diagnoser.Results.Values); + Assert.Equal([diagnoser], BaseMockInProcessDiagnoser.s_completedResults); + BaseMockInProcessDiagnoser.s_completedResults.Clear(); } public class WasmBenchmark