Skip to content

Commit cd9e4e9

Browse files
FlatlinerDOAAndreyAkinshin
FlatlinerDOA
authored and
AndreyAkinshin
committed
Feature - Added Box plots
Also improved readability by setting most font sizes to 14 by default (customisable)
1 parent 3927ce7 commit cd9e4e9

File tree

3 files changed

+293
-8
lines changed

3 files changed

+293
-8
lines changed

‎src/BenchmarkDotNet.Exporters.Plotting/ScottPlotExporter.cs

Lines changed: 186 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ public ScottPlotExporter(int width = 1920, int height = 1080)
3636
this.Width = width;
3737
this.Height = height;
3838
this.IncludeBarPlot = true;
39+
this.IncludeBoxPlot = true;
3940
this.RotateLabels = true;
4041
}
4142

@@ -49,6 +50,16 @@ public ScottPlotExporter(int width = 1920, int height = 1080)
4950
/// </summary>
5051
public int Height { get; set; }
5152

53+
/// <summary>
54+
/// Gets or sets the common font size for ticks, labels etc. (defaults to 14).
55+
/// </summary>
56+
public int FontSize { get; set; } = 14;
57+
58+
/// <summary>
59+
/// Gets or sets the font size for the chart title. (defaults to 28).
60+
/// </summary>
61+
public int TitleFontSize { get; set; } = 28;
62+
5263
/// <summary>
5364
/// Gets or sets a value indicating whether labels for Plot X-axis should be rotated.
5465
/// This allows for longer labels at the expense of chart height.
@@ -61,6 +72,12 @@ public ScottPlotExporter(int width = 1920, int height = 1080)
6172
/// </summary>
6273
public bool IncludeBarPlot { get; set; }
6374

75+
/// <summary>
76+
/// Gets or sets a value indicating whether a box plot or whisker plot for time-per-op
77+
/// measurement values should be exported.
78+
/// </summary>
79+
public bool IncludeBoxPlot { get; set; }
80+
6481
/// <summary>
6582
/// Not supported.
6683
/// </summary>
@@ -98,7 +115,7 @@ from measurement in report.AllMeasurements
98115
where measurement.Is(IterationMode.Workload, IterationStage.Result)
99116
let measurementValue = measurement.Nanoseconds / measurement.Operations
100117
group measurementValue / timeScale by (Target: report.BenchmarkCase.Descriptor.WorkloadMethodDisplayInfo, JobId: jobId) into g
101-
select (g.Key.Target, g.Key.JobId, Mean: g.Average(), StdError: StandardError(g.ToList()));
118+
select new ChartStats(g.Key.Target, g.Key.JobId, g.ToList());
102119

103120
if (this.IncludeBarPlot)
104121
{
@@ -112,8 +129,19 @@ where measurement.Is(IterationMode.Workload, IterationStage.Result)
112129
annotations);
113130
}
114131

132+
if (this.IncludeBoxPlot)
133+
{
134+
// <BenchmarkName>-boxplot.png
135+
yield return CreateBoxPlot(
136+
$"{title} - {benchmarkName}",
137+
Path.Combine(summary.ResultsDirectoryPath, $"{title}-{benchmarkName}-boxplot.png"),
138+
$"Time ({timeUnit})",
139+
"Target",
140+
timeStats,
141+
annotations);
142+
}
143+
115144
/* TODO: Rest of the RPlotExporter plots.
116-
<BenchmarkName>-boxplot.png
117145
<BenchmarkName>-<MethodName>-density.png
118146
<BenchmarkName>-<MethodName>-facetTimeline.png
119147
<BenchmarkName>-<MethodName>-facetTimelineSmooth.png
@@ -161,12 +189,12 @@ private static double StandardError(IReadOnlyList<double> values)
161189
return ("ns", 1d);
162190
}
163191

164-
private string CreateBarPlot(string title, string fileName, string yLabel, string xLabel, IEnumerable<(string Target, string JobId, double Mean, double StdError)> data, IReadOnlyList<Annotation> annotations)
192+
private string CreateBarPlot(string title, string fileName, string yLabel, string xLabel, IEnumerable<ChartStats> data, IReadOnlyList<Annotation> annotations)
165193
{
166194
Plot plt = new Plot();
167-
plt.Title(title, 28);
168-
plt.YLabel(yLabel);
169-
plt.XLabel(xLabel);
195+
plt.Title(title, this.TitleFontSize);
196+
plt.YLabel(yLabel, this.FontSize);
197+
plt.XLabel(xLabel, this.FontSize);
170198

171199
var palette = new ScottPlot.Palettes.Category10();
172200

@@ -177,6 +205,7 @@ private string CreateBarPlot(string title, string fileName, string yLabel, strin
177205

178206
plt.Legend.IsVisible = true;
179207
plt.Legend.Location = Alignment.UpperRight;
208+
plt.Legend.Font.Size = this.FontSize;
180209
var legend = data.Select(d => d.JobId)
181210
.Distinct()
182211
.Select((label, index) => new LegendItem()
@@ -192,8 +221,11 @@ private string CreateBarPlot(string title, string fileName, string yLabel, strin
192221
var ticks = data
193222
.Select((d, index) => new Tick(index, d.Target))
194223
.ToArray();
224+
225+
plt.Axes.Left.TickLabelStyle.FontSize = this.FontSize;
195226
plt.Axes.Bottom.TickGenerator = new ScottPlot.TickGenerators.NumericManual(ticks);
196227
plt.Axes.Bottom.MajorTickStyle.Length = 0;
228+
plt.Axes.Bottom.TickLabelStyle.FontSize = this.FontSize;
197229

198230
if (this.RotateLabels)
199231
{
@@ -209,7 +241,7 @@ private string CreateBarPlot(string title, string fileName, string yLabel, strin
209241
}
210242

211243
// ensure axis panels do not get smaller than the largest label
212-
plt.Axes.Bottom.MinimumSize = largestLabelWidth;
244+
plt.Axes.Bottom.MinimumSize = largestLabelWidth * 2;
213245
plt.Axes.Right.MinimumSize = largestLabelWidth;
214246
}
215247

@@ -232,6 +264,89 @@ private string CreateBarPlot(string title, string fileName, string yLabel, strin
232264
return Path.GetFullPath(fileName);
233265
}
234266

267+
private string CreateBoxPlot(string title, string fileName, string yLabel, string xLabel, IEnumerable<ChartStats> data, IReadOnlyList<Annotation> annotations)
268+
{
269+
Plot plt = new Plot();
270+
plt.Title(title, this.TitleFontSize);
271+
plt.YLabel(yLabel, this.FontSize);
272+
plt.XLabel(xLabel, this.FontSize);
273+
274+
var palette = new ScottPlot.Palettes.Category10();
275+
276+
var legendPalette = data.Select(d => d.JobId)
277+
.Distinct()
278+
.Select((jobId, index) => (jobId, index))
279+
.ToDictionary(t => t.jobId, t => palette.GetColor(t.index));
280+
281+
plt.Legend.IsVisible = true;
282+
plt.Legend.Location = Alignment.UpperRight;
283+
plt.Legend.Font.Size = this.FontSize;
284+
var legend = data.Select(d => d.JobId)
285+
.Distinct()
286+
.Select((label, index) => new LegendItem()
287+
{
288+
Label = label,
289+
FillColor = legendPalette[label]
290+
})
291+
.ToList();
292+
293+
plt.Legend.ManualItems.AddRange(legend);
294+
295+
var jobCount = plt.Legend.ManualItems.Count;
296+
var ticks = data
297+
.Select((d, index) => new Tick(index, d.Target))
298+
.ToArray();
299+
300+
plt.Axes.Left.TickLabelStyle.FontSize = this.FontSize;
301+
plt.Axes.Bottom.TickGenerator = new ScottPlot.TickGenerators.NumericManual(ticks);
302+
plt.Axes.Bottom.MajorTickStyle.Length = 0;
303+
plt.Axes.Bottom.TickLabelStyle.FontSize = this.FontSize;
304+
305+
if (this.RotateLabels)
306+
{
307+
plt.Axes.Bottom.TickLabelStyle.Rotation = 45;
308+
plt.Axes.Bottom.TickLabelStyle.Alignment = Alignment.MiddleLeft;
309+
310+
// determine the width of the largest tick label
311+
float largestLabelWidth = 0;
312+
foreach (Tick tick in ticks)
313+
{
314+
PixelSize size = plt.Axes.Bottom.TickLabelStyle.Measure(tick.Label);
315+
largestLabelWidth = Math.Max(largestLabelWidth, size.Width);
316+
}
317+
318+
// ensure axis panels do not get smaller than the largest label
319+
plt.Axes.Bottom.MinimumSize = largestLabelWidth * 2;
320+
plt.Axes.Right.MinimumSize = largestLabelWidth;
321+
}
322+
323+
int globalIndex = 0;
324+
foreach (var (targetGroup, targetGroupIndex) in data.GroupBy(s => s.Target).Select((targetGroup, index) => (targetGroup, index)))
325+
{
326+
var boxes = targetGroup.Select(job => (job.JobId, Stats: job.CalculateBoxPlotStatistics())).Select((j, jobIndex) => new Box()
327+
{
328+
Position = ticks[globalIndex++].Position,
329+
Fill = new FillStyle() { Color = legendPalette[j.JobId] },
330+
Stroke = new LineStyle() { Color = Colors.Black },
331+
BoxMin = j.Stats.Q1,
332+
BoxMax = j.Stats.Q3,
333+
WhiskerMin = j.Stats.Min,
334+
WhiskerMax = j.Stats.Max,
335+
BoxMiddle = j.Stats.Median
336+
})
337+
.ToList();
338+
plt.Add.Boxes(boxes);
339+
}
340+
341+
// Tell the plot to autoscale with a small padding below the boxes.
342+
plt.Axes.Margins(bottom: 0.05, right: .2);
343+
344+
plt.PlottableList.AddRange(annotations);
345+
346+
plt.SavePng(fileName, this.Width, this.Height);
347+
return Path.GetFullPath(fileName);
348+
}
349+
235350
/// <summary>
236351
/// Provides a list of annotations to put over the data area.
237352
/// </summary>
@@ -255,5 +370,69 @@ private IReadOnlyList<Annotation> GetAnnotations(string version)
255370

256371
return new[] { versionAnnotation };
257372
}
373+
374+
private class ChartStats
375+
{
376+
public ChartStats(string Target, string JobId, IReadOnlyList<double> Values)
377+
{
378+
this.Target = Target;
379+
this.JobId = JobId;
380+
this.Values = Values;
381+
}
382+
383+
public string Target { get; }
384+
385+
public string JobId { get; }
386+
387+
public IReadOnlyList<double> Values { get; }
388+
389+
public double Min => this.Values.DefaultIfEmpty(0d).Min();
390+
391+
public double Max => this.Values.DefaultIfEmpty(0d).Max();
392+
393+
public double Mean => this.Values.DefaultIfEmpty(0d).Average();
394+
395+
public double StdError => StandardError(this.Values);
396+
397+
398+
private static (int MidPoint, double Median) CalculateMedian(ReadOnlySpan<double> values)
399+
{
400+
int n = values.Length;
401+
var midPoint = n / 2;
402+
403+
// Check if count is even, if so use average of the two middle values,
404+
// otherwise take the middle value.
405+
var median = n % 2 == 0 ? (values[midPoint - 1] + values[midPoint]) / 2d : values[midPoint];
406+
return (midPoint, median);
407+
}
408+
409+
/// <summary>
410+
/// Calculate the mid points.
411+
/// </summary>
412+
/// <returns></returns>
413+
public (double Min, double Q1, double Median, double Q3, double Max, double[] Outliers) CalculateBoxPlotStatistics()
414+
{
415+
var values = this.Values.ToArray();
416+
Array.Sort(values);
417+
var s = values.AsSpan();
418+
var (midPoint, median) = CalculateMedian(s);
419+
420+
var (q1Index, q1) = midPoint > 0 ? CalculateMedian(s.Slice(0, midPoint)) : (midPoint, median);
421+
var (q3Index, q3) = midPoint + 1 < s.Length ? CalculateMedian(s.Slice(midPoint + 1)) : (midPoint, median);
422+
var iqr = q3 - q1;
423+
var lowerFence = q1 - 1.5d * iqr;
424+
var upperFence = q3 + 1.5d * iqr;
425+
var outliers = values.Where(v => v < lowerFence || v > upperFence).ToArray();
426+
var nonOutliers = values.Where(v => v >= lowerFence && v <= upperFence).ToArray();
427+
return (
428+
nonOutliers.FirstOrDefault(),
429+
q1,
430+
median,
431+
q3,
432+
nonOutliers.LastOrDefault(),
433+
outliers
434+
);
435+
}
436+
}
258437
}
259438
}

‎tests/BenchmarkDotNet.Exporters.Plotting.Tests/ScottPlotExporterTests.cs

Lines changed: 52 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
using BenchmarkDotNet.Attributes;
22
using BenchmarkDotNet.Configs;
33
using BenchmarkDotNet.Loggers;
4+
using BenchmarkDotNet.Reports;
5+
using BenchmarkDotNet.Running;
6+
using BenchmarkDotNet.Tests.Builders;
47
using BenchmarkDotNet.Tests.Mocks;
58
using System;
69
using System.Diagnostics.CodeAnalysis;
@@ -28,7 +31,11 @@ public void BarPlots(Type benchmarkType)
2831
var logger = new AccumulationLogger();
2932
logger.WriteLine("=== " + benchmarkType.Name + " ===");
3033

31-
var exporter = new ScottPlotExporter();
34+
var exporter = new ScottPlotExporter()
35+
{
36+
IncludeBarPlot = true,
37+
IncludeBoxPlot = false,
38+
};
3239
var summary = MockFactory.CreateSummary(benchmarkType);
3340
var filePaths = exporter.ExportToFiles(summary, logger).ToList();
3441
Assert.NotEmpty(filePaths);
@@ -39,6 +46,50 @@ public void BarPlots(Type benchmarkType)
3946
output.WriteLine(logger.GetLog());
4047
}
4148

49+
[Theory]
50+
[MemberData(nameof(GetGroupBenchmarkTypes))]
51+
public void BoxPlots(Type benchmarkType)
52+
{
53+
var logger = new AccumulationLogger();
54+
logger.WriteLine("=== " + benchmarkType.Name + " ===");
55+
56+
var exporter = new ScottPlotExporter()
57+
{
58+
IncludeBarPlot = false,
59+
IncludeBoxPlot = true,
60+
};
61+
var summary = MockFactory.CreateSummaryWithBiasedDistribution(benchmarkType, 1, 4, 10, 9);
62+
var filePaths = exporter.ExportToFiles(summary, logger).ToList();
63+
Assert.NotEmpty(filePaths);
64+
Assert.All(filePaths, f => File.Exists(f));
65+
66+
foreach (string filePath in filePaths)
67+
logger.WriteLine($"* {filePath}");
68+
output.WriteLine(logger.GetLog());
69+
}
70+
71+
[Theory]
72+
[MemberData(nameof(GetGroupBenchmarkTypes))]
73+
public void BoxPlotsWithOneMeasurement(Type benchmarkType)
74+
{
75+
var logger = new AccumulationLogger();
76+
logger.WriteLine("=== " + benchmarkType.Name + " ===");
77+
78+
var exporter = new ScottPlotExporter()
79+
{
80+
IncludeBarPlot = false,
81+
IncludeBoxPlot = true,
82+
};
83+
var summary = MockFactory.CreateSummaryWithBiasedDistribution(benchmarkType, 1, 4, 10, 1);
84+
var filePaths = exporter.ExportToFiles(summary, logger).ToList();
85+
Assert.NotEmpty(filePaths);
86+
Assert.All(filePaths, f => File.Exists(f));
87+
88+
foreach (string filePath in filePaths)
89+
logger.WriteLine($"* {filePath}");
90+
output.WriteLine(logger.GetLog());
91+
}
92+
4293
[SuppressMessage("ReSharper", "InconsistentNaming")]
4394
public static class BaselinesBenchmarks
4495
{

0 commit comments

Comments
 (0)