Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

specjbb parser parses N/A metrics gracefully #346

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

specjbb parser parses N/A metrics gracefully #346

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
277 changes: 277 additions & 0 deletions src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/SPECjbb/specjbbNanOutput1.txt
View file
Open in desktop

Large diffs are not rendered by default.

24 changes: 12 additions & 12 deletions src/VirtualClient/VirtualClient.Actions.UnitTests/FIO/FioMetricsParserTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -245,8 +245,8 @@ public void FioResultsParserReadsTheExpectedMeasurementValuesFromTheResults_Read
Assert.IsNotNull(metrics);
Assert.IsNotEmpty(metrics);

IEnumerable<Tuple<string, double>> expectedValues = FioMetricsParserTests.ExpectedReadMetrics.Select(entry => new Tuple<string, double>(entry.Key, entry.Value));
IEnumerable<Tuple<string, double>> actualValues = metrics.Select(m => new Tuple<string, double>(m.Name, m.Value));
IEnumerable<Tuple<string, double?>> expectedValues = FioMetricsParserTests.ExpectedReadMetrics.Select(entry => new Tuple<string, double?>(entry.Key, entry.Value));
IEnumerable<Tuple<string, double?>> actualValues = metrics.Select(m => new Tuple<string, double?>(m.Name, m.Value));

CollectionAssert.AreEquivalent(expectedValues, actualValues);
}
Expand All @@ -262,8 +262,8 @@ public void FioResultsParserReadsTheExpectedMeasurementValuesFromTheResults_Writ
Assert.IsNotNull(metrics);
Assert.IsNotEmpty(metrics);

IEnumerable<Tuple<string, double>> expectedValues = FioMetricsParserTests.ExpectedWriteMetrics.Select(entry => new Tuple<string, double>(entry.Key, entry.Value));
IEnumerable<Tuple<string, double>> actualValues = metrics.Select(m => new Tuple<string, double>(m.Name, m.Value));
IEnumerable<Tuple<string, double?>> expectedValues = FioMetricsParserTests.ExpectedWriteMetrics.Select(entry => new Tuple<string, double?>(entry.Key, entry.Value));
IEnumerable<Tuple<string, double?>> actualValues = metrics.Select(m => new Tuple<string, double?>(m.Name, m.Value));

CollectionAssert.AreEquivalent(expectedValues, actualValues);
}
Expand All @@ -282,11 +282,11 @@ public void FioResultsParserAppliesTheExpectedDefaultConversionFactorToMeasureme

// All latency metrics are emitted in milliseconds form (vs. nanoseconds).
double expectedConversionFactor = 0.000001;
IEnumerable<Tuple<string, double>> expectedValues = FioMetricsParserTests.ExpectedReadMetrics.Where(entry => entry.Key.Contains("latency"))
.Select(entry => new Tuple<string, double>(entry.Key, entry.Value * expectedConversionFactor));
IEnumerable<Tuple<string, double?>> expectedValues = FioMetricsParserTests.ExpectedReadMetrics.Where(entry => entry.Key.Contains("latency"))
.Select(entry => new Tuple<string, double?>(entry.Key, entry.Value * expectedConversionFactor));

IEnumerable<Tuple<string, double>> actualValues = metrics.Where(m => m.Name.Contains("latency"))
.Select(m => new Tuple<string, double>(m.Name, m.Value));
IEnumerable<Tuple<string, double?>> actualValues = metrics.Where(m => m.Name.Contains("latency"))
.Select(m => new Tuple<string, double?>(m.Name, m.Value));

Assert.IsTrue(expectedValues.Count() == actualValues.Count());
CollectionAssert.AreEquivalent(expectedValues, actualValues);
Expand All @@ -306,11 +306,11 @@ public void FioResultsParserAppliesTheExpectedDefaultConversionFactorToMeasureme

// All latency metrics are emitted in milliseconds form (vs. nanoseconds).
double expectedConversionFactor = 0.000001;
IEnumerable<Tuple<string, double>> expectedValues = FioMetricsParserTests.ExpectedWriteMetrics.Where(entry => entry.Key.Contains("latency"))
.Select(entry => new Tuple<string, double>(entry.Key, entry.Value * expectedConversionFactor));
IEnumerable<Tuple<string, double?>> expectedValues = FioMetricsParserTests.ExpectedWriteMetrics.Where(entry => entry.Key.Contains("latency"))
.Select(entry => new Tuple<string, double?>(entry.Key, entry.Value * expectedConversionFactor));

IEnumerable<Tuple<string, double>> actualValues = metrics.Where(m => m.Name.Contains("latency"))
.Select(m => new Tuple<string, double>(m.Name, m.Value));
IEnumerable<Tuple<string, double?>> actualValues = metrics.Where(m => m.Name.Contains("latency"))
.Select(m => new Tuple<string, double?>(m.Name, m.Value));

Assert.IsTrue(expectedValues.Count() == actualValues.Count());
CollectionAssert.AreEquivalent(expectedValues, actualValues);
Expand Down
17 changes: 17 additions & 0 deletions src/VirtualClient/VirtualClient.Actions.UnitTests/SPEC/SpecJbbMetricsParserTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,5 +40,22 @@ public void SpecJbbParserVerifyMetricsFpRate()
MetricAssert.Exists(metrics, "max-jOPS", 4188, "jOPS");
MetricAssert.Exists(metrics, "critical-jOPS", 1666, "jOPS");
}


[Test]
public void SpecJbbParserVerifyNanMetricsFpRate()
{
string workingDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
string outputPath = Path.Combine(workingDirectory, "Examples", "SPECjbb", "specjbbNanOutput1.txt");
this.rawText = File.ReadAllText(outputPath);
this.testParser = new SpecJbbMetricsParser(this.rawText);
IList<Metric> metrics = this.testParser.Parse();

Assert.AreEqual(4, metrics.Count);
MetricAssert.Exists(metrics, "hbIR (max attempted)", 304872, "jOPS");
MetricAssert.Exists(metrics, "hbIR (settled)", null, "jOPS");
MetricAssert.Exists(metrics, "max-jOPS", 234751, "jOPS");
MetricAssert.Exists(metrics, "critical-jOPS", null , "jOPS");
}
}
}
12 changes: 6 additions & 6 deletions src/VirtualClient/VirtualClient.Actions/3DMark/ThreeDMarkExecutor.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -344,9 +344,9 @@ private Task ExecuteWorkloadAsync(EventContext telemetryContext, CancellationTok
private IList<Metric> CalculateTimeSpyAggregates(IList<Metric> metrics)
{
IList<Metric> aggregates = new List<Metric>();
double tsgt1 = 0;
double tsgt2 = 0;
double tsct = 0;
double? tsgt1 = 0;
double? tsgt2 = 0;
double? tsct = 0;
foreach (Metric metric in metrics)
{
if (metric.Name == "timespy.graphics.1")
Expand All @@ -366,9 +366,9 @@ private IList<Metric> CalculateTimeSpyAggregates(IList<Metric> metrics)
// Weighted Harmonic Mean of Individual Scores
if (tsgt1 != 0 && tsgt2 != 0 && tsct != 0)
{
double graphicsScore = 165 * (2 / ((1 / tsgt1) + (1 / tsgt2)));
double cpuScore = 298 * tsct;
double aggScore = 1 / ((0.85 / graphicsScore) + (0.15 / cpuScore));
double? graphicsScore = 165 * (2 / ((1 / tsgt1) + (1 / tsgt2)));
double? cpuScore = 298 * tsct;
double? aggScore = 1 / ((0.85 / graphicsScore) + (0.15 / cpuScore));
aggregates.Add(new Metric("timespy.graphics.agg", graphicsScore, "score", MetricRelativity.HigherIsBetter));
aggregates.Add(new Metric("timespy.cpu.agg", cpuScore, "score", MetricRelativity.HigherIsBetter));
aggregates.Add(new Metric("timespy.finalscore", aggScore, "score", MetricRelativity.HigherIsBetter));
Expand Down
3 changes: 2 additions & 1 deletion src/VirtualClient/VirtualClient.Actions/SPECjbb/SpecJbbMetricsParser.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,8 @@ public override IList<Metric> Parse()
string[] tokens = metric.Split("=");
string name = tokens[0];
string value = tokens[1];
this.Metrics.Add(new Metric(name.Trim(), Convert.ToDouble(value), SpecJbbMetricsParser.OperationPerSecond, MetricRelativity.HigherIsBetter));

this.Metrics.Add(new Metric(name.Trim(), (value.Trim().Equals("N/A", StringComparison.OrdinalIgnoreCase)) ? null : Convert.ToDouble(value), SpecJbbMetricsParser.OperationPerSecond, MetricRelativity.HigherIsBetter));
}

return this.Metrics;
Expand Down
12 changes: 6 additions & 6 deletions src/VirtualClient/VirtualClient.Contracts/Metric.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ public class Metric : IEquatable<Metric>
/// <summary>
/// Creates a metric
/// </summary>
public Metric(string name, double value)
public Metric(string name, double? value)
{
this.Name = name;
this.Value = value;
Expand All @@ -30,7 +30,7 @@ public Metric(string name, double value)
/// <summary>
/// Creates a metric
/// </summary>
public Metric(string name, double value, IEnumerable<string> tags = null, string description = null, IDictionary<string, IConvertible> metadata = null)
public Metric(string name, double? value, IEnumerable<string> tags = null, string description = null, IDictionary<string, IConvertible> metadata = null)
: this(name, value)
{
this.Description = description;
Expand All @@ -51,7 +51,7 @@ public Metric(string name, double value, IEnumerable<string> tags = null, string
/// <summary>
/// Creates a metric
/// </summary>
public Metric(string name, double value, MetricRelativity relativity, IEnumerable<string> tags = null, string description = null, IDictionary<string, IConvertible> metadata = null)
public Metric(string name, double? value, MetricRelativity relativity, IEnumerable<string> tags = null, string description = null, IDictionary<string, IConvertible> metadata = null)
: this(name, value, tags: tags, description: description, metadata: metadata)
{
this.Relativity = relativity;
Expand All @@ -60,7 +60,7 @@ public Metric(string name, double value, MetricRelativity relativity, IEnumerabl
/// <summary>
/// Creates a metric
/// </summary>
public Metric(string name, double value, string unit, IEnumerable<string> tags = null, string description = null, IDictionary<string, IConvertible> metadata = null)
public Metric(string name, double? value, string unit, IEnumerable<string> tags = null, string description = null, IDictionary<string, IConvertible> metadata = null)
: this(name, value, tags: tags, description: description, metadata: metadata)
{
this.Unit = unit;
Expand All @@ -69,7 +69,7 @@ public Metric(string name, double value, string unit, IEnumerable<string> tags =
/// <summary>
/// Creates a metric
/// </summary>
public Metric(string name, double value, string unit, MetricRelativity relativity, IEnumerable<string> tags = null, string description = null, IDictionary<string, IConvertible> metadata = null)
public Metric(string name, double? value, string unit, MetricRelativity relativity, IEnumerable<string> tags = null, string description = null, IDictionary<string, IConvertible> metadata = null)
: this(name, value, unit, tags: tags, description: description, metadata: metadata)
{
this.Relativity = relativity;
Expand Down Expand Up @@ -99,7 +99,7 @@ public Metric(string name, double value, string unit, MetricRelativity relativit
/// <summary>
/// Result of test
/// </summary>
public double Value { get; }
public double? Value { get; }

/// <summary>
/// Unit of result
Expand Down
6 changes: 3 additions & 3 deletions src/VirtualClient/VirtualClient.Contracts/MetricAggregate.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ namespace VirtualClient.Contracts
/// Provides features for capturing Windows performance counter values over a period
/// of time.
/// </summary>
public class MetricAggregate : ConcurrentBag<double>
public class MetricAggregate : ConcurrentBag<double?>
{
/// <summary>
/// Initializes a new instance of the <see cref="MetricAggregate"/> class.
Expand Down Expand Up @@ -115,11 +115,11 @@ public Metric ToMetric(MetricAggregateType aggregateType)
return Metric.None;
}

double value = 0;
double? value = 0;
switch (aggregateType)
{
case MetricAggregateType.Average:
double sum = this.Sum();
double? sum = this.Sum();
value = sum / this.Count;
break;

Expand Down
2 changes: 1 addition & 1 deletion src/VirtualClient/VirtualClient.Contracts/VirtualClientLoggingExtensions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -744,7 +744,7 @@ public static void LogMetrics(
DateTime scenarioStartTime,
DateTime scenarioEndTime,
string metricName,
double metricValue,
double? metricValue,
nmalkapuram marked this conversation as resolved.
Show resolved Hide resolved
string metricUnits,
string metricCategorization,
string scenarioArguments,
Expand Down
2 changes: 1 addition & 1 deletion src/VirtualClient/VirtualClient.TestExtensions/MetricAssert.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ public static void Exists(
/// <summary>
/// Asserts the metric exists in the given list of results
/// </summary>
public static void Exists(IList<Metric> results, string expectedMetric, double expectedMetricValue, string expectedMetricUnit = null, List<string> expectedTags = null)
public static void Exists(IList<Metric> results, string expectedMetric, double? expectedMetricValue, string expectedMetricUnit = null, List<string> expectedTags = null)
{
List<Metric> matchingMetrics = results.Where(m => m.Name == expectedMetric).ToList();

Expand Down
Loading