Modify benchmarks to run in perf test harness

- Add result validation - Add [Benchmark] attributes and appropriate iteration counts - Minor edits here and there to target .NET Standard 1.4 - Exception: pi-digits rewritten to use managed BitInteger instead of p/invoke out to GMP.
dotnet-bot · Sep 15, 2017 · e055116 · e055116
1 parent d0099ff
commit e055116
Show file tree

Hide file tree

Showing 42 changed files with 6,015 additions and 221 deletions.
diff --git a/.gitattributes b/.gitattributes
@@ -69,6 +69,8 @@ src/pal/tests/palsuite/paltestlist_to_be_reviewed.txt text eol=lf
 
 tests/src/JIT/Performance/CodeQuality/BenchmarksGame/regexdna/regexdna-input25.txt text eol=lf
 tests/src/JIT/Performance/CodeQuality/BenchmarksGame/regexdna/regexdna-input25000.txt text eol=lf
+tests/src/JIT/Performance/CodeQuality/BenchmarksGame/regex-redux/regexdna-input25.txt text eol=lf
+tests/src/JIT/Performance/CodeQuality/BenchmarksGame/regex-redux/regexdna-input25000.txt text eol=lf
 tests/src/JIT/Performance/CodeQuality/BenchmarksGame/reverse-complement/revcomp-input25.txt text eol=lf
 tests/src/JIT/Performance/CodeQuality/BenchmarksGame/reverse-complement/revcomp-input25000.txt text eol=lf
 tests/src/JIT/Performance/CodeQuality/BenchmarksGame/k-nucleotide/knucleotide-input.txt text eol=lf

diff --git a/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/binarytrees/binarytrees-best.cs b/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/binarytrees/binarytrees-best.cs
@@ -18,16 +18,36 @@ minor improvements by Alex Yakunin
 using System;
 using System.Runtime.CompilerServices;
 using System.Threading.Tasks;
+using Microsoft.Xunit.Performance;
+
+[assembly: OptimizeForBenchmarks]
+[assembly: MeasureGCCounts]
 
 namespace BenchmarksGame
 {
-    public sealed class BinaryTrees
+    public sealed class BinaryTrees_5
     {
         public const int MinDepth = 4;
 
-        public static void Main(string[] args)
+        public static int Main(string[] args)
         {
             var n = args.Length == 0 ? 0 : int.Parse(args[0]);
+
+            int check = Bench(n, true);
+            int expected = 4398;
+
+            // Return 100 on success, anything else on failure.
+            return check - expected + 100;
+        }
+
+        [Benchmark(InnerIterationCount = 7)]
+        public static void RunBench()
+        {
+            Benchmark.Iterate(() => Bench(16, false));
+        }
+
+        static int Bench(int n, bool verbose)
+        {
             var maxDepth = n < (MinDepth + 2) ? MinDepth + 2 : n;
             var stretchDepth = maxDepth + 1;
 
@@ -44,8 +64,8 @@ public static void Main(string[] args)
                     var count = 0;
                     if (depthCopy >= 17)
                     {
-                    // Parallelized computation for relatively large tasks
-                    var miniTasks = new Task<int>[iterationCount];
+                        // Parallelized computation for relatively large tasks
+                        var miniTasks = new Task<int>[iterationCount];
                         for (var i = 0; i < iterationCount; i++)
                             miniTasks[i] = Task.Run(() => TreeNode.CreateTree(depthCopy).CountNodes());
                         Task.WaitAll(miniTasks);
@@ -54,21 +74,35 @@ public static void Main(string[] args)
                     }
                     else
                     {
-                    // Sequential computation for smaller tasks
-                    for (var i = 0; i < iterationCount; i++)
+                        // Sequential computation for smaller tasks
+                        for (var i = 0; i < iterationCount; i++)
                             count += TreeNode.CreateTree(depthCopy).CountNodes();
                     }
                     return $"{iterationCount}\t trees of depth {depthCopy}\t check: {count}";
                 });
             }
             Task.WaitAll(tasks);
 
-            Console.WriteLine("stretch tree of depth {0}\t check: {1}",
-                stretchDepth, stretchDepthTask.Result);
-            foreach (var task in tasks)
-                Console.WriteLine(task.Result);
-            Console.WriteLine("long lived tree of depth {0}\t check: {1}",
-                maxDepth, maxDepthTask.Result);
+            if (verbose)
+            {
+                int count = 0;
+                Action<string> printAndSum = (string s) =>
+                {
+                    Console.WriteLine(s);
+                    count += int.Parse(s.Substring(s.LastIndexOf(':') + 1).TrimStart());
+                };
+
+                printAndSum(String.Format("stretch tree of depth {0}\t check: {1}",
+                    stretchDepth, stretchDepthTask.Result));
+                foreach (var task in tasks)
+                    printAndSum(task.Result);
+                printAndSum(String.Format("long lived tree of depth {0}\t check: {1}",
+                    maxDepth, maxDepthTask.Result));
+
+                return count;
+            }
+
+            return 0;
         }
     }
 

diff --git a/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/binarytrees/binarytrees-best.csproj b/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/binarytrees/binarytrees-best.csproj
@@ -0,0 +1,41 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.props))\dir.props" />
+  <PropertyGroup>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+    <SchemaVersion>2.0</SchemaVersion>
+    <ProjectGuid>{95DFC527-4DC1-495E-97D7-E94EE1F7140D}</ProjectGuid>
+    <OutputType>Exe</OutputType>
+    <ProjectTypeGuids>{786C830F-07A1-408B-BD7F-6EE04809D6DB};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}</ProjectTypeGuids>
+    <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\</SolutionDir>
+    <NuGetTargetMoniker>.NETStandard,Version=v1.4</NuGetTargetMoniker>
+    <NuGetTargetMonikerShort>netstandard1.4</NuGetTargetMonikerShort>
+    <GCStressIncompatible>true</GCStressIncompatible>
+  </PropertyGroup>
+  <!-- Default configurations to help VS understand the configurations -->
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
+  </PropertyGroup>
+  <PropertyGroup>
+    <DebugType>pdbonly</DebugType>
+    <Optimize>true</Optimize>
+  </PropertyGroup>
+  <ItemGroup>
+    <CodeAnalysisDependentAssemblyPaths Condition=" '$(VS100COMNTOOLS)' != '' " Include="$(VS100COMNTOOLS)..\IDE\PrivateAssemblies">
+      <Visible>False</Visible>
+    </CodeAnalysisDependentAssemblyPaths>
+  </ItemGroup>
+  <ItemGroup>
+    <Service Include="{82A7F48D-3B50-4B1E-B82E-3ADA8210C358}" />
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Include="$(MSBuildProjectName).cs" />
+  </ItemGroup>
+  <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.targets))\dir.targets" />
+  <PropertyGroup Condition=" '$(MsBuildProjectDirOverride)' != '' "></PropertyGroup>
+  <PropertyGroup>
+    <ProjectAssetsFile>$(JitPackagesConfigFileDirectory)benchmark\obj\project.assets.json</ProjectAssetsFile>
+  </PropertyGroup>
+</Project>
diff --git a/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/binarytrees/binarytrees-serial.cs b/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/binarytrees/binarytrees-serial.cs
@@ -14,23 +14,43 @@ contributed by Marek Safar
 */
 
 using System;
+using Microsoft.Xunit.Performance;
+
+[assembly: OptimizeForBenchmarks]
+[assembly: MeasureGCCounts]
 
 namespace BenchmarksGame
 {
-    class BinaryTrees
+    public class BinaryTrees_2
     {
         const int minDepth = 4;
 
-        public static void Main(String[] args)
+        public static int Main(String[] args)
         {
             int n = 0;
             if (args.Length > 0) n = Int32.Parse(args[0]);
 
+            int check = Bench(n, true);
+            int expected = 4398;
+
+            // Return 100 on success, anything else on failure.
+            return check - expected + 100;
+        }
+
+        [Benchmark(InnerIterationCount = 7)]
+        public static void RunBench()
+        {
+            Benchmark.Iterate(() => Bench(16, false));
+        }
+
+        static int Bench(int n, bool verbose)
+        {
             int maxDepth = Math.Max(minDepth + 2, n);
             int stretchDepth = maxDepth + 1;
 
             int check = (TreeNode.bottomUpTree(stretchDepth)).itemCheck();
-            Console.WriteLine("stretch tree of depth {0}\t check: {1}", stretchDepth, check);
+            int checkSum = check;
+            if (verbose) Console.WriteLine("stretch tree of depth {0}\t check: {1}", stretchDepth, check);
 
             TreeNode longLivedTree = TreeNode.bottomUpTree(maxDepth);
 
@@ -43,13 +63,19 @@ public static void Main(String[] args)
                 {
                     check += (TreeNode.bottomUpTree(depth)).itemCheck();
                 }
+                checkSum += check;
 
-                Console.WriteLine("{0}\t trees of depth {1}\t check: {2}",
-                   iterations, depth, check);
+                if (verbose)
+                    Console.WriteLine("{0}\t trees of depth {1}\t check: {2}", iterations, depth, check);
             }
 
-            Console.WriteLine("long lived tree of depth {0}\t check: {1}",
-               maxDepth, longLivedTree.itemCheck());
+            check = longLivedTree.itemCheck();
+            checkSum += check;
+
+            if (verbose)
+                Console.WriteLine("long lived tree of depth {0}\t check: {1}", maxDepth, check);
+
+            return checkSum;
         }
 
 

diff --git a/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/binarytrees/binarytrees-serial.csproj b/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/binarytrees/binarytrees-serial.csproj
@@ -0,0 +1,41 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.props))\dir.props" />
+  <PropertyGroup>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+    <SchemaVersion>2.0</SchemaVersion>
+    <ProjectGuid>{95DFC527-4DC1-495E-97D7-E94EE1F7140D}</ProjectGuid>
+    <OutputType>Exe</OutputType>
+    <ProjectTypeGuids>{786C830F-07A1-408B-BD7F-6EE04809D6DB};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}</ProjectTypeGuids>
+    <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\</SolutionDir>
+    <NuGetTargetMoniker>.NETStandard,Version=v1.4</NuGetTargetMoniker>
+    <NuGetTargetMonikerShort>netstandard1.4</NuGetTargetMonikerShort>
+    <GCStressIncompatible>true</GCStressIncompatible>
+  </PropertyGroup>
+  <!-- Default configurations to help VS understand the configurations -->
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
+  </PropertyGroup>
+  <PropertyGroup>
+    <DebugType>pdbonly</DebugType>
+    <Optimize>true</Optimize>
+  </PropertyGroup>
+  <ItemGroup>
+    <CodeAnalysisDependentAssemblyPaths Condition=" '$(VS100COMNTOOLS)' != '' " Include="$(VS100COMNTOOLS)..\IDE\PrivateAssemblies">
+      <Visible>False</Visible>
+    </CodeAnalysisDependentAssemblyPaths>
+  </ItemGroup>
+  <ItemGroup>
+    <Service Include="{82A7F48D-3B50-4B1E-B82E-3ADA8210C358}" />
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Include="$(MSBuildProjectName).cs" />
+  </ItemGroup>
+  <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.targets))\dir.targets" />
+  <PropertyGroup Condition=" '$(MsBuildProjectDirOverride)' != '' "></PropertyGroup>
+  <PropertyGroup>
+    <ProjectAssetsFile>$(JitPackagesConfigFileDirectory)benchmark\obj\project.assets.json</ProjectAssetsFile>
+  </PropertyGroup>
+</Project>
diff --git a/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/fannkuch-redux/fannkuch-redux-best.cs b/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/fannkuch-redux/fannkuch-redux-best.cs
@@ -17,10 +17,14 @@ parallel and small optimisations by Anthony Lloyd
 using System;
 using System.Threading;
 using System.Runtime.CompilerServices;
+using Microsoft.Xunit.Performance;
+using Xunit;
+
+[assembly: OptimizeForBenchmarks]
 
 namespace BenchmarksGame
 {
-    public static class FannkuchRedux
+    public static class FannkuchRedux_5
     {
         static int[] fact, chkSums, maxFlips;
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
@@ -98,9 +102,30 @@ static void run(int n, int taskId, int taskSize)
             maxFlips[taskId] = maxflips;
         }
 
-        public static void Main(string[] args)
+        public static int Main(string[] args)
         {
             int n = args.Length > 0 ? int.Parse(args[0]) : 7;
+            int sum = Bench(n, true);
+
+            int expected = 228;
+
+            // Return 100 on success, anything else on failure.
+            return sum - expected + 100;
+        }
+
+        [Benchmark(InnerIterationCount = 20)]
+        [InlineData(10, 73196)]
+        public static void RunBench(int n, int expectedSum)
+        {
+            Benchmark.Iterate(() =>
+            {
+                int sum = Bench(n, false);
+                Assert.Equal(expectedSum, sum);
+            });
+        }
+
+        static int Bench(int n, bool verbose)
+        {
             fact = new int[n + 1];
             fact[0] = 1;
             var factn = 1;
@@ -124,7 +149,9 @@ public static void Main(string[] args)
                 chksum += chkSums[i];
                 if (maxFlips[i] > maxflips) maxflips = maxFlips[i];
             }
-            Console.Out.WriteLineAsync(chksum + "\nPfannkuchen(" + n + ") = " + maxflips);
+            if (verbose) Console.Out.WriteLineAsync(chksum + "\nPfannkuchen(" + n + ") = " + maxflips);
+
+            return chksum;
         }
     }
 }
diff --git a/.../src/JIT/Performance/CodeQuality/BenchmarksGame/fannkuch-redux/fannkuch-redux-best.csproj b/.../src/JIT/Performance/CodeQuality/BenchmarksGame/fannkuch-redux/fannkuch-redux-best.csproj
@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.props))\dir.props" />
+  <PropertyGroup>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+    <SchemaVersion>2.0</SchemaVersion>
+    <ProjectGuid>{F49D82D3-9D13-47B5-83F8-52B1FE3FF452}</ProjectGuid>
+    <OutputType>Exe</OutputType>
+    <ProjectTypeGuids>{786C830F-07A1-408B-BD7F-6EE04809D6DB};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}</ProjectTypeGuids>
+    <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\</SolutionDir>
+    <NuGetTargetMoniker>.NETStandard,Version=v1.4</NuGetTargetMoniker>
+    <NuGetTargetMonikerShort>netstandard1.4</NuGetTargetMonikerShort>
+  </PropertyGroup>
+  <!-- Default configurations to help VS understand the configurations -->
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' "></PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' " />
+  <PropertyGroup>
+    <DebugType>pdbonly</DebugType>
+    <Optimize>true</Optimize>
+  </PropertyGroup>
+  <ItemGroup>
+    <CodeAnalysisDependentAssemblyPaths Condition=" '$(VS100COMNTOOLS)' != '' " Include="$(VS100COMNTOOLS)..\IDE\PrivateAssemblies">
+      <Visible>False</Visible>
+    </CodeAnalysisDependentAssemblyPaths>
+  </ItemGroup>
+  <ItemGroup>
+    <Service Include="{82A7F48D-3B50-4B1E-B82E-3ADA8210C358}" />
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Include="$(MSBuildProjectName).cs" />
+  </ItemGroup>
+  <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.targets))\dir.targets" />
+  <PropertyGroup Condition=" '$(MsBuildProjectDirOverride)' != '' "></PropertyGroup>
+  <PropertyGroup>
+    <ProjectAssetsFile>$(JitPackagesConfigFileDirectory)benchmark\obj\project.assets.json</ProjectAssetsFile>
+  </PropertyGroup>
+</Project>