fix: have the seed database called from outside the benchmark setup to avoid multiple unnecessary seeding

SockworkOrange · SockworkOrange · commit 33d8149bc64e · 2025-12-14T19:20:01.000+01:00
diff --git a/benchmark/BenchmarkRunner/Benchmarks/BaseReadBenchmark.cs b/benchmark/BenchmarkRunner/Benchmarks/BaseReadBenchmark.cs
@@ -23,7 +23,7 @@ private static int CalculateMaxConcurrency(int totalTasks, int maxConcurrency)
             maxConcurrency, totalTasks, Environment.ProcessorCount
         }.Min(x => x);
     }
-    public static async Task<List<T>> ExecuteConcurrentlyAsync<T>(
+    protected static async Task<List<T>> ExecuteConcurrentlyAsync<T>(
         int totalTasks,
         int maxConcurrency,
         Func<int, Task<List<T>>> taskFactory)
@@ -41,7 +41,9 @@ public static async Task<List<T>> ExecuteConcurrentlyAsync<T>(
         return [.. results.SelectMany(r => r)];
     }
 
-    private static async Task<List<T>> ExecuteWithThrottleAsync<T>(SemaphoreSlim semaphore, Func<Task<List<T>>> taskFactory)
+    private static async Task<List<T>> ExecuteWithThrottleAsync<T>(
+        SemaphoreSlim semaphore,
+        Func<Task<List<T>>> taskFactory)
     {
         await semaphore.WaitAsync();
         try
diff --git a/benchmark/BenchmarkRunner/Benchmarks/MysqlReadBenchmark.cs b/benchmark/BenchmarkRunner/Benchmarks/MysqlReadBenchmark.cs
@@ -20,23 +20,6 @@ public class MysqlReadBenchmark : BaseReadBenchmark
     [Params(25, 100, 200)]
     public int ConcurrentQueries { get; set; }
 
-    [GlobalSetup]
-    public async Task GlobalSetup()
-    {
-        await Helpers.InitializeOnceAsync(async () =>
-        {
-            await MysqlDatabaseHelper.CleanupDatabaseAsync(_connectionString);
-            var seeder = new MysqlDatabaseSeeder(_connectionString);
-            await seeder.SeedAsync(
-                 customerCount: CustomerCount, // selectivity: 1/500 of the table returned
-                 productsPerCategory: 150,
-                 ordersPerCustomer: 1000,
-                 itemsPerOrder: 20
-             // 20 * 1000 = 20,000 possible rows returned
-             );
-        });
-    }
-
     [BenchmarkCategory("Read")]
     [Benchmark(Baseline = true, Description = "SQLC - GetCustomerOrders")]
     public override async Task Sqlc_GetCustomerOrders()
@@ -82,4 +65,19 @@ await ExecuteConcurrentlyAsync(QueriesToRun, ConcurrentQueries, async _ =>
             ));
         });
     }
+
+    public static Func<Task> GetSeedMethod()
+    {
+        return async () =>
+        {
+            await MysqlDatabaseHelper.CleanupDatabaseAsync(_connectionString);
+            var seeder = new MysqlDatabaseSeeder(_connectionString);
+            await seeder.SeedAsync(
+                customerCount: 500, // selectivity: 1/500 of the table returned
+                productsPerCategory: 150,
+                ordersPerCustomer: 1000,
+                itemsPerOrder: 20 // 20 * 1000 = 20,000 possible rows returned
+            );
+        };
+    }
 }
diff --git a/benchmark/BenchmarkRunner/Benchmarks/MysqlWriteBenchmark.cs b/benchmark/BenchmarkRunner/Benchmarks/MysqlWriteBenchmark.cs
@@ -30,27 +30,15 @@ public class MysqlWriteBenchmark : BaseWriteBenchmark
     [GlobalSetup]
     public async Task GlobalSetup()
     {
-        await Helpers.InitializeOnceAsync(async () =>
-        {
-            await MysqlDatabaseHelper.CleanupDatabaseAsync(_connectionString);
-            var seeder = new MysqlDatabaseSeeder(_connectionString);
-            await seeder.SeedAsync(
-                customerCount: 10,
-                productsPerCategory: 15,
-                ordersPerCustomer: 300,
-                itemsPerOrder: 0
-            );
+        var orderIds = await _sqlcImpl.GetOrderIdsAsync(new QuerySql.GetOrderIdsArgs(Limit: 1000));
+        var productIds = await _sqlcImpl.GetProductIdsAsync(new QuerySql.GetProductIdsArgs(Limit: 1000));
 
-            var orderIds = await _sqlcImpl.GetOrderIdsAsync(new QuerySql.GetOrderIdsArgs(Limit: 1000));
-            var productIds = await _sqlcImpl.GetProductIdsAsync(new QuerySql.GetProductIdsArgs(Limit: 1000));
-
-            _testOrderItems = [.. Enumerable.Range(0, TotalRecords).Select(i => new QuerySql.AddOrderItemsArgs(
-                OrderId: orderIds[i % orderIds.Count].OrderId,
-                ProductId: productIds[i % productIds.Count].ProductId,
-                Quantity: Random.Shared.Next(1, 10),
-                UnitPrice: (decimal)(Random.Shared.NextDouble() * 100 + 5)
-            ))];
-        });
+        _testOrderItems = [.. Enumerable.Range(0, TotalRecords).Select(i => new QuerySql.AddOrderItemsArgs(
+            OrderId: orderIds[i % orderIds.Count].OrderId,
+            ProductId: productIds[i % productIds.Count].ProductId,
+            Quantity: Random.Shared.Next(1, 10),
+            UnitPrice: (decimal)(Random.Shared.NextDouble() * 100 + 5)
+        ))];
     }
 
     [IterationSetup]
@@ -76,4 +64,19 @@ public override async Task EFCore_AddOrderItems()
         )).ToList();
         await Helpers.InsertInBatchesAsync(args, BatchSize, _efCoreImpl.AddOrderItems);
     }
+
+    public static Func<Task> GetSeedMethod()
+    {
+        return async () =>
+        {
+            await MysqlDatabaseHelper.CleanupDatabaseAsync(_connectionString);
+            var seeder = new MysqlDatabaseSeeder(_connectionString);
+            await seeder.SeedAsync(
+                customerCount: 10,
+                productsPerCategory: 15,
+                ordersPerCustomer: 300,
+                itemsPerOrder: 0
+            );
+        };
+    }
 }
diff --git a/benchmark/BenchmarkRunner/Benchmarks/PostgresqlReadBenchmark.cs b/benchmark/BenchmarkRunner/Benchmarks/PostgresqlReadBenchmark.cs
@@ -20,23 +20,6 @@ public class PostgresqlReadBenchmark : BaseReadBenchmark
     [Params(25, 50, 100)]
     public int ConcurrentQueries { get; set; }
 
-    [GlobalSetup]
-    public async Task GlobalSetup()
-    {
-        await Helpers.InitializeOnceAsync(async () =>
-        {
-            await PostgresqlDatabaseHelper.CleanupDatabaseAsync(_connectionString);
-            var seeder = new PostgresqlDatabaseSeeder(_connectionString);
-            await seeder.SeedAsync(
-                customerCount: CustomerCount,
-                productsPerCategory: 150, // with customer_id filter, this is 1/500 of the table returned
-                ordersPerCustomer: 1000,
-                itemsPerOrder: 20
-            // 20 * 1000 = 20,000 possible rows returned
-            );
-        });
-    }
-
     [BenchmarkCategory("Read")]
     [Benchmark(Baseline = true, Description = "SQLC - GetCustomerOrders")]
     public override async Task Sqlc_GetCustomerOrders()
@@ -82,4 +65,19 @@ await ExecuteConcurrentlyAsync(QueriesToRun, ConcurrentQueries, async _ =>
             ));
         });
     }
+
+    public static Func<Task> GetSeedMethod()
+    {
+        return async () =>
+        {
+            await PostgresqlDatabaseHelper.CleanupDatabaseAsync(_connectionString);
+            var seeder = new PostgresqlDatabaseSeeder(_connectionString);
+            await seeder.SeedAsync(
+                customerCount: 500, // with customer_id filter, this is 1/500 of the table returned
+                productsPerCategory: 150,
+                ordersPerCustomer: 1000,
+                itemsPerOrder: 20 // 20 * 1000 = 20,000 possible rows returned
+            );
+        };
+    }
 }
diff --git a/benchmark/BenchmarkRunner/Benchmarks/PostgresqlWriteBenchmark.cs b/benchmark/BenchmarkRunner/Benchmarks/PostgresqlWriteBenchmark.cs
@@ -28,27 +28,15 @@ public class PostgresqlWriteBenchmark : BaseWriteBenchmark
     [GlobalSetup]
     public async Task GlobalSetup()
     {
-        await Helpers.InitializeOnceAsync(async () =>
-        {
-            PostgresqlDatabaseHelper.CleanupDatabaseAsync(_connectionString).GetAwaiter().GetResult();
-            var seeder = new PostgresqlDatabaseSeeder(_connectionString);
-            await seeder.SeedAsync(
-                customerCount: 10,
-                productsPerCategory: 15,
-                ordersPerCustomer: 300,
-                itemsPerOrder: 0
-            );
+        var orderIds = await _sqlcImpl.GetOrderIdsAsync(new QuerySql.GetOrderIdsArgs(Limit: 1000));
+        var productIds = await _sqlcImpl.GetProductIdsAsync(new QuerySql.GetProductIdsArgs(Limit: 1000));
 
-            var orderIds = await _sqlcImpl.GetOrderIdsAsync(new QuerySql.GetOrderIdsArgs(Limit: 1000));
-            var productIds = await _sqlcImpl.GetProductIdsAsync(new QuerySql.GetProductIdsArgs(Limit: 1000));
-
-            _testOrderItems = [.. Enumerable.Range(0, TotalRecords).Select(i => new QuerySql.AddOrderItemsArgs(
-                OrderId: orderIds[i % orderIds.Count].OrderId,
-                ProductId: productIds[i % productIds.Count].ProductId,
-                Quantity: Random.Shared.Next(1, 10),
-                UnitPrice: (decimal)(Random.Shared.NextDouble() * 100 + 5)
-            ))];
-        });
+        _testOrderItems = [.. Enumerable.Range(0, TotalRecords).Select(i => new QuerySql.AddOrderItemsArgs(
+            OrderId: orderIds[i % orderIds.Count].OrderId,
+            ProductId: productIds[i % productIds.Count].ProductId,
+            Quantity: Random.Shared.Next(1, 10),
+            UnitPrice: (decimal)(Random.Shared.NextDouble() * 100 + 5)
+        ))];
     }
 
     [IterationSetup]
@@ -74,4 +62,19 @@ public override async Task EFCore_AddOrderItems()
         )).ToList();
         await Helpers.InsertInBatchesAsync(args, BatchSize, _efCoreImpl.AddOrderItems);
     }
+
+    public static Func<Task> GetSeedMethod()
+    {
+        return async () =>
+        {
+            PostgresqlDatabaseHelper.CleanupDatabaseAsync(_connectionString).GetAwaiter().GetResult();
+            var seeder = new PostgresqlDatabaseSeeder(_connectionString);
+            await seeder.SeedAsync(
+                customerCount: 10,
+                productsPerCategory: 15,
+                ordersPerCustomer: 300,
+                itemsPerOrder: 0
+            );
+        };
+    }
 }
diff --git a/benchmark/BenchmarkRunner/Benchmarks/SqliteReadBenchmark.cs b/benchmark/BenchmarkRunner/Benchmarks/SqliteReadBenchmark.cs
@@ -23,24 +23,6 @@ public class SqliteReadBenchmark : BaseReadBenchmark
     [Params(5, 25, 50)]
     public int ConcurrentQueries { get; set; }
 
-    [GlobalSetup]
-    public async Task GlobalSetup()
-    {
-        await Helpers.InitializeOnceAsync(async () =>
-        {
-            SqliteDatabaseHelper.CleanupDatabase(_connectionString);
-            await SqliteDatabaseHelper.InitializeDatabaseAsync(_connectionString);
-            var seeder = new SqliteDatabaseSeeder(_connectionString);
-            await seeder.SeedAsync(
-                customerCount: CustomerCount, // with customer_id filter, this is 1/500 of the table returned
-                productsPerCategory: 150,
-                ordersPerCustomer: 500,
-                itemsPerOrder: 10
-            // 10 * 500 = 5,000 possible rows returned
-            );
-        });
-    }
-
     [BenchmarkCategory("Read")]
     [Benchmark(Baseline = true, Description = "SQLC - GetCustomerOrders")]
     public override async Task Sqlc_GetCustomerOrders()
@@ -86,4 +68,20 @@ await ExecuteConcurrentlyAsync(QueriesToRun, ConcurrentQueries, async _ =>
             ));
         });
     }
+
+    public static Func<Task> GetSeedMethod()
+    {
+        return async () =>
+        {
+            SqliteDatabaseHelper.CleanupDatabase(_connectionString);
+            await SqliteDatabaseHelper.InitializeDatabaseAsync(_connectionString);
+            var seeder = new SqliteDatabaseSeeder(_connectionString);
+            await seeder.SeedAsync(
+                customerCount: 500, // with customer_id filter, this is 1/500 of the table returned
+                productsPerCategory: 150,
+                ordersPerCustomer: 500,
+                itemsPerOrder: 10 // 10 * 500 = 5,000 possible rows returned
+            );
+        };
+    }
 }
diff --git a/benchmark/BenchmarkRunner/Benchmarks/SqliteWriteBenchmark.cs b/benchmark/BenchmarkRunner/Benchmarks/SqliteWriteBenchmark.cs
@@ -30,28 +30,15 @@ public class SqliteWriteBenchmark : BaseWriteBenchmark
     [GlobalSetup]
     public async Task GlobalSetup()
     {
-        await Helpers.InitializeOnceAsync(async () =>
-        {
-            SqliteDatabaseHelper.CleanupDatabase(_connectionString);
-            await SqliteDatabaseHelper.InitializeDatabaseAsync(_connectionString);
-            var seeder = new SqliteDatabaseSeeder(_connectionString);
-            await seeder.SeedAsync(
-                customerCount: 10,
-                productsPerCategory: 15,
-                ordersPerCustomer: 300,
-                itemsPerOrder: 0
-            );
-
-            var orderIds = await _sqlcImpl.GetOrderIdsAsync(new QuerySql.GetOrderIdsArgs(Limit: 1000));
-            var productIds = await _sqlcImpl.GetProductIdsAsync(new QuerySql.GetProductIdsArgs(Limit: 1000));
+        var orderIds = await _sqlcImpl.GetOrderIdsAsync(new QuerySql.GetOrderIdsArgs(Limit: 1000));
+        var productIds = await _sqlcImpl.GetProductIdsAsync(new QuerySql.GetProductIdsArgs(Limit: 1000));
 
-            _testOrderItems = [.. Enumerable.Range(0, TotalRecords).Select(i => new QuerySql.AddOrderItemsArgs(
-                OrderId: orderIds[i % orderIds.Count].OrderId,
-                ProductId: productIds[i % productIds.Count].ProductId,
-                Quantity: Random.Shared.Next(1, 10),
-                UnitPrice: (decimal)(Random.Shared.NextDouble() * 100 + 5)
-            ))];
-        });
+        _testOrderItems = [.. Enumerable.Range(0, TotalRecords).Select(i => new QuerySql.AddOrderItemsArgs(
+            OrderId: orderIds[i % orderIds.Count].OrderId,
+            ProductId: productIds[i % productIds.Count].ProductId,
+            Quantity: Random.Shared.Next(1, 10),
+            UnitPrice: (decimal)(Random.Shared.NextDouble() * 100 + 5)
+        ))];
     }
 
     [IterationSetup]
@@ -77,4 +64,20 @@ public override async Task EFCore_AddOrderItems()
         )).ToList();
         await Helpers.InsertInBatchesAsync(args, BatchSize, _efCoreImpl.AddOrderItems);
     }
+
+    public static Func<Task> GetSeedMethod()
+    {
+        return async () =>
+        {
+            SqliteDatabaseHelper.CleanupDatabase(_connectionString);
+            await SqliteDatabaseHelper.InitializeDatabaseAsync(_connectionString);
+            var seeder = new SqliteDatabaseSeeder(_connectionString);
+            await seeder.SeedAsync(
+                customerCount: 10,
+                productsPerCategory: 15,
+                ordersPerCustomer: 300,
+                itemsPerOrder: 0
+            );
+        };
+    }
 }
diff --git a/benchmark/BenchmarkRunner/Runners/BaseRunner.cs b/benchmark/BenchmarkRunner/Runners/BaseRunner.cs
@@ -1,4 +1,4 @@
-public abstract class BaseBenchmarkRunner
+public abstract class BaseRunner
 {
     public virtual string GetBasePath() => Path.Combine("benchmark", "BenchmarkDotNet.Artifacts");
     public abstract Task RunReadsAsync();
diff --git a/benchmark/BenchmarkRunner/Runners/MysqlRunner.cs b/benchmark/BenchmarkRunner/Runners/MysqlRunner.cs
@@ -1,25 +1,25 @@
 using BenchmarkDotNet.Configs;
 using BenchmarkRunner.Benchmarks;
 
-public sealed class MysqlRunner : BaseBenchmarkRunner
+public sealed class MysqlRunner : BaseRunner
 {
     public override string GetBasePath() => Path.Combine(base.GetBasePath(), "mysql");
 
-    public override Task RunReadsAsync()
+    public override async Task RunReadsAsync()
     {
+        await MysqlReadBenchmark.GetSeedMethod().Invoke();
         var path = Path.Combine(GetBasePath(), "reads");
         BenchmarkDotNet.Running.BenchmarkRunner.Run<MysqlReadBenchmark>(
             DefaultConfig.Instance.WithArtifactsPath(path)
         );
-        return Task.CompletedTask;
     }
 
-    public override Task RunWritesAsync()
+    public override async Task RunWritesAsync()
     {
+        await MysqlWriteBenchmark.GetSeedMethod().Invoke();
         var path = Path.Combine(GetBasePath(), "writes");
         BenchmarkDotNet.Running.BenchmarkRunner.Run<MysqlWriteBenchmark>(
             DefaultConfig.Instance.WithArtifactsPath(path)
         );
-        return Task.CompletedTask;
     }
 }
diff --git a/benchmark/BenchmarkRunner/Runners/PostgresqlRunner.cs b/benchmark/BenchmarkRunner/Runners/PostgresqlRunner.cs
@@ -1,25 +1,25 @@
 using BenchmarkDotNet.Configs;
 using BenchmarkRunner.Benchmarks;
 
-public sealed class PostgresqlRunner : BaseBenchmarkRunner
+public sealed class PostgresqlRunner : BaseRunner
 {
     public override string GetBasePath() => Path.Combine(base.GetBasePath(), "postgresql");
 
-    public override Task RunReadsAsync()
+    public override async Task RunReadsAsync()
     {
+        await PostgresqlReadBenchmark.GetSeedMethod().Invoke();
         var path = Path.Combine(GetBasePath(), "reads");
         BenchmarkDotNet.Running.BenchmarkRunner.Run<PostgresqlReadBenchmark>(
             DefaultConfig.Instance.WithArtifactsPath(path)
         );
-        return Task.CompletedTask;
     }
 
-    public override Task RunWritesAsync()
+    public override async Task RunWritesAsync()
     {
+        await PostgresqlWriteBenchmark.GetSeedMethod().Invoke();
         var path = Path.Combine(GetBasePath(), "writes");
         BenchmarkDotNet.Running.BenchmarkRunner.Run<PostgresqlWriteBenchmark>(
             DefaultConfig.Instance.WithArtifactsPath(path)
         );
-        return Task.CompletedTask;
     }
 }
diff --git a/benchmark/BenchmarkRunner/Runners/SqliteRunner.cs b/benchmark/BenchmarkRunner/Runners/SqliteRunner.cs

Original file line number	Diff line number	Diff line change
`@@ -23,7 +23,7 @@ private static int CalculateMaxConcurrency(int totalTasks, int maxConcurrency)`
`23`	`23`	`maxConcurrency, totalTasks, Environment.ProcessorCount`
`24`	`24`	`}.Min(x => x);`
`25`	`25`	`}`
`26`		`- public static async Task<List<T>> ExecuteConcurrentlyAsync<T>(`
	`26`	`+ protected static async Task<List<T>> ExecuteConcurrentlyAsync<T>(`
`27`	`27`	`int totalTasks,`
`28`	`28`	`int maxConcurrency,`
`29`	`29`	`Func<int, Task<List<T>>> taskFactory)`
`@@ -41,7 +41,9 @@ public static async Task<List<T>> ExecuteConcurrentlyAsync<T>(`
`41`	`41`	`return [.. results.SelectMany(r => r)];`
`42`	`42`	`}`
`43`	`43`
`44`		`- private static async Task<List<T>> ExecuteWithThrottleAsync<T>(SemaphoreSlim semaphore, Func<Task<List<T>>> taskFactory)`
	`44`	`+ private static async Task<List<T>> ExecuteWithThrottleAsync<T>(`
	`45`	`+ SemaphoreSlim semaphore,`
	`46`	`+ Func<Task<List<T>>> taskFactory)`
`45`	`47`	`{`
`46`	`48`	`await semaphore.WaitAsync();`
`47`	`49`	`try`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-public abstract class BaseBenchmarkRunner`
	`1`	`+public abstract class BaseRunner`
`2`	`2`	`{`
`3`	`3`	`public virtual string GetBasePath() => Path.Combine("benchmark", "BenchmarkDotNet.Artifacts");`
`4`	`4`	`public abstract Task RunReadsAsync();`
Original file line number	Diff line number	Diff line change
`@@ -1,25 +1,25 @@`
`1`	`1`	`using BenchmarkDotNet.Configs;`
`2`	`2`	`using BenchmarkRunner.Benchmarks;`
`3`	`3`
`4`		`-public sealed class MysqlRunner : BaseBenchmarkRunner`
	`4`	`+public sealed class MysqlRunner : BaseRunner`
`5`	`5`	`{`
`6`	`6`	`public override string GetBasePath() => Path.Combine(base.GetBasePath(), "mysql");`
`7`	`7`
`8`		`- public override Task RunReadsAsync()`
	`8`	`+ public override async Task RunReadsAsync()`
`9`	`9`	`{`
	`10`	`+ await MysqlReadBenchmark.GetSeedMethod().Invoke();`
`10`	`11`	`var path = Path.Combine(GetBasePath(), "reads");`
`11`	`12`	`BenchmarkDotNet.Running.BenchmarkRunner.Run<MysqlReadBenchmark>(`
`12`	`13`	`DefaultConfig.Instance.WithArtifactsPath(path)`
`13`	`14`	`);`
`14`		`- return Task.CompletedTask;`
`15`	`15`	`}`
`16`	`16`
`17`		`- public override Task RunWritesAsync()`
	`17`	`+ public override async Task RunWritesAsync()`
`18`	`18`	`{`
	`19`	`+ await MysqlWriteBenchmark.GetSeedMethod().Invoke();`
`19`	`20`	`var path = Path.Combine(GetBasePath(), "writes");`
`20`	`21`	`BenchmarkDotNet.Running.BenchmarkRunner.Run<MysqlWriteBenchmark>(`
`21`	`22`	`DefaultConfig.Instance.WithArtifactsPath(path)`
`22`	`23`	`);`
`23`		`- return Task.CompletedTask;`
`24`	`24`	`}`
`25`	`25`	`}`