From f1c4e7df2899f39e12164fadb763bdaf854d8118 Mon Sep 17 00:00:00 2001 From: nikhitha malkapuram <nmalkapuram@microsoft.com> Date: Fri, 18 Oct 2024 12:08:08 +0530 Subject: [PATCH 01/11] adding truncate tables feature for MYSQL sysbench OTLP recipe --- .../Sysbench/SysbenchClientExecutor.cs | 96 +++++++- .../Sysbench/SysbenchConfiguration.cs | 35 +++ .../Sysbench/SysbenchExecutor.cs | 39 +++- .../Sysbench/original-populate-databse.py | 86 +++++++ .../populate-database-oldnewbalance.py | 102 ++++++++ .../Sysbench/populate-database.py | 99 ++++++++ .../Sysbench/truncate-tables.py | 35 +++ .../MySqlServer/MySqlServerConfiguration.cs | 118 +++++++++- .../COMPETE-PERF-MYSQL-SYSBENCH-OLTP.json | 220 ++++++++++++++++++ 9 files changed, 827 insertions(+), 3 deletions(-) create mode 100644 src/VirtualClient/VirtualClient.Actions/Sysbench/original-populate-databse.py create mode 100644 src/VirtualClient/VirtualClient.Actions/Sysbench/populate-database-oldnewbalance.py create mode 100644 src/VirtualClient/VirtualClient.Actions/Sysbench/populate-database.py create mode 100644 src/VirtualClient/VirtualClient.Actions/Sysbench/truncate-tables.py create mode 100644 src/VirtualClient/VirtualClient.Main/profiles/COMPETE-PERF-MYSQL-SYSBENCH-OLTP.json diff --git a/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchClientExecutor.cs b/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchClientExecutor.cs index 3f82e2ed71..b7b17d1a3a 100644 --- a/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchClientExecutor.cs +++ b/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchClientExecutor.cs @@ -7,6 +7,7 @@ namespace VirtualClient.Actions using System.Collections.Generic; using System.Globalization; using System.Linq; + using System.Net; using System.Net.Http; using System.Text.RegularExpressions; using System.Threading; @@ -25,8 +26,11 @@ namespace VirtualClient.Actions /// </summary> public class SysbenchClientExecutor : SysbenchExecutor { + private const string PythonCommand = "python3"; private string sysbenchExecutionArguments; private string sysbenchLoggingArguments; + private string sysbenchPrepareArguments; + private string packageDirectory; /// <summary> /// Initializes a new instance of the <see cref="SysbenchClientExecutor"/> class. @@ -169,7 +173,24 @@ private Task ExecuteWorkloadAsync(EventContext telemetryContext, CancellationTok { if (this.Benchmark == BenchmarkName.OLTP) { - await this.RunOLTPWorkloadAsync(telemetryContext, cancellationToken); + if (this.Action == ClientAction.TruncateDatabase) + { + DependencyPath workloadPackage = await this.GetPackageAsync(this.PackageName, cancellationToken).ConfigureAwait(false); + workloadPackage.ThrowIfNull(this.PackageName); + + DependencyPath package = await this.GetPlatformSpecificPackageAsync(this.PackageName, cancellationToken); + this.packageDirectory = package.Path; + + await this.TruncateMySQLDatabaseAsync(telemetryContext, cancellationToken).ConfigureAwait(false); + } + else if (this.Action == ClientAction.PopulateDatabase) + { + await this.PrepareOLTPMySQLDatabase(telemetryContext, cancellationToken); + } + else + { + await this.RunOLTPWorkloadAsync(telemetryContext, cancellationToken); + } } else if (this.Benchmark == BenchmarkName.TPCC) { @@ -242,5 +263,78 @@ private async Task RunTPCCWorkloadAsync(EventContext telemetryContext, Cancellat } } } + + private async Task TruncateMySQLDatabaseAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + string arguments = $"{this.packageDirectory}/truncate-tables.py --dbName {this.DatabaseName}"; + + if (this.IsMultiRoleLayout()) + { + ClientInstance instance = this.Layout.GetClientInstance(this.AgentId); + string serverIps = (instance.Role == ClientRole.Server) ? "localhost" : this.GetServerIpAddress(); + arguments += $" --clientIps \"{serverIps}\""; + } + + using (IProcessProxy process = await this.ExecuteCommandAsync( + PythonCommand, + arguments, + Environment.CurrentDirectory, + telemetryContext, + cancellationToken)) + { + if (!cancellationToken.IsCancellationRequested) + { + await this.LogProcessDetailsAsync(process, telemetryContext, "MySQLServerConfiguration", logToFile: true); + process.ThrowIfDependencyInstallationFailed(process.StandardError.ToString()); + } + } + } + + private async Task PrepareOLTPMySQLDatabase(EventContext telemetryContext, CancellationToken cancellationToken) + { + int tableCount = GetTableCount(this.DatabaseScenario, this.TableCount, this.Workload); + int threadCount = GetThreadCount(this.SystemManager, this.DatabaseScenario, this.Threads); + int recordCount = GetRecordCount(this.SystemManager, this.DatabaseScenario, this.RecordCount); + + this.sysbenchPrepareArguments = $"--dbName {this.DatabaseName} --databaseSystem {this.DatabaseSystem} --benchmark {this.Benchmark} --tableCount {tableCount} --recordCount {recordCount} --threadCount {threadCount} --password {this.SuperUserPassword}"; + + if (this.IsMultiRoleLayout()) + { + ClientInstance instance = this.Layout.GetClientInstance(this.AgentId); + string serverIp = (instance.Role == ClientRole.Server) ? "localhost" : this.GetServerIpAddress(); + this.sysbenchPrepareArguments += $" --host \"{serverIp}\""; + } + + string command = $"python3"; + string arguments = $"{this.SysbenchPackagePath}/populate-database.py "; + + using (IProcessProxy process = await this.ExecuteCommandAsync( + command, + arguments + this.sysbenchPrepareArguments, + this.SysbenchPackagePath, + telemetryContext, + cancellationToken)) + { + if (!cancellationToken.IsCancellationRequested) + { + await this.LogProcessDetailsAsync(process, telemetryContext, "Sysbench", logToFile: true); + process.ThrowIfErrored<WorkloadException>(process.StandardError.ToString(), ErrorReason.WorkloadUnexpectedAnomaly); + } + } + } + + private string GetServerIpAddress() + { + string serverIPAddress = IPAddress.Loopback.ToString(); + + if (this.IsMultiRoleLayout()) + { + ClientInstance serverInstance = this.GetLayoutClientInstances(ClientRole.Server).First(); + IPAddress.TryParse(serverInstance.IPAddress, out IPAddress serverIP); + serverIPAddress = serverIP.ToString(); + } + + return serverIPAddress; + } } } diff --git a/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchConfiguration.cs b/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchConfiguration.cs index 42e0f102c8..a717e00423 100644 --- a/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchConfiguration.cs +++ b/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchConfiguration.cs @@ -5,6 +5,9 @@ namespace VirtualClient.Actions { using System; using System.Collections.Generic; + using System.ComponentModel; + using System.Linq; + using System.Net; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; @@ -32,6 +35,17 @@ public SysbenchConfiguration(IServiceCollection dependencies, IDictionary<string this.stateManager = this.Dependencies.GetService<IStateManager>(); } +/* /// <summary> + /// The specifed action that controls the execution of the dependency. + /// </summary> + public string WaitForServer + { + get + { + return this.Parameters.GetValue<string>(nameof(this.WaitForServer)); + } + }*/ + /// <summary> /// Executes the workload. /// </summary> @@ -81,6 +95,13 @@ private async Task PrepareOLTPMySQLDatabase(EventContext telemetryContext, Cance this.sysbenchPrepareArguments = $"--dbName {this.DatabaseName} --databaseSystem {this.DatabaseSystem} --benchmark {this.Benchmark} --tableCount {tableCount} --recordCount {recordCount} --threadCount {threadCount} --password {this.SuperUserPassword}"; + if (this.IsMultiRoleLayout()) + { + ClientInstance instance = this.Layout.GetClientInstance(this.AgentId); + string serverIp = (instance.Role == ClientRole.Server) ? "localhost" : this.GetServerIpAddress(); + this.sysbenchPrepareArguments += $" --host \"{serverIp}\""; + } + string command = $"python3"; string arguments = $"{this.SysbenchPackagePath}/populate-database.py "; @@ -124,5 +145,19 @@ private async Task PrepareTPCCMySQLDatabase(EventContext telemetryContext, Cance } } } + + private string GetServerIpAddress() + { + string serverIPAddress = IPAddress.Loopback.ToString(); + + if (this.IsMultiRoleLayout()) + { + ClientInstance serverInstance = this.GetLayoutClientInstances(ClientRole.Server).First(); + IPAddress.TryParse(serverInstance.IPAddress, out IPAddress serverIP); + serverIPAddress = serverIP.ToString(); + } + + return serverIPAddress; + } } } diff --git a/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchExecutor.cs b/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchExecutor.cs index dd0bed98a5..bebad56b38 100644 --- a/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchExecutor.cs +++ b/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchExecutor.cs @@ -175,6 +175,19 @@ public string Workload } } + /// <summary> + /// The specifed action that controls the execution of the dependency. + /// </summary> + public string Action + { + get + { + this.Parameters.TryGetValue(nameof(this.Action), out IConvertible action); + return action?.ToString(); + // return this.Parameters.GetValue<string>(nameof(this.Action), null); + } + } + /// <summary> /// Client used to communicate with the hosted instance of the /// Virtual Client API at server side. @@ -299,7 +312,10 @@ await this.CheckDistroSupportAsync(telemetryContext, cancellationToken) DependencyPath package = await this.GetPackageAsync(this.PackageName, cancellationToken); this.SysbenchPackagePath = package.Path; - await this.InitializeExecutablesAsync(telemetryContext, cancellationToken); + if (this.Action != ClientAction.TruncateDatabase) + { + await this.InitializeExecutablesAsync(telemetryContext, cancellationToken); + } this.InitializeApiClients(telemetryContext, cancellationToken); @@ -460,5 +476,26 @@ internal class BenchmarkName public const string OLTP = nameof(OLTP); public const string TPCC = nameof(TPCC); } + + /// <summary> + /// Supported Sysbench Client actions. + /// </summary> + internal class ClientAction + { + /// <summary> + /// Creates Database on MySQL server and Users on Server and any Clients. + /// </summary> + public const string PopulateDatabase = nameof(PopulateDatabase); + + /// <summary> + /// Truncates all tables existing in database + /// </summary> + public const string TruncateDatabase = nameof(TruncateDatabase); + + /// <summary> + /// Truncates all tables existing in database + /// </summary> + public const string RunWorkload = nameof(RunWorkload); + } } } \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Actions/Sysbench/original-populate-databse.py b/src/VirtualClient/VirtualClient.Actions/Sysbench/original-populate-databse.py new file mode 100644 index 0000000000..7a33d07ac6 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions/Sysbench/original-populate-databse.py @@ -0,0 +1,86 @@ +import subprocess, argparse, os + +parser = argparse.ArgumentParser() + +parser.add_argument('-n', '--dbName', type=str, help='Database Name', required=True) +parser.add_argument('-y', '--databaseSystem', type=str, help='Database Type', required=False) +parser.add_argument('-b', '--benchmark', type=str, help="Benchmark Name", required=True) +parser.add_argument('-t', '--tableCount', type=str, help='Number of Tables', required=True) +parser.add_argument('-r', '--recordCount', type=str, help='Number of Records', required=False) +parser.add_argument('-u', '--warehouses', type=str, help='Warehouse Count', required=False) +parser.add_argument('-e', '--threadCount', type=str, help="Number of Threads", required=True) +parser.add_argument('--password', type=str, help="PostgreSQL Password", required=False) + +args = parser.parse_args() +dbName = args.dbName +databaseSystem = args.databaseSystem +benchmark = args.benchmark +warehouses = args.warehouses +tableCount = args.tableCount +recordCount = args.recordCount +threadCount = args.threadCount +password = args.password + +if databaseSystem == "MySQL": + if benchmark == "TPCC": + subprocess.run(f'sudo src/sysbench tpcc --tables={tableCount} --scale={warehouses} --threads={threadCount} --mysql-db={dbName} --use_fk=0 prepare', shell=True, check=True) + if int(warehouses) == 1: + for i in range(1,int(tableCount)+1): + table = str(i) + # drop idxs + subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "DROP INDEX idx_customer{i} ON customer{i};"', shell=True, check=True) + subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "DROP INDEX idx_orders{i} ON orders{i};"', shell=True, check=True) + subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "DROP INDEX fkey_stock_2{i} ON stock{i};"', shell=True, check=True) + subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "DROP INDEX fkey_order_line_2{i} ON order_line{i};"', shell=True, check=True) + subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "DROP INDEX fkey_history_1{i} ON history{i};"', shell=True, check=True) + subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "DROP INDEX fkey_history_2{i} ON history{i};"', shell=True, check=True) + + # truncate, to make distributing faster + subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE warehouse{i};"', shell=True, check=True) + subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE district{i};"', shell=True, check=True) + subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE customer{i};"', shell=True, check=True) + subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE history{i};"', shell=True, check=True) + subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE orders{i};"', shell=True, check=True) + subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE new_orders{i};"', shell=True, check=True) + subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE order_line{i};"', shell=True, check=True) + subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE stock{i};"', shell=True, check=True) + subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE item{i};"', shell=True, check=True) + else: + subprocess.run(f'sudo src/sysbench oltp_common --tables={tableCount} --table-size={recordCount} --threads={threadCount} --mysql-db={dbName} prepare', shell=True, check=True) + if int(recordCount) == 1: + for i in range(1,int(tableCount)+1): + table = str(i) + subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "DROP INDEX k_{i} ON sbtest{i};"', shell=True, check=True) +elif databaseSystem == "PostgreSQL": + os.environ['PGPASSWORD'] = password + if benchmark == "TPCC": + subprocess.run(f'sudo src/sysbench tpcc --db-driver=pgsql --tables={tableCount} --scale={warehouses} --threads={threadCount} --pgsql-user=postgres --pgsql-password={password} --pgsql-db={dbName} --use_fk=0 prepare', shell=True, check=True) + if int(warehouses) == 1: + for i in range(1,int(tableCount)+1): + table = str(i) + # drop idxs + subprocess.run(f'psql -U postgres -d {dbName} -c "DROP INDEX IF EXISTS idx_customer{i};"', shell=True, check=True) + subprocess.run(f'psql -U postgres -d {dbName} -c "DROP INDEX IF EXISTS idx_orders{i};"', shell=True, check=True) + subprocess.run(f'psql -U postgres -d {dbName} -c "DROP INDEX IF EXISTS fkey_stock_2{i};"', shell=True, check=True) + subprocess.run(f'psql -U postgres -d {dbName} -c "DROP INDEX IF EXISTS fkey_order_line_2{i};"', shell=True, check=True) + subprocess.run(f'psql -U postgres -d {dbName} -c "DROP INDEX IF EXISTS fkey_history_1{i};"', shell=True, check=True) + subprocess.run(f'psql -U postgres -d {dbName} -c "DROP INDEX IF EXISTS fkey_history_2{i};"', shell=True, check=True) + + # truncate, to make distributing faster + subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE warehouse{i};"', shell=True, check=True) + subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE district{i};"', shell=True, check=True) + subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE customer{i};"', shell=True, check=True) + subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE history{i};"', shell=True, check=True) + subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE orders{i};"', shell=True, check=True) + subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE new_orders{i};"', shell=True, check=True) + subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE order_line{i};"', shell=True, check=True) + subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE stock{i};"', shell=True, check=True) + subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE item{i};"', shell=True, check=True) + else: + subprocess.run(f'sudo src/sysbench oltp_common --db-driver=pgsql --tables={tableCount} --table-size={recordCount} --threads={threadCount} --pgsql-user=postgres --pgsql-password={password} --pgsql-db={dbName} prepare', shell=True, check=True) + if int(recordCount) == 1: + for i in range(1,int(tableCount)+1): + table = str(i) + subprocess.run(f'psql -U postgres -d {dbName} -c "DROP INDEX IF EXISTS k_{i};"', shell=True, check=True) +else: + parser.error("You are running on a database system type that has not been onboarded to Virtual Client. Available options are: MySQL, PostgreSQL") \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Actions/Sysbench/populate-database-oldnewbalance.py b/src/VirtualClient/VirtualClient.Actions/Sysbench/populate-database-oldnewbalance.py new file mode 100644 index 0000000000..107c394821 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions/Sysbench/populate-database-oldnewbalance.py @@ -0,0 +1,102 @@ +import subprocess +import argparse +import os + +# Define arguments +parser = argparse.ArgumentParser() + +parser.add_argument('-n', '--dbName', type=str, help='Database Name', required=True) +parser.add_argument('-y', '--databaseSystem', type=str, help='Database Type (MySQL/PostgreSQL)', required=False) +parser.add_argument('-b', '--benchmark', type=str, help="Benchmark Name", required=True) +parser.add_argument('-t', '--tableCount', type=str, help='Number of Tables', required=True) +parser.add_argument('-r', '--recordCount', type=str, help='Number of Records', required=False) +parser.add_argument('-u', '--warehouses', type=str, help='Warehouse Count', required=False) +parser.add_argument('-e', '--threadCount', type=str, help="Number of Threads", required=True) +parser.add_argument('--password', type=str, help="PostgreSQL Password", required=False) +parser.add_argument('--host', type=str, help="Database Server Host IP Address", required=False) + +# Parse arguments +args = parser.parse_args() +dbName = args.dbName +databaseSystem = args.databaseSystem +benchmark = args.benchmark +warehouses = args.warehouses +tableCount = args.tableCount +recordCount = args.recordCount +threadCount = args.threadCount +password = args.password +host = args.host # Host can be None if not provided + +# Function to add host if provided +def add_host_to_command(command_base, host): + if host: + return f"{command_base} --mysql-host={host}" if "mysql" in command_base else f"{command_base} --pgsql-host={host}" + return command_base + +# MySQL handling +if databaseSystem == "MySQL": + if benchmark == "TPCC": + command_base = f'sudo src/sysbench tpcc --tables={tableCount} --scale={warehouses} --threads={threadCount} --mysql-db={dbName} --use_fk=0 prepare' + command = add_host_to_command(command_base, host) + subprocess.run(command, shell=True, check=True) + + if int(warehouses) == 1: + for i in range(1, int(tableCount) + 1): + table = str(i) + # drop indexes + for index in ["idx_customer", "idx_orders", "fkey_stock_2", "fkey_order_line_2", "fkey_history_1", "fkey_history_2"]: + drop_index_command = f'mysql -u {dbName} -p -e "DROP INDEX {index}{i} ON {index.split("_")[1]}{i};"' + drop_index_command = add_host_to_command(drop_index_command, host) + subprocess.run(drop_index_command, shell=True, check=True) + + # truncate tables + for table in ["warehouse", "district", "customer", "history", "orders", "new_orders", "order_line", "stock", "item"]: + truncate_command = f'mysql -u {dbName} -p -e "TRUNCATE TABLE {table}{i};"' + truncate_command = add_host_to_command(truncate_command, host) + subprocess.run(truncate_command, shell=True, check=True) + else: + command_base = f'sudo src/sysbench oltp_common --tables={tableCount} --table-size={recordCount} --threads={threadCount} --mysql-db={dbName} prepare' + command = add_host_to_command(command_base, host) + subprocess.run(command, shell=True, check=True) + + if int(recordCount) == 1: + for i in range(1, int(tableCount) + 1): + drop_index_command = f'mysql -u {dbName} -p -e "DROP INDEX k_{i} ON sbtest{i};"' + drop_index_command = add_host_to_command(drop_index_command, host) + subprocess.run(drop_index_command, shell=True, check=True) + +# PostgreSQL handling +elif databaseSystem == "PostgreSQL": + os.environ['PGPASSWORD'] = password + if benchmark == "TPCC": + command_base = f'sudo src/sysbench tpcc --db-driver=pgsql --tables={tableCount} --scale={warehouses} --threads={threadCount} --pgsql-user=postgres --pgsql-password={password} --pgsql-db={dbName} --use_fk=0 prepare' + command = add_host_to_command(command_base, host) + subprocess.run(command, shell=True, check=True) + + if int(warehouses) == 1: + for i in range(1, int(tableCount) + 1): + table = str(i) + # drop indexes + for index in ["idx_customer", "idx_orders", "fkey_stock_2", "fkey_order_line_2", "fkey_history_1", "fkey_history_2"]: + drop_index_command = f'psql -U postgres -d {dbName} -c "DROP INDEX IF EXISTS {index}{i};"' + drop_index_command = add_host_to_command(drop_index_command, host) + subprocess.run(drop_index_command, shell=True, check=True) + + # truncate tables + for table in ["warehouse", "district", "customer", "history", "orders", "new_orders", "order_line", "stock", "item"]: + truncate_command = f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE {table}{i};"' + truncate_command = add_host_to_command(truncate_command, host) + subprocess.run(truncate_command, shell=True, check=True) + else: + command_base = f'sudo src/sysbench oltp_common --db-driver=pgsql --tables={tableCount} --table-size={recordCount} --threads={threadCount} --pgsql-user=postgres --pgsql-password={password} --pgsql-db={dbName} prepare' + command = add_host_to_command(command_base, host) + subprocess.run(command, shell=True, check=True) + + if int(recordCount) == 1: + for i in range(1, int(tableCount) + 1): + drop_index_command = f'psql -U postgres -d {dbName} -c "DROP INDEX IF EXISTS k_{i};"' + drop_index_command = add_host_to_command(drop_index_command, host) + subprocess.run(drop_index_command, shell=True, check=True) + +else: + parser.error("You are running on a database system type that has not been onboarded to Virtual Client. Available options are: MySQL, PostgreSQL") diff --git a/src/VirtualClient/VirtualClient.Actions/Sysbench/populate-database.py b/src/VirtualClient/VirtualClient.Actions/Sysbench/populate-database.py new file mode 100644 index 0000000000..c4f36de5d1 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions/Sysbench/populate-database.py @@ -0,0 +1,99 @@ +import subprocess, argparse, os + +parser = argparse.ArgumentParser() + +parser.add_argument('-n', '--dbName', type=str, help='Database Name', required=True) +parser.add_argument('-y', '--databaseSystem', type=str, help='Database Type', required=False) +parser.add_argument('-b', '--benchmark', type=str, help="Benchmark Name", required=True) +parser.add_argument('-t', '--tableCount', type=str, help='Number of Tables', required=True) +parser.add_argument('-r', '--recordCount', type=str, help='Number of Records', required=False) +parser.add_argument('-u', '--warehouses', type=str, help='Warehouse Count', required=False) +parser.add_argument('-e', '--threadCount', type=str, help="Number of Threads", required=True) +parser.add_argument('--password', type=str, help="PostgreSQL Password", required=False) +parser.add_argument('--host', type=str, help="Database Server Host IP Address", required=False) + + +args = parser.parse_args() +dbName = args.dbName +databaseSystem = args.databaseSystem +benchmark = args.benchmark +warehouses = args.warehouses +tableCount = args.tableCount +recordCount = args.recordCount +threadCount = args.threadCount +password = args.password +host = args.host + +def add_host_if_needed(command_base, host, benchmark): + if host and benchmark != "TPCC": + if "sysbench" not in command_base: + return command_base.replace('-u', f'-h {host} -u') + else: + return f"{command_base} --mysql-host={host}" + +if databaseSystem == "MySQL": + if benchmark == "TPCC": + subprocess.run(f'sudo src/sysbench tpcc --tables={tableCount} --scale={warehouses} --threads={threadCount} --mysql-db={dbName} --use_fk=0 prepare', shell=True, check=True) + if int(warehouses) == 1: + for i in range(1,int(tableCount)+1): + table = str(i) + # drop idxs + subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "DROP INDEX idx_customer{i} ON customer{i};"', shell=True, check=True) + subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "DROP INDEX idx_orders{i} ON orders{i};"', shell=True, check=True) + subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "DROP INDEX fkey_stock_2{i} ON stock{i};"', shell=True, check=True) + subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "DROP INDEX fkey_order_line_2{i} ON order_line{i};"', shell=True, check=True) + subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "DROP INDEX fkey_history_1{i} ON history{i};"', shell=True, check=True) + subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "DROP INDEX fkey_history_2{i} ON history{i};"', shell=True, check=True) + + # truncate, to make distributing faster + subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE warehouse{i};"', shell=True, check=True) + subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE district{i};"', shell=True, check=True) + subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE customer{i};"', shell=True, check=True) + subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE history{i};"', shell=True, check=True) + subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE orders{i};"', shell=True, check=True) + subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE new_orders{i};"', shell=True, check=True) + subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE order_line{i};"', shell=True, check=True) + subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE stock{i};"', shell=True, check=True) + subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE item{i};"', shell=True, check=True) + else: + command_base = f'sudo src/sysbench oltp_common --tables={tableCount} --table-size={recordCount} --threads={threadCount} --mysql-db={dbName} prepare' + command = add_host_if_needed(command_base, host, benchmark) + subprocess.run(command, shell=True, check=True) + if int(recordCount) == 1: + for i in range(1, int(tableCount) + 1): + drop_index_command = f'sudo mysql -u {dbName} {dbName} -e "DROP INDEX k_{i} ON sbtest{i};"' + drop_index_command = add_host_if_needed(drop_index_command, host, benchmark) + subprocess.run(drop_index_command, shell=True, check=True) +elif databaseSystem == "PostgreSQL": + os.environ['PGPASSWORD'] = password + if benchmark == "TPCC": + subprocess.run(f'sudo src/sysbench tpcc --db-driver=pgsql --tables={tableCount} --scale={warehouses} --threads={threadCount} --pgsql-user=postgres --pgsql-password={password} --pgsql-db={dbName} --use_fk=0 prepare', shell=True, check=True) + if int(warehouses) == 1: + for i in range(1,int(tableCount)+1): + table = str(i) + # drop idxs + subprocess.run(f'psql -U postgres -d {dbName} -c "DROP INDEX IF EXISTS idx_customer{i};"', shell=True, check=True) + subprocess.run(f'psql -U postgres -d {dbName} -c "DROP INDEX IF EXISTS idx_orders{i};"', shell=True, check=True) + subprocess.run(f'psql -U postgres -d {dbName} -c "DROP INDEX IF EXISTS fkey_stock_2{i};"', shell=True, check=True) + subprocess.run(f'psql -U postgres -d {dbName} -c "DROP INDEX IF EXISTS fkey_order_line_2{i};"', shell=True, check=True) + subprocess.run(f'psql -U postgres -d {dbName} -c "DROP INDEX IF EXISTS fkey_history_1{i};"', shell=True, check=True) + subprocess.run(f'psql -U postgres -d {dbName} -c "DROP INDEX IF EXISTS fkey_history_2{i};"', shell=True, check=True) + + # truncate, to make distributing faster + subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE warehouse{i};"', shell=True, check=True) + subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE district{i};"', shell=True, check=True) + subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE customer{i};"', shell=True, check=True) + subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE history{i};"', shell=True, check=True) + subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE orders{i};"', shell=True, check=True) + subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE new_orders{i};"', shell=True, check=True) + subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE order_line{i};"', shell=True, check=True) + subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE stock{i};"', shell=True, check=True) + subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE item{i};"', shell=True, check=True) + else: + subprocess.run(f'sudo src/sysbench oltp_common --db-driver=pgsql --tables={tableCount} --table-size={recordCount} --threads={threadCount} --pgsql-user=postgres --pgsql-password={password} --pgsql-db={dbName} prepare', shell=True, check=True) + if int(recordCount) == 1: + for i in range(1,int(tableCount)+1): + table = str(i) + subprocess.run(f'psql -U postgres -d {dbName} -c "DROP INDEX IF EXISTS k_{i};"', shell=True, check=True) +else: + parser.error("You are running on a database system type that has not been onboarded to Virtual Client. Available options are: MySQL, PostgreSQL") \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Actions/Sysbench/truncate-tables.py b/src/VirtualClient/VirtualClient.Actions/Sysbench/truncate-tables.py new file mode 100644 index 0000000000..91b455c01b --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions/Sysbench/truncate-tables.py @@ -0,0 +1,35 @@ +import subprocess +import argparse + +# Set up argument parsing +parser = argparse.ArgumentParser() + +parser.add_argument('--dbName', type=str, help='Database Name', required=True) +parser.add_argument('--clientIps', type=str, help='Client IP Address (separate multiple with semicolons)', required=False) + +args = parser.parse_args() +dbName = args.dbName +clientIps = args.clientIps + + # Function to truncate all tables in the specified database +def truncate_tables(db_user, host): + # Get all table names from the database + output = subprocess.run(f'sudo mysql -u {db_user} -h {host} {dbName} -e "SHOW TABLES;"', shell=True, check=True, capture_output=True, text=True) + + # Extract table names from the output + tables = [line.strip() for line in output.stdout.strip().split('\n') if line.startswith('sbtest')] + + # Truncate each table + for table in tables: + subprocess.run(f'sudo mysql -u {db_user} -h {host} {dbName} -e "TRUNCATE TABLE {table};"', shell=True, check=True) + +# Multi-VM: If client IPs are specified, truncate tables for each client VM IP +if clientIps: + clientIps = clientIps.split(';') + clientIps = list(filter(None, clientIps)) + + for clientIp in clientIps: + truncate_tables(db_user=dbName, host=clientIp) +else: + truncate_tables(db_user=dbName, host='localhost') +print(f"Successfully truncated tables in database '{dbName}' for all specified hosts.") \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Dependencies/MySqlServer/MySqlServerConfiguration.cs b/src/VirtualClient/VirtualClient.Dependencies/MySqlServer/MySqlServerConfiguration.cs index e6957dd718..e14076d9c0 100644 --- a/src/VirtualClient/VirtualClient.Dependencies/MySqlServer/MySqlServerConfiguration.cs +++ b/src/VirtualClient/VirtualClient.Dependencies/MySqlServer/MySqlServerConfiguration.cs @@ -52,6 +52,17 @@ public string Action } } + /*/// <summary> + /// The specifed action that controls the execution of the dependency. + /// </summary> + public bool WaitForServer + { + get + { + return this.Parameters.GetValue<bool>(nameof(this.WaitForServer), false); + } + }*/ + /// <summary> /// The specifed action that controls the execution of the dependency. /// </summary> @@ -113,6 +124,68 @@ public bool InMemory /// </summary> protected ISystemManagement SystemManager { get; } + /* + /// <summary> + /// Client used to communicate with the hosted instance of the + /// Virtual Client API at server side. + /// </summary> + public IApiClient ServerApiClient { get; set; }*/ + + /* + /// <summary> + /// Server IpAddress on which MySQL Server runs. + /// </summary> + protected string ServerIpAddress { get; set; } + + /// <summary> + /// Initializes the environment for execution of the Sysbench workload. + /// </summary> + protected override async Task InitializeAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + DependencyPath workloadPackage = await this.GetPackageAsync(this.PackageName, cancellationToken).ConfigureAwait(false); + workloadPackage.ThrowIfNull(this.PackageName); + + DependencyPath package = await this.GetPlatformSpecificPackageAsync(this.PackageName, cancellationToken); + this.packageDirectory = package.Path; + + this.InitializeApiClients(telemetryContext, cancellationToken); + + if (this.IsMultiRoleLayout()) + { + ClientInstance clientInstance = this.GetLayoutClientInstance(); + string layoutIPAddress = clientInstance.IPAddress; + + this.ThrowIfLayoutClientIPAddressNotFound(layoutIPAddress); + this.ThrowIfRoleNotSupported(clientInstance.Role); + } + } + + /// <summary> + /// Initializes API client. + /// </summary> + protected void InitializeApiClients(EventContext telemetryContext, CancellationToken cancellationToken) + { + if (!cancellationToken.IsCancellationRequested) + { + IApiClientManager clientManager = this.Dependencies.GetService<IApiClientManager>(); + + if (!this.IsMultiRoleLayout()) + { + this.ServerIpAddress = IPAddress.Loopback.ToString(); + this.ServerApiClient = clientManager.GetOrCreateApiClient(this.ServerIpAddress, IPAddress.Loopback); + } + else + { + ClientInstance serverInstance = this.GetLayoutClientInstances(ClientRole.Server).First(); + IPAddress.TryParse(serverInstance.IPAddress, out IPAddress serverIPAddress); + + this.ServerIpAddress = serverIPAddress.ToString(); + this.ServerApiClient = clientManager.GetOrCreateApiClient(this.ServerIpAddress, serverIPAddress); + this.RegisterToSendExitNotifications($"{this.TypeName}.ExitNotification", this.ServerApiClient); + } + } + }*/ + /// <summary> /// Installs MySQL /// </summary> @@ -123,13 +196,20 @@ protected override async Task ExecuteAsync(EventContext telemetryContext, Cancel ConfigurationState configurationState = await this.stateManager.GetStateAsync<ConfigurationState>(stateId, cancellationToken) .ConfigureAwait(false); + telemetryContext.AddContext(nameof(configurationState), configurationState); + DependencyPath workloadPackage = await this.GetPackageAsync(this.PackageName, cancellationToken).ConfigureAwait(false); workloadPackage.ThrowIfNull(this.PackageName); DependencyPath package = await this.GetPlatformSpecificPackageAsync(this.PackageName, cancellationToken); this.packageDirectory = package.Path; - telemetryContext.AddContext(nameof(configurationState), configurationState); + /*if (this.WaitForServer) + { + string serverstateId = $"{nameof(MySQLServerConfiguration)}-{this.Action}-action-success"; + ConfigurationState configurationState = await this.stateManager.GetStateAsync<ConfigurationState>(serverstateId, cancellationToken) + .ConfigureAwait(false); + }*/ if (!this.SkipInitialize) { @@ -158,6 +238,11 @@ await this.DistributeMySQLDatabaseAsync(telemetryContext, cancellationToken) await this.SetMySQLGlobalVariableAsync(telemetryContext, cancellationToken) .ConfigureAwait(false); } + else if (this.Action == ConfigurationAction.TruncateDatabase) + { + await this.TruncateMySQLDatabaseAsync(telemetryContext, cancellationToken) + .ConfigureAwait(false); + } } } @@ -254,6 +339,32 @@ private async Task DistributeMySQLDatabaseAsync(EventContext telemetryContext, C } } + private async Task TruncateMySQLDatabaseAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + string arguments = $"{this.packageDirectory}/truncate-tables.py --dbName {this.DatabaseName}"; + + if (this.IsMultiRoleLayout()) + { + ClientInstance instance = this.Layout.GetClientInstance(this.AgentId); + string serverIps = (instance.Role == ClientRole.Server) ? "localhost" : this.GetServerIpAddress(); + arguments += $" --clientIps \"{serverIps}\""; + } + + using (IProcessProxy process = await this.ExecuteCommandAsync( + PythonCommand, + arguments, + Environment.CurrentDirectory, + telemetryContext, + cancellationToken)) + { + if (!cancellationToken.IsCancellationRequested) + { + await this.LogProcessDetailsAsync(process, telemetryContext, "MySQLServerConfiguration", logToFile: true); + process.ThrowIfDependencyInstallationFailed(process.StandardError.ToString()); + } + } + } + private async Task<string> GetMySQLInnodbDirectoriesAsync(CancellationToken cancellationToken) { string diskPaths = string.Empty; @@ -354,6 +465,11 @@ internal class ConfigurationAction /// </summary> public const string DistributeDatabase = nameof(DistributeDatabase); + /// <summary> + /// Truncates all tables existing in database + /// </summary> + public const string TruncateDatabase = nameof(TruncateDatabase); + } internal class ConfigurationState diff --git a/src/VirtualClient/VirtualClient.Main/profiles/COMPETE-PERF-MYSQL-SYSBENCH-OLTP.json b/src/VirtualClient/VirtualClient.Main/profiles/COMPETE-PERF-MYSQL-SYSBENCH-OLTP.json new file mode 100644 index 0000000000..392eb78d54 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Main/profiles/COMPETE-PERF-MYSQL-SYSBENCH-OLTP.json @@ -0,0 +1,220 @@ +{ + "Description": "Sysbench OLTP MySQL Database Server Performance Workload for compete", + "MinimumExecutionInterval": "00:01:00", + "Metadata": { + "RecommendedMinimumExecutionTime": "04:00:00", + "SupportedPlatforms": "linux-x64,linux-arm64", + "SupportedOperatingSystems": "Debian,Ubuntu" + }, + "Parameters": { + "DatabaseName": "sbtest", + "DatabaseScenario": "Configure", // should use Configure for configuration of below variables + "DiskFilter": "osdisk:false&sizegreaterthan:256g", + "Duration": "00:05:00", + "RecordCount": 40000000, //--table-size + "TableCount": 40, //default value to be updated + "Threads": "{LogicalCoreCount}" + }, + "Actions": [ + { + "Type": "SysbenchServerExecutor", + "Parameters": { + "Scenario": "ExecuteServer", + "PackageName": "sysbench", + "Threads": "$.Parameters.Threads", + "RecordCount": "$.Parameters.RecordCount", + "TableCount": "$.Parameters.TableCount", + "Role": "Server" + } + }, + { + "Type": "SysbenchClientExecutor", + "Parameters": { + "Scenario": "TruncateMySQLDatabaseTables", // or may be drop all the tables in database. // do not drop all tables, just clean up rows from the tables preserving schema on different disks( from dependencies). + "Action": "TruncateDatabase", + "DatabaseSystem": "MySQL", + "Benchmark": "OLTP", + "DatabaseName": "$.Parameters.DatabaseName", + // "WaitForServer": true, + "PackageName": "mysql-server", + "Role": "Client" + } + }, + { + "Type": "SysbenchClientExecutor", //This is the exact place where complete database is getting created with many records/rows. This has to be moved to actions. + "Parameters": { + "Scenario": "PopulateMySQLDatabase", + "Action": "PopulateDatabase", + "DatabaseSystem": "MySQL", + "Benchmark": "OLTP", + "DatabaseName": "$.Parameters.DatabaseName", + "DatabaseScenario": "$.Parameters.DatabaseScenario", + "PackageName": "sysbench", + "Threads": "$.Parameters.Threads", + "RecordCount": "$.Parameters.RecordCount", + "TableCount": "$.Parameters.TableCount", + // "WaitForServer": true, + "Role": "Client" + } + }, + { + "Type": "SysbenchClientExecutor", + "Parameters": { + "Scenario": "oltp_read_write", + "Action": "RunWorkload", + "DatabaseSystem": "MySQL", + "Benchmark": "OLTP", + "DatabaseName": "$.Parameters.DatabaseName", + "DatabaseScenario": "$.Parameters.DatabaseScenario", + "Duration": "$.Parameters.Duration", + "Workload": "oltp_read_write", + "PackageName": "sysbench", + "Threads": "$.Parameters.Threads", + "RecordCount": "$.Parameters.RecordCount", + "TableCount": "$.Parameters.TableCount", + "Role": "Client" + } + }, + { + "Type": "SysbenchClientExecutor", + "Parameters": { + "Scenario": "TruncateMySQLDatabaseTables", + "Action": "TruncateDatabase", + "DatabaseSystem": "MySQL", + "Benchmark": "OLTP", + "DatabaseName": "$.Parameters.DatabaseName", + "PackageName": "mysql-server", + "Role": "Client" + } + }, + { + "Type": "WaitExecutor", + "Parameters": { + "Scenario": "WaitForTimeProvided", + "Duration": "00:02:00" + } + } + ], + "Dependencies": [ + { + "Type": "FormatDisks", + "Parameters": { + "Scenario": "FormatDisks", + "Role": "Server" + } + }, + { + "Type": "MountDisks", + "Parameters": { + "Scenario": "CreateMountPoints", + "MountLocation": "Root", + "Role": "Server" + } + }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "DownloadMySqlServerPackage", + "BlobContainer": "packages", + "BlobName": "mysql-server-8.0.36-v1.zip", //updated version with addition of the truncate.py file + "PackageName": "mysql-server", + "Extract": true + // "Role": "Server" let's install it both on server and client for truncating tables. + } + }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "DownloadSysbenchPackage", + "BlobContainer": "packages", + "BlobName": "sysbench-1.0.20.rev2.zip", // updated version with change in populate-database.py file + "PackageName": "sysbench", + "Extract": true + } + }, + { + "Type": "LinuxPackageInstallation", + "Parameters": { + "Scenario": "InstallLinuxPackages", + "Packages": "python3" + } + }, + { + "Type": "MySQLServerInstallation", + "Parameters": { + "Scenario": "InstallMySQLServer", + "Action": "InstallServer", + "Benchmark": "OLTP", + "PackageName": "mysql-server" + // "Role": "Server" //needs to be installed at client side too for truncating tables. + } + }, + { + "Type": "MySQLServerConfiguration", + "Parameters": { + "Scenario": "ConfigureMySQLServer", + "Action": "ConfigureServer", + "Benchmark": "OLTP", + "DiskFilter": "$.Parameters.DiskFilter", + "PackageName": "mysql-server", + "Role": "Server" + } + }, + { + "Type": "MySQLServerConfiguration", + "Parameters": { + "Scenario": "CreateMySQLDatabase", + "Action": "CreateDatabase", + "Benchmark": "OLTP", + "DatabaseName": "$.Parameters.DatabaseName", + "PackageName": "mysql-server", + "Role": "Server" + } + }, + { + "Type": "MySQLServerConfiguration", + "Parameters": { + "Scenario": "SetMySQLGlobalVariables", + "Action": "SetGlobalVariables", + "Benchmark": "OLTP", + "DiskFilter": "$.Parameters.DiskFilter", + "Variables": "MAX_PREPARED_STMT_COUNT=1000000;MAX_CONNECTIONS=1000000", + "PackageName": "mysql-server", + "Role": "Server" + } + }, + { + "Type": "SysbenchConfiguration", // Need not move because here actual database is not getting created. It's just creating all tables and after this next step distributes the data in different disks based on disk filters. + "Parameters": { + "Scenario": "PopulateMySQLDatabase", + "DatabaseSystem": "MySQL", + "Benchmark": "OLTP", + "DatabaseName": "$.Parameters.DatabaseName", + "RecordCount": 1, + "DatabaseScenario": "$.Parameters.DatabaseScenario", + "PackageName": "sysbench", + "Threads": "$.Parameters.Threads", + "TableCount": "$.Parameters.TableCount", + "Role": "Server" + } + }, + { + "Type": "MySQLServerConfiguration", + "Parameters": { + "Scenario": "DistributeMySQLDatabase", + "Action": "DistributeDatabase", + "DiskFilter": "$.Parameters.DiskFilter", + "DatabaseName": "$.Parameters.DatabaseName", + "PackageName": "mysql-server", + "Role": "Server" + } + }, + { + "Type": "ApiServer", + "Parameters": { + "Scenario": "StartAPIServer", + "Role": "Server" + } + } + ] +} \ No newline at end of file From 4f1f079b3b9b9e2e343fe8e45e6026dc1dd4227c Mon Sep 17 00:00:00 2001 From: nikhitha malkapuram <nmalkapuram@microsoft.com> Date: Fri, 15 Nov 2024 07:20:56 +0530 Subject: [PATCH 02/11] removing unnecessary files --- .../Sysbench/original-populate-databse.py | 86 ------- .../populate-database-oldnewbalance.py | 102 -------- .../Sysbench/populate-database.py | 99 -------- .../Sysbench/truncate-tables.py | 35 --- .../COMPETE-PERF-MYSQL-SYSBENCH-OLTP.json | 220 ------------------ 5 files changed, 542 deletions(-) delete mode 100644 src/VirtualClient/VirtualClient.Actions/Sysbench/original-populate-databse.py delete mode 100644 src/VirtualClient/VirtualClient.Actions/Sysbench/populate-database-oldnewbalance.py delete mode 100644 src/VirtualClient/VirtualClient.Actions/Sysbench/populate-database.py delete mode 100644 src/VirtualClient/VirtualClient.Actions/Sysbench/truncate-tables.py delete mode 100644 src/VirtualClient/VirtualClient.Main/profiles/COMPETE-PERF-MYSQL-SYSBENCH-OLTP.json diff --git a/src/VirtualClient/VirtualClient.Actions/Sysbench/original-populate-databse.py b/src/VirtualClient/VirtualClient.Actions/Sysbench/original-populate-databse.py deleted file mode 100644 index 7a33d07ac6..0000000000 --- a/src/VirtualClient/VirtualClient.Actions/Sysbench/original-populate-databse.py +++ /dev/null @@ -1,86 +0,0 @@ -import subprocess, argparse, os - -parser = argparse.ArgumentParser() - -parser.add_argument('-n', '--dbName', type=str, help='Database Name', required=True) -parser.add_argument('-y', '--databaseSystem', type=str, help='Database Type', required=False) -parser.add_argument('-b', '--benchmark', type=str, help="Benchmark Name", required=True) -parser.add_argument('-t', '--tableCount', type=str, help='Number of Tables', required=True) -parser.add_argument('-r', '--recordCount', type=str, help='Number of Records', required=False) -parser.add_argument('-u', '--warehouses', type=str, help='Warehouse Count', required=False) -parser.add_argument('-e', '--threadCount', type=str, help="Number of Threads", required=True) -parser.add_argument('--password', type=str, help="PostgreSQL Password", required=False) - -args = parser.parse_args() -dbName = args.dbName -databaseSystem = args.databaseSystem -benchmark = args.benchmark -warehouses = args.warehouses -tableCount = args.tableCount -recordCount = args.recordCount -threadCount = args.threadCount -password = args.password - -if databaseSystem == "MySQL": - if benchmark == "TPCC": - subprocess.run(f'sudo src/sysbench tpcc --tables={tableCount} --scale={warehouses} --threads={threadCount} --mysql-db={dbName} --use_fk=0 prepare', shell=True, check=True) - if int(warehouses) == 1: - for i in range(1,int(tableCount)+1): - table = str(i) - # drop idxs - subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "DROP INDEX idx_customer{i} ON customer{i};"', shell=True, check=True) - subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "DROP INDEX idx_orders{i} ON orders{i};"', shell=True, check=True) - subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "DROP INDEX fkey_stock_2{i} ON stock{i};"', shell=True, check=True) - subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "DROP INDEX fkey_order_line_2{i} ON order_line{i};"', shell=True, check=True) - subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "DROP INDEX fkey_history_1{i} ON history{i};"', shell=True, check=True) - subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "DROP INDEX fkey_history_2{i} ON history{i};"', shell=True, check=True) - - # truncate, to make distributing faster - subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE warehouse{i};"', shell=True, check=True) - subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE district{i};"', shell=True, check=True) - subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE customer{i};"', shell=True, check=True) - subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE history{i};"', shell=True, check=True) - subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE orders{i};"', shell=True, check=True) - subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE new_orders{i};"', shell=True, check=True) - subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE order_line{i};"', shell=True, check=True) - subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE stock{i};"', shell=True, check=True) - subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE item{i};"', shell=True, check=True) - else: - subprocess.run(f'sudo src/sysbench oltp_common --tables={tableCount} --table-size={recordCount} --threads={threadCount} --mysql-db={dbName} prepare', shell=True, check=True) - if int(recordCount) == 1: - for i in range(1,int(tableCount)+1): - table = str(i) - subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "DROP INDEX k_{i} ON sbtest{i};"', shell=True, check=True) -elif databaseSystem == "PostgreSQL": - os.environ['PGPASSWORD'] = password - if benchmark == "TPCC": - subprocess.run(f'sudo src/sysbench tpcc --db-driver=pgsql --tables={tableCount} --scale={warehouses} --threads={threadCount} --pgsql-user=postgres --pgsql-password={password} --pgsql-db={dbName} --use_fk=0 prepare', shell=True, check=True) - if int(warehouses) == 1: - for i in range(1,int(tableCount)+1): - table = str(i) - # drop idxs - subprocess.run(f'psql -U postgres -d {dbName} -c "DROP INDEX IF EXISTS idx_customer{i};"', shell=True, check=True) - subprocess.run(f'psql -U postgres -d {dbName} -c "DROP INDEX IF EXISTS idx_orders{i};"', shell=True, check=True) - subprocess.run(f'psql -U postgres -d {dbName} -c "DROP INDEX IF EXISTS fkey_stock_2{i};"', shell=True, check=True) - subprocess.run(f'psql -U postgres -d {dbName} -c "DROP INDEX IF EXISTS fkey_order_line_2{i};"', shell=True, check=True) - subprocess.run(f'psql -U postgres -d {dbName} -c "DROP INDEX IF EXISTS fkey_history_1{i};"', shell=True, check=True) - subprocess.run(f'psql -U postgres -d {dbName} -c "DROP INDEX IF EXISTS fkey_history_2{i};"', shell=True, check=True) - - # truncate, to make distributing faster - subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE warehouse{i};"', shell=True, check=True) - subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE district{i};"', shell=True, check=True) - subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE customer{i};"', shell=True, check=True) - subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE history{i};"', shell=True, check=True) - subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE orders{i};"', shell=True, check=True) - subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE new_orders{i};"', shell=True, check=True) - subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE order_line{i};"', shell=True, check=True) - subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE stock{i};"', shell=True, check=True) - subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE item{i};"', shell=True, check=True) - else: - subprocess.run(f'sudo src/sysbench oltp_common --db-driver=pgsql --tables={tableCount} --table-size={recordCount} --threads={threadCount} --pgsql-user=postgres --pgsql-password={password} --pgsql-db={dbName} prepare', shell=True, check=True) - if int(recordCount) == 1: - for i in range(1,int(tableCount)+1): - table = str(i) - subprocess.run(f'psql -U postgres -d {dbName} -c "DROP INDEX IF EXISTS k_{i};"', shell=True, check=True) -else: - parser.error("You are running on a database system type that has not been onboarded to Virtual Client. Available options are: MySQL, PostgreSQL") \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Actions/Sysbench/populate-database-oldnewbalance.py b/src/VirtualClient/VirtualClient.Actions/Sysbench/populate-database-oldnewbalance.py deleted file mode 100644 index 107c394821..0000000000 --- a/src/VirtualClient/VirtualClient.Actions/Sysbench/populate-database-oldnewbalance.py +++ /dev/null @@ -1,102 +0,0 @@ -import subprocess -import argparse -import os - -# Define arguments -parser = argparse.ArgumentParser() - -parser.add_argument('-n', '--dbName', type=str, help='Database Name', required=True) -parser.add_argument('-y', '--databaseSystem', type=str, help='Database Type (MySQL/PostgreSQL)', required=False) -parser.add_argument('-b', '--benchmark', type=str, help="Benchmark Name", required=True) -parser.add_argument('-t', '--tableCount', type=str, help='Number of Tables', required=True) -parser.add_argument('-r', '--recordCount', type=str, help='Number of Records', required=False) -parser.add_argument('-u', '--warehouses', type=str, help='Warehouse Count', required=False) -parser.add_argument('-e', '--threadCount', type=str, help="Number of Threads", required=True) -parser.add_argument('--password', type=str, help="PostgreSQL Password", required=False) -parser.add_argument('--host', type=str, help="Database Server Host IP Address", required=False) - -# Parse arguments -args = parser.parse_args() -dbName = args.dbName -databaseSystem = args.databaseSystem -benchmark = args.benchmark -warehouses = args.warehouses -tableCount = args.tableCount -recordCount = args.recordCount -threadCount = args.threadCount -password = args.password -host = args.host # Host can be None if not provided - -# Function to add host if provided -def add_host_to_command(command_base, host): - if host: - return f"{command_base} --mysql-host={host}" if "mysql" in command_base else f"{command_base} --pgsql-host={host}" - return command_base - -# MySQL handling -if databaseSystem == "MySQL": - if benchmark == "TPCC": - command_base = f'sudo src/sysbench tpcc --tables={tableCount} --scale={warehouses} --threads={threadCount} --mysql-db={dbName} --use_fk=0 prepare' - command = add_host_to_command(command_base, host) - subprocess.run(command, shell=True, check=True) - - if int(warehouses) == 1: - for i in range(1, int(tableCount) + 1): - table = str(i) - # drop indexes - for index in ["idx_customer", "idx_orders", "fkey_stock_2", "fkey_order_line_2", "fkey_history_1", "fkey_history_2"]: - drop_index_command = f'mysql -u {dbName} -p -e "DROP INDEX {index}{i} ON {index.split("_")[1]}{i};"' - drop_index_command = add_host_to_command(drop_index_command, host) - subprocess.run(drop_index_command, shell=True, check=True) - - # truncate tables - for table in ["warehouse", "district", "customer", "history", "orders", "new_orders", "order_line", "stock", "item"]: - truncate_command = f'mysql -u {dbName} -p -e "TRUNCATE TABLE {table}{i};"' - truncate_command = add_host_to_command(truncate_command, host) - subprocess.run(truncate_command, shell=True, check=True) - else: - command_base = f'sudo src/sysbench oltp_common --tables={tableCount} --table-size={recordCount} --threads={threadCount} --mysql-db={dbName} prepare' - command = add_host_to_command(command_base, host) - subprocess.run(command, shell=True, check=True) - - if int(recordCount) == 1: - for i in range(1, int(tableCount) + 1): - drop_index_command = f'mysql -u {dbName} -p -e "DROP INDEX k_{i} ON sbtest{i};"' - drop_index_command = add_host_to_command(drop_index_command, host) - subprocess.run(drop_index_command, shell=True, check=True) - -# PostgreSQL handling -elif databaseSystem == "PostgreSQL": - os.environ['PGPASSWORD'] = password - if benchmark == "TPCC": - command_base = f'sudo src/sysbench tpcc --db-driver=pgsql --tables={tableCount} --scale={warehouses} --threads={threadCount} --pgsql-user=postgres --pgsql-password={password} --pgsql-db={dbName} --use_fk=0 prepare' - command = add_host_to_command(command_base, host) - subprocess.run(command, shell=True, check=True) - - if int(warehouses) == 1: - for i in range(1, int(tableCount) + 1): - table = str(i) - # drop indexes - for index in ["idx_customer", "idx_orders", "fkey_stock_2", "fkey_order_line_2", "fkey_history_1", "fkey_history_2"]: - drop_index_command = f'psql -U postgres -d {dbName} -c "DROP INDEX IF EXISTS {index}{i};"' - drop_index_command = add_host_to_command(drop_index_command, host) - subprocess.run(drop_index_command, shell=True, check=True) - - # truncate tables - for table in ["warehouse", "district", "customer", "history", "orders", "new_orders", "order_line", "stock", "item"]: - truncate_command = f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE {table}{i};"' - truncate_command = add_host_to_command(truncate_command, host) - subprocess.run(truncate_command, shell=True, check=True) - else: - command_base = f'sudo src/sysbench oltp_common --db-driver=pgsql --tables={tableCount} --table-size={recordCount} --threads={threadCount} --pgsql-user=postgres --pgsql-password={password} --pgsql-db={dbName} prepare' - command = add_host_to_command(command_base, host) - subprocess.run(command, shell=True, check=True) - - if int(recordCount) == 1: - for i in range(1, int(tableCount) + 1): - drop_index_command = f'psql -U postgres -d {dbName} -c "DROP INDEX IF EXISTS k_{i};"' - drop_index_command = add_host_to_command(drop_index_command, host) - subprocess.run(drop_index_command, shell=True, check=True) - -else: - parser.error("You are running on a database system type that has not been onboarded to Virtual Client. Available options are: MySQL, PostgreSQL") diff --git a/src/VirtualClient/VirtualClient.Actions/Sysbench/populate-database.py b/src/VirtualClient/VirtualClient.Actions/Sysbench/populate-database.py deleted file mode 100644 index c4f36de5d1..0000000000 --- a/src/VirtualClient/VirtualClient.Actions/Sysbench/populate-database.py +++ /dev/null @@ -1,99 +0,0 @@ -import subprocess, argparse, os - -parser = argparse.ArgumentParser() - -parser.add_argument('-n', '--dbName', type=str, help='Database Name', required=True) -parser.add_argument('-y', '--databaseSystem', type=str, help='Database Type', required=False) -parser.add_argument('-b', '--benchmark', type=str, help="Benchmark Name", required=True) -parser.add_argument('-t', '--tableCount', type=str, help='Number of Tables', required=True) -parser.add_argument('-r', '--recordCount', type=str, help='Number of Records', required=False) -parser.add_argument('-u', '--warehouses', type=str, help='Warehouse Count', required=False) -parser.add_argument('-e', '--threadCount', type=str, help="Number of Threads", required=True) -parser.add_argument('--password', type=str, help="PostgreSQL Password", required=False) -parser.add_argument('--host', type=str, help="Database Server Host IP Address", required=False) - - -args = parser.parse_args() -dbName = args.dbName -databaseSystem = args.databaseSystem -benchmark = args.benchmark -warehouses = args.warehouses -tableCount = args.tableCount -recordCount = args.recordCount -threadCount = args.threadCount -password = args.password -host = args.host - -def add_host_if_needed(command_base, host, benchmark): - if host and benchmark != "TPCC": - if "sysbench" not in command_base: - return command_base.replace('-u', f'-h {host} -u') - else: - return f"{command_base} --mysql-host={host}" - -if databaseSystem == "MySQL": - if benchmark == "TPCC": - subprocess.run(f'sudo src/sysbench tpcc --tables={tableCount} --scale={warehouses} --threads={threadCount} --mysql-db={dbName} --use_fk=0 prepare', shell=True, check=True) - if int(warehouses) == 1: - for i in range(1,int(tableCount)+1): - table = str(i) - # drop idxs - subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "DROP INDEX idx_customer{i} ON customer{i};"', shell=True, check=True) - subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "DROP INDEX idx_orders{i} ON orders{i};"', shell=True, check=True) - subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "DROP INDEX fkey_stock_2{i} ON stock{i};"', shell=True, check=True) - subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "DROP INDEX fkey_order_line_2{i} ON order_line{i};"', shell=True, check=True) - subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "DROP INDEX fkey_history_1{i} ON history{i};"', shell=True, check=True) - subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "DROP INDEX fkey_history_2{i} ON history{i};"', shell=True, check=True) - - # truncate, to make distributing faster - subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE warehouse{i};"', shell=True, check=True) - subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE district{i};"', shell=True, check=True) - subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE customer{i};"', shell=True, check=True) - subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE history{i};"', shell=True, check=True) - subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE orders{i};"', shell=True, check=True) - subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE new_orders{i};"', shell=True, check=True) - subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE order_line{i};"', shell=True, check=True) - subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE stock{i};"', shell=True, check=True) - subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE item{i};"', shell=True, check=True) - else: - command_base = f'sudo src/sysbench oltp_common --tables={tableCount} --table-size={recordCount} --threads={threadCount} --mysql-db={dbName} prepare' - command = add_host_if_needed(command_base, host, benchmark) - subprocess.run(command, shell=True, check=True) - if int(recordCount) == 1: - for i in range(1, int(tableCount) + 1): - drop_index_command = f'sudo mysql -u {dbName} {dbName} -e "DROP INDEX k_{i} ON sbtest{i};"' - drop_index_command = add_host_if_needed(drop_index_command, host, benchmark) - subprocess.run(drop_index_command, shell=True, check=True) -elif databaseSystem == "PostgreSQL": - os.environ['PGPASSWORD'] = password - if benchmark == "TPCC": - subprocess.run(f'sudo src/sysbench tpcc --db-driver=pgsql --tables={tableCount} --scale={warehouses} --threads={threadCount} --pgsql-user=postgres --pgsql-password={password} --pgsql-db={dbName} --use_fk=0 prepare', shell=True, check=True) - if int(warehouses) == 1: - for i in range(1,int(tableCount)+1): - table = str(i) - # drop idxs - subprocess.run(f'psql -U postgres -d {dbName} -c "DROP INDEX IF EXISTS idx_customer{i};"', shell=True, check=True) - subprocess.run(f'psql -U postgres -d {dbName} -c "DROP INDEX IF EXISTS idx_orders{i};"', shell=True, check=True) - subprocess.run(f'psql -U postgres -d {dbName} -c "DROP INDEX IF EXISTS fkey_stock_2{i};"', shell=True, check=True) - subprocess.run(f'psql -U postgres -d {dbName} -c "DROP INDEX IF EXISTS fkey_order_line_2{i};"', shell=True, check=True) - subprocess.run(f'psql -U postgres -d {dbName} -c "DROP INDEX IF EXISTS fkey_history_1{i};"', shell=True, check=True) - subprocess.run(f'psql -U postgres -d {dbName} -c "DROP INDEX IF EXISTS fkey_history_2{i};"', shell=True, check=True) - - # truncate, to make distributing faster - subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE warehouse{i};"', shell=True, check=True) - subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE district{i};"', shell=True, check=True) - subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE customer{i};"', shell=True, check=True) - subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE history{i};"', shell=True, check=True) - subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE orders{i};"', shell=True, check=True) - subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE new_orders{i};"', shell=True, check=True) - subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE order_line{i};"', shell=True, check=True) - subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE stock{i};"', shell=True, check=True) - subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE item{i};"', shell=True, check=True) - else: - subprocess.run(f'sudo src/sysbench oltp_common --db-driver=pgsql --tables={tableCount} --table-size={recordCount} --threads={threadCount} --pgsql-user=postgres --pgsql-password={password} --pgsql-db={dbName} prepare', shell=True, check=True) - if int(recordCount) == 1: - for i in range(1,int(tableCount)+1): - table = str(i) - subprocess.run(f'psql -U postgres -d {dbName} -c "DROP INDEX IF EXISTS k_{i};"', shell=True, check=True) -else: - parser.error("You are running on a database system type that has not been onboarded to Virtual Client. Available options are: MySQL, PostgreSQL") \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Actions/Sysbench/truncate-tables.py b/src/VirtualClient/VirtualClient.Actions/Sysbench/truncate-tables.py deleted file mode 100644 index 91b455c01b..0000000000 --- a/src/VirtualClient/VirtualClient.Actions/Sysbench/truncate-tables.py +++ /dev/null @@ -1,35 +0,0 @@ -import subprocess -import argparse - -# Set up argument parsing -parser = argparse.ArgumentParser() - -parser.add_argument('--dbName', type=str, help='Database Name', required=True) -parser.add_argument('--clientIps', type=str, help='Client IP Address (separate multiple with semicolons)', required=False) - -args = parser.parse_args() -dbName = args.dbName -clientIps = args.clientIps - - # Function to truncate all tables in the specified database -def truncate_tables(db_user, host): - # Get all table names from the database - output = subprocess.run(f'sudo mysql -u {db_user} -h {host} {dbName} -e "SHOW TABLES;"', shell=True, check=True, capture_output=True, text=True) - - # Extract table names from the output - tables = [line.strip() for line in output.stdout.strip().split('\n') if line.startswith('sbtest')] - - # Truncate each table - for table in tables: - subprocess.run(f'sudo mysql -u {db_user} -h {host} {dbName} -e "TRUNCATE TABLE {table};"', shell=True, check=True) - -# Multi-VM: If client IPs are specified, truncate tables for each client VM IP -if clientIps: - clientIps = clientIps.split(';') - clientIps = list(filter(None, clientIps)) - - for clientIp in clientIps: - truncate_tables(db_user=dbName, host=clientIp) -else: - truncate_tables(db_user=dbName, host='localhost') -print(f"Successfully truncated tables in database '{dbName}' for all specified hosts.") \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Main/profiles/COMPETE-PERF-MYSQL-SYSBENCH-OLTP.json b/src/VirtualClient/VirtualClient.Main/profiles/COMPETE-PERF-MYSQL-SYSBENCH-OLTP.json deleted file mode 100644 index 392eb78d54..0000000000 --- a/src/VirtualClient/VirtualClient.Main/profiles/COMPETE-PERF-MYSQL-SYSBENCH-OLTP.json +++ /dev/null @@ -1,220 +0,0 @@ -{ - "Description": "Sysbench OLTP MySQL Database Server Performance Workload for compete", - "MinimumExecutionInterval": "00:01:00", - "Metadata": { - "RecommendedMinimumExecutionTime": "04:00:00", - "SupportedPlatforms": "linux-x64,linux-arm64", - "SupportedOperatingSystems": "Debian,Ubuntu" - }, - "Parameters": { - "DatabaseName": "sbtest", - "DatabaseScenario": "Configure", // should use Configure for configuration of below variables - "DiskFilter": "osdisk:false&sizegreaterthan:256g", - "Duration": "00:05:00", - "RecordCount": 40000000, //--table-size - "TableCount": 40, //default value to be updated - "Threads": "{LogicalCoreCount}" - }, - "Actions": [ - { - "Type": "SysbenchServerExecutor", - "Parameters": { - "Scenario": "ExecuteServer", - "PackageName": "sysbench", - "Threads": "$.Parameters.Threads", - "RecordCount": "$.Parameters.RecordCount", - "TableCount": "$.Parameters.TableCount", - "Role": "Server" - } - }, - { - "Type": "SysbenchClientExecutor", - "Parameters": { - "Scenario": "TruncateMySQLDatabaseTables", // or may be drop all the tables in database. // do not drop all tables, just clean up rows from the tables preserving schema on different disks( from dependencies). - "Action": "TruncateDatabase", - "DatabaseSystem": "MySQL", - "Benchmark": "OLTP", - "DatabaseName": "$.Parameters.DatabaseName", - // "WaitForServer": true, - "PackageName": "mysql-server", - "Role": "Client" - } - }, - { - "Type": "SysbenchClientExecutor", //This is the exact place where complete database is getting created with many records/rows. This has to be moved to actions. - "Parameters": { - "Scenario": "PopulateMySQLDatabase", - "Action": "PopulateDatabase", - "DatabaseSystem": "MySQL", - "Benchmark": "OLTP", - "DatabaseName": "$.Parameters.DatabaseName", - "DatabaseScenario": "$.Parameters.DatabaseScenario", - "PackageName": "sysbench", - "Threads": "$.Parameters.Threads", - "RecordCount": "$.Parameters.RecordCount", - "TableCount": "$.Parameters.TableCount", - // "WaitForServer": true, - "Role": "Client" - } - }, - { - "Type": "SysbenchClientExecutor", - "Parameters": { - "Scenario": "oltp_read_write", - "Action": "RunWorkload", - "DatabaseSystem": "MySQL", - "Benchmark": "OLTP", - "DatabaseName": "$.Parameters.DatabaseName", - "DatabaseScenario": "$.Parameters.DatabaseScenario", - "Duration": "$.Parameters.Duration", - "Workload": "oltp_read_write", - "PackageName": "sysbench", - "Threads": "$.Parameters.Threads", - "RecordCount": "$.Parameters.RecordCount", - "TableCount": "$.Parameters.TableCount", - "Role": "Client" - } - }, - { - "Type": "SysbenchClientExecutor", - "Parameters": { - "Scenario": "TruncateMySQLDatabaseTables", - "Action": "TruncateDatabase", - "DatabaseSystem": "MySQL", - "Benchmark": "OLTP", - "DatabaseName": "$.Parameters.DatabaseName", - "PackageName": "mysql-server", - "Role": "Client" - } - }, - { - "Type": "WaitExecutor", - "Parameters": { - "Scenario": "WaitForTimeProvided", - "Duration": "00:02:00" - } - } - ], - "Dependencies": [ - { - "Type": "FormatDisks", - "Parameters": { - "Scenario": "FormatDisks", - "Role": "Server" - } - }, - { - "Type": "MountDisks", - "Parameters": { - "Scenario": "CreateMountPoints", - "MountLocation": "Root", - "Role": "Server" - } - }, - { - "Type": "DependencyPackageInstallation", - "Parameters": { - "Scenario": "DownloadMySqlServerPackage", - "BlobContainer": "packages", - "BlobName": "mysql-server-8.0.36-v1.zip", //updated version with addition of the truncate.py file - "PackageName": "mysql-server", - "Extract": true - // "Role": "Server" let's install it both on server and client for truncating tables. - } - }, - { - "Type": "DependencyPackageInstallation", - "Parameters": { - "Scenario": "DownloadSysbenchPackage", - "BlobContainer": "packages", - "BlobName": "sysbench-1.0.20.rev2.zip", // updated version with change in populate-database.py file - "PackageName": "sysbench", - "Extract": true - } - }, - { - "Type": "LinuxPackageInstallation", - "Parameters": { - "Scenario": "InstallLinuxPackages", - "Packages": "python3" - } - }, - { - "Type": "MySQLServerInstallation", - "Parameters": { - "Scenario": "InstallMySQLServer", - "Action": "InstallServer", - "Benchmark": "OLTP", - "PackageName": "mysql-server" - // "Role": "Server" //needs to be installed at client side too for truncating tables. - } - }, - { - "Type": "MySQLServerConfiguration", - "Parameters": { - "Scenario": "ConfigureMySQLServer", - "Action": "ConfigureServer", - "Benchmark": "OLTP", - "DiskFilter": "$.Parameters.DiskFilter", - "PackageName": "mysql-server", - "Role": "Server" - } - }, - { - "Type": "MySQLServerConfiguration", - "Parameters": { - "Scenario": "CreateMySQLDatabase", - "Action": "CreateDatabase", - "Benchmark": "OLTP", - "DatabaseName": "$.Parameters.DatabaseName", - "PackageName": "mysql-server", - "Role": "Server" - } - }, - { - "Type": "MySQLServerConfiguration", - "Parameters": { - "Scenario": "SetMySQLGlobalVariables", - "Action": "SetGlobalVariables", - "Benchmark": "OLTP", - "DiskFilter": "$.Parameters.DiskFilter", - "Variables": "MAX_PREPARED_STMT_COUNT=1000000;MAX_CONNECTIONS=1000000", - "PackageName": "mysql-server", - "Role": "Server" - } - }, - { - "Type": "SysbenchConfiguration", // Need not move because here actual database is not getting created. It's just creating all tables and after this next step distributes the data in different disks based on disk filters. - "Parameters": { - "Scenario": "PopulateMySQLDatabase", - "DatabaseSystem": "MySQL", - "Benchmark": "OLTP", - "DatabaseName": "$.Parameters.DatabaseName", - "RecordCount": 1, - "DatabaseScenario": "$.Parameters.DatabaseScenario", - "PackageName": "sysbench", - "Threads": "$.Parameters.Threads", - "TableCount": "$.Parameters.TableCount", - "Role": "Server" - } - }, - { - "Type": "MySQLServerConfiguration", - "Parameters": { - "Scenario": "DistributeMySQLDatabase", - "Action": "DistributeDatabase", - "DiskFilter": "$.Parameters.DiskFilter", - "DatabaseName": "$.Parameters.DatabaseName", - "PackageName": "mysql-server", - "Role": "Server" - } - }, - { - "Type": "ApiServer", - "Parameters": { - "Scenario": "StartAPIServer", - "Role": "Server" - } - } - ] -} \ No newline at end of file From 4fe4cc1688459e59386b30748b2cea9c4b7b2d68 Mon Sep 17 00:00:00 2001 From: nikhitha malkapuram <nmalkapuram@microsoft.com> Date: Fri, 15 Nov 2024 07:26:31 +0530 Subject: [PATCH 03/11] removing unnecessary comments --- .../Sysbench/SysbenchConfiguration.cs | 11 ----------- .../MySqlServer/MySqlServerConfiguration.cs | 11 ----------- 2 files changed, 22 deletions(-) diff --git a/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchConfiguration.cs b/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchConfiguration.cs index a717e00423..b3090ce017 100644 --- a/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchConfiguration.cs +++ b/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchConfiguration.cs @@ -35,17 +35,6 @@ public SysbenchConfiguration(IServiceCollection dependencies, IDictionary<string this.stateManager = this.Dependencies.GetService<IStateManager>(); } -/* /// <summary> - /// The specifed action that controls the execution of the dependency. - /// </summary> - public string WaitForServer - { - get - { - return this.Parameters.GetValue<string>(nameof(this.WaitForServer)); - } - }*/ - /// <summary> /// Executes the workload. /// </summary> diff --git a/src/VirtualClient/VirtualClient.Dependencies/MySqlServer/MySqlServerConfiguration.cs b/src/VirtualClient/VirtualClient.Dependencies/MySqlServer/MySqlServerConfiguration.cs index e14076d9c0..707c6acf11 100644 --- a/src/VirtualClient/VirtualClient.Dependencies/MySqlServer/MySqlServerConfiguration.cs +++ b/src/VirtualClient/VirtualClient.Dependencies/MySqlServer/MySqlServerConfiguration.cs @@ -52,17 +52,6 @@ public string Action } } - /*/// <summary> - /// The specifed action that controls the execution of the dependency. - /// </summary> - public bool WaitForServer - { - get - { - return this.Parameters.GetValue<bool>(nameof(this.WaitForServer), false); - } - }*/ - /// <summary> /// The specifed action that controls the execution of the dependency. /// </summary> From 6f6de38c4fc74b05f94d21eeeced065925216e23 Mon Sep 17 00:00:00 2001 From: nikhitha malkapuram <nmalkapuram@microsoft.com> Date: Mon, 18 Nov 2024 16:01:14 +0530 Subject: [PATCH 04/11] resolving comments --- .../Sysbench/SysbenchClientExecutor.cs | 6 +- .../MySqlServer/MySqlServerConfiguration.cs | 108 +----------------- 2 files changed, 3 insertions(+), 111 deletions(-) diff --git a/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchClientExecutor.cs b/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchClientExecutor.cs index b7b17d1a3a..239e92cc9f 100644 --- a/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchClientExecutor.cs +++ b/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchClientExecutor.cs @@ -215,11 +215,10 @@ private async Task RunOLTPWorkloadAsync(EventContext telemetryContext, Cancellat this.sysbenchLoggingArguments = $"--dbName {this.DatabaseName} --databaseSystem {this.DatabaseSystem} --benchmark {this.Benchmark} --workload {this.Workload} --threadCount {threadCount} --tableCount {tableCount} --recordCount {recordCount} "; this.sysbenchExecutionArguments = this.sysbenchLoggingArguments + $"--hostIpAddress {this.ServerIpAddress} --durationSecs {this.Duration.TotalSeconds} --password {this.SuperUserPassword}"; - string command = "python3"; string script = $"{this.SysbenchPackagePath}/run-workload.py "; using (IProcessProxy process = await this.ExecuteCommandAsync( - command, + PythonCommand, script + this.sysbenchExecutionArguments, this.SysbenchPackagePath, telemetryContext, @@ -244,11 +243,10 @@ private async Task RunTPCCWorkloadAsync(EventContext telemetryContext, Cancellat this.sysbenchLoggingArguments = $"--dbName {this.DatabaseName} --databaseSystem {this.DatabaseSystem} --benchmark {this.Benchmark} --workload tpcc --threadCount {threadCount} --tableCount {tableCount} --warehouses {warehouseCount} "; this.sysbenchExecutionArguments = this.sysbenchLoggingArguments + $"--hostIpAddress {this.ServerIpAddress} --durationSecs {this.Duration.TotalSeconds} --password {this.SuperUserPassword}"; - string command = "python3"; string script = $"{this.SysbenchPackagePath}/run-workload.py "; using (IProcessProxy process = await this.ExecuteCommandAsync( - command, + PythonCommand, script + this.sysbenchExecutionArguments, this.SysbenchPackagePath, telemetryContext, diff --git a/src/VirtualClient/VirtualClient.Dependencies/MySqlServer/MySqlServerConfiguration.cs b/src/VirtualClient/VirtualClient.Dependencies/MySqlServer/MySqlServerConfiguration.cs index 707c6acf11..1331137d65 100644 --- a/src/VirtualClient/VirtualClient.Dependencies/MySqlServer/MySqlServerConfiguration.cs +++ b/src/VirtualClient/VirtualClient.Dependencies/MySqlServer/MySqlServerConfiguration.cs @@ -113,68 +113,6 @@ public bool InMemory /// </summary> protected ISystemManagement SystemManager { get; } - /* - /// <summary> - /// Client used to communicate with the hosted instance of the - /// Virtual Client API at server side. - /// </summary> - public IApiClient ServerApiClient { get; set; }*/ - - /* - /// <summary> - /// Server IpAddress on which MySQL Server runs. - /// </summary> - protected string ServerIpAddress { get; set; } - - /// <summary> - /// Initializes the environment for execution of the Sysbench workload. - /// </summary> - protected override async Task InitializeAsync(EventContext telemetryContext, CancellationToken cancellationToken) - { - DependencyPath workloadPackage = await this.GetPackageAsync(this.PackageName, cancellationToken).ConfigureAwait(false); - workloadPackage.ThrowIfNull(this.PackageName); - - DependencyPath package = await this.GetPlatformSpecificPackageAsync(this.PackageName, cancellationToken); - this.packageDirectory = package.Path; - - this.InitializeApiClients(telemetryContext, cancellationToken); - - if (this.IsMultiRoleLayout()) - { - ClientInstance clientInstance = this.GetLayoutClientInstance(); - string layoutIPAddress = clientInstance.IPAddress; - - this.ThrowIfLayoutClientIPAddressNotFound(layoutIPAddress); - this.ThrowIfRoleNotSupported(clientInstance.Role); - } - } - - /// <summary> - /// Initializes API client. - /// </summary> - protected void InitializeApiClients(EventContext telemetryContext, CancellationToken cancellationToken) - { - if (!cancellationToken.IsCancellationRequested) - { - IApiClientManager clientManager = this.Dependencies.GetService<IApiClientManager>(); - - if (!this.IsMultiRoleLayout()) - { - this.ServerIpAddress = IPAddress.Loopback.ToString(); - this.ServerApiClient = clientManager.GetOrCreateApiClient(this.ServerIpAddress, IPAddress.Loopback); - } - else - { - ClientInstance serverInstance = this.GetLayoutClientInstances(ClientRole.Server).First(); - IPAddress.TryParse(serverInstance.IPAddress, out IPAddress serverIPAddress); - - this.ServerIpAddress = serverIPAddress.ToString(); - this.ServerApiClient = clientManager.GetOrCreateApiClient(this.ServerIpAddress, serverIPAddress); - this.RegisterToSendExitNotifications($"{this.TypeName}.ExitNotification", this.ServerApiClient); - } - } - }*/ - /// <summary> /// Installs MySQL /// </summary> @@ -193,13 +131,6 @@ protected override async Task ExecuteAsync(EventContext telemetryContext, Cancel DependencyPath package = await this.GetPlatformSpecificPackageAsync(this.PackageName, cancellationToken); this.packageDirectory = package.Path; - /*if (this.WaitForServer) - { - string serverstateId = $"{nameof(MySQLServerConfiguration)}-{this.Action}-action-success"; - ConfigurationState configurationState = await this.stateManager.GetStateAsync<ConfigurationState>(serverstateId, cancellationToken) - .ConfigureAwait(false); - }*/ - if (!this.SkipInitialize) { if (configurationState == null) @@ -227,11 +158,6 @@ await this.DistributeMySQLDatabaseAsync(telemetryContext, cancellationToken) await this.SetMySQLGlobalVariableAsync(telemetryContext, cancellationToken) .ConfigureAwait(false); } - else if (this.Action == ConfigurationAction.TruncateDatabase) - { - await this.TruncateMySQLDatabaseAsync(telemetryContext, cancellationToken) - .ConfigureAwait(false); - } } } @@ -328,32 +254,6 @@ private async Task DistributeMySQLDatabaseAsync(EventContext telemetryContext, C } } - private async Task TruncateMySQLDatabaseAsync(EventContext telemetryContext, CancellationToken cancellationToken) - { - string arguments = $"{this.packageDirectory}/truncate-tables.py --dbName {this.DatabaseName}"; - - if (this.IsMultiRoleLayout()) - { - ClientInstance instance = this.Layout.GetClientInstance(this.AgentId); - string serverIps = (instance.Role == ClientRole.Server) ? "localhost" : this.GetServerIpAddress(); - arguments += $" --clientIps \"{serverIps}\""; - } - - using (IProcessProxy process = await this.ExecuteCommandAsync( - PythonCommand, - arguments, - Environment.CurrentDirectory, - telemetryContext, - cancellationToken)) - { - if (!cancellationToken.IsCancellationRequested) - { - await this.LogProcessDetailsAsync(process, telemetryContext, "MySQLServerConfiguration", logToFile: true); - process.ThrowIfDependencyInstallationFailed(process.StandardError.ToString()); - } - } - } - private async Task<string> GetMySQLInnodbDirectoriesAsync(CancellationToken cancellationToken) { string diskPaths = string.Empty; @@ -452,13 +352,7 @@ internal class ConfigurationAction /// <summary> /// Distributes existing database to disks on the system /// </summary> - public const string DistributeDatabase = nameof(DistributeDatabase); - - /// <summary> - /// Truncates all tables existing in database - /// </summary> - public const string TruncateDatabase = nameof(TruncateDatabase); - + public const string DistributeDatabase = nameof(DistributeDatabase); } internal class ConfigurationState From c5d25ffc31503c7cdefcf2759640712494a4e446 Mon Sep 17 00:00:00 2001 From: nikhitha malkapuram <nmalkapuram@microsoft.com> Date: Fri, 29 Nov 2024 13:15:38 +0530 Subject: [PATCH 05/11] updating profile --- .../Sysbench/SysbenchClientExecutor.cs | 2 +- .../profiles/PERF-MYSQL-SYSBENCH-OLTP.json | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchClientExecutor.cs b/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchClientExecutor.cs index 239e92cc9f..61514a72a6 100644 --- a/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchClientExecutor.cs +++ b/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchClientExecutor.cs @@ -282,7 +282,7 @@ private async Task TruncateMySQLDatabaseAsync(EventContext telemetryContext, Can { if (!cancellationToken.IsCancellationRequested) { - await this.LogProcessDetailsAsync(process, telemetryContext, "MySQLServerConfiguration", logToFile: true); + await this.LogProcessDetailsAsync(process, telemetryContext, "Sysbench", logToFile: true); process.ThrowIfDependencyInstallationFailed(process.StandardError.ToString()); } } diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-MYSQL-SYSBENCH-OLTP.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-MYSQL-SYSBENCH-OLTP.json index afeae9c139..f2de30a4ee 100644 --- a/src/VirtualClient/VirtualClient.Main/profiles/PERF-MYSQL-SYSBENCH-OLTP.json +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-MYSQL-SYSBENCH-OLTP.json @@ -169,7 +169,7 @@ "Parameters": { "Scenario": "DownloadMySqlServerPackage", "BlobContainer": "packages", - "BlobName": "mysql-server-8.0.36.zip", + "BlobName": "mysql-server-8.0.36-v1.zip", "PackageName": "mysql-server", "Extract": true, "Role": "Server" @@ -180,7 +180,7 @@ "Parameters": { "Scenario": "DownloadSysbenchPackage", "BlobContainer": "packages", - "BlobName": "sysbench-1.0.20.rev1.zip", + "BlobName": "sysbench-1.0.20.rev2.zip", "PackageName": "sysbench", "Extract": true } @@ -231,7 +231,7 @@ "Action": "SetGlobalVariables", "Benchmark": "OLTP", "DiskFilter": "$.Parameters.DiskFilter", - "Variables": "MAX_PREPARED_STMT_COUNT=1000000;MAX_CONNECTIONS=1000000", + "Variables": "MAX_PREPARED_STMT_COUNT=1000000;MAX_CONNECTIONS=100000", "PackageName": "mysql-server", "Role": "Server" } From bae410d872fe5f4ec13fd8cebb77c130bbb297a4 Mon Sep 17 00:00:00 2001 From: nikhitha malkapuram <nmalkapuram@microsoft.com> Date: Fri, 29 Nov 2024 14:13:22 +0530 Subject: [PATCH 06/11] updating documentation --- .../workloads/sysbench/sysbench-profiles.md | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/website/docs/workloads/sysbench/sysbench-profiles.md b/website/docs/workloads/sysbench/sysbench-profiles.md index 9ef2ec5184..5436412b7c 100644 --- a/website/docs/workloads/sysbench/sysbench-profiles.md +++ b/website/docs/workloads/sysbench/sysbench-profiles.md @@ -157,6 +157,71 @@ There are a lot of moving parts to this workload that allows for both out-of-box with 10,000 records each can be created, a workload can be run on 5 tables for 1000 records if desired. VC does not support dropping and recreating a new database or table configuration within the same profile or system. + VC now supports truncating tables and populating the table again. Following are changes to be made in profile. + ``` bash + { + "Type": "SysbenchClientExecutor", + "Parameters": { + "Scenario": "TruncateMySQLDatabaseTables", + "Action": "TruncateDatabase", + "DatabaseSystem": "MySQL", + "Benchmark": "OLTP", + "DatabaseName": "$.Parameters.DatabaseName", + "PackageName": "mysql-server", + "Role": "Client" + } + }, + { + "Type": "WaitExecutor", + "Parameters": { + "Scenario": "WaitForTimeProvided", + "Duration": "00:01:00" + } + }, + { + "Type": "SysbenchClientExecutor", + "Parameters": { + "Scenario": "PopulateMySQLDatabase", + "Action": "PopulateDatabase", + "DatabaseSystem": "MySQL", + "Benchmark": "OLTP", + "DatabaseName": "$.Parameters.DatabaseName", + "DatabaseScenario": "$.Parameters.DatabaseScenario", + "PackageName": "sysbench", + "Threads": "$.Parameters.Threads", + "RecordCount": "$.Parameters.RecordCount", + "TableCount": "$.Parameters.TableCount", + "Role": "Client" + } + }, + { + "Type": "WaitExecutor", + "Parameters": { + "Scenario": "WaitForTimeProvided", + "Duration": "00:01:00" + } + }, + { + "Type": "SysbenchClientExecutor", + "Parameters": { + "Scenario": "oltp_read_only", + "Action": "RunWorkload", + "DatabaseSystem": "MySQL", + "Benchmark": "OLTP", + "DatabaseName": "$.Parameters.DatabaseName", + "DatabaseScenario": "$.Parameters.DatabaseScenario", + "Duration": "$.Parameters.Duration", + "Workload": "oltp_read_only", + "PackageName": "sysbench", + "Threads": "$.Parameters.Threads", + "RecordCount": "$.Parameters.RecordCount", + "TableCount": "$.Parameters.TableCount", + "Role": "Client" + } + }, + ``` + + * **SysbenchServerExecutor** Sets the server online for client interaction. From f9c27d7a990ba182e0bd6bb58ae1527ed00481d9 Mon Sep 17 00:00:00 2001 From: nikhitha malkapuram <nmalkapuram@microsoft.com> Date: Mon, 2 Dec 2024 16:15:01 +0530 Subject: [PATCH 07/11] optimizing code --- .../CtsTraffic/CtsTrafficClientExecutor.cs | 17 +++----------- .../Sysbench/SysbenchClientExecutor.cs | 23 ++++--------------- .../Sysbench/SysbenchConfiguration.cs | 14 ----------- .../Sysbench/SysbenchExecutor.cs | 1 - .../Extensions/EnvironmentLayoutExtensions.cs | 19 +++++++++++++++ .../MySqlServer/MySqlServerConfiguration.cs | 14 ----------- .../PostgreSQLServerConfiguration.cs | 14 ----------- 7 files changed, 26 insertions(+), 76 deletions(-) diff --git a/src/VirtualClient/VirtualClient.Actions/CtsTraffic/CtsTrafficClientExecutor.cs b/src/VirtualClient/VirtualClient.Actions/CtsTraffic/CtsTrafficClientExecutor.cs index 9f51eeb72d..35d63645d7 100644 --- a/src/VirtualClient/VirtualClient.Actions/CtsTraffic/CtsTrafficClientExecutor.cs +++ b/src/VirtualClient/VirtualClient.Actions/CtsTraffic/CtsTrafficClientExecutor.cs @@ -8,6 +8,7 @@ namespace VirtualClient.Actions using System.Linq; using System.Threading; using System.Threading.Tasks; + using MathNet.Numerics; using Microsoft.Extensions.DependencyInjection; using VirtualClient; using VirtualClient.Common; @@ -98,7 +99,8 @@ await this.ServerApiClient.PollForExpectedStateAsync<CtsTrafficServerState>( cancellationToken, this.PollingInterval); - string targetIPAddress = this.GetServerIPAddress(cancellationToken); + ClientInstance instance = this.Layout.GetClientInstance(this.AgentId); + string targetIPAddress = (instance.Role == ClientRole.Server) ? "localhost" : this.GetServerIpAddress(); string ctsTrafficCommandArgs = $"-Target:{targetIPAddress} -Consoleverbosity:1 -StatusFilename:{this.StatusFileName} " + $@"-ConnectionFilename:{this.ConnectionsFileName} -ErrorFileName:{this.ErrorFileName} -Port:{this.Port} " + @@ -145,18 +147,5 @@ await this.ServerApiClient.PollForExpectedStateAsync<CtsTrafficServerState>( } } - - private string GetServerIPAddress(CancellationToken cancellationToken) - { - string targetIPAddress = "localhost"; - - if (this.IsMultiRoleLayout()) - { - ClientInstance serverInstance = this.GetLayoutClientInstances(ClientRole.Server).First(); - targetIPAddress = serverInstance.IPAddress; - } - - return targetIPAddress; - } } } \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchClientExecutor.cs b/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchClientExecutor.cs index 61514a72a6..fc695dad0e 100644 --- a/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchClientExecutor.cs +++ b/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchClientExecutor.cs @@ -218,7 +218,7 @@ private async Task RunOLTPWorkloadAsync(EventContext telemetryContext, Cancellat string script = $"{this.SysbenchPackagePath}/run-workload.py "; using (IProcessProxy process = await this.ExecuteCommandAsync( - PythonCommand, + SysbenchClientExecutor.PythonCommand, script + this.sysbenchExecutionArguments, this.SysbenchPackagePath, telemetryContext, @@ -246,7 +246,7 @@ private async Task RunTPCCWorkloadAsync(EventContext telemetryContext, Cancellat string script = $"{this.SysbenchPackagePath}/run-workload.py "; using (IProcessProxy process = await this.ExecuteCommandAsync( - PythonCommand, + SysbenchClientExecutor.PythonCommand, script + this.sysbenchExecutionArguments, this.SysbenchPackagePath, telemetryContext, @@ -274,7 +274,7 @@ private async Task TruncateMySQLDatabaseAsync(EventContext telemetryContext, Can } using (IProcessProxy process = await this.ExecuteCommandAsync( - PythonCommand, + SysbenchClientExecutor.PythonCommand, arguments, Environment.CurrentDirectory, telemetryContext, @@ -303,11 +303,10 @@ private async Task PrepareOLTPMySQLDatabase(EventContext telemetryContext, Cance this.sysbenchPrepareArguments += $" --host \"{serverIp}\""; } - string command = $"python3"; string arguments = $"{this.SysbenchPackagePath}/populate-database.py "; using (IProcessProxy process = await this.ExecuteCommandAsync( - command, + SysbenchClientExecutor.PythonCommand, arguments + this.sysbenchPrepareArguments, this.SysbenchPackagePath, telemetryContext, @@ -320,19 +319,5 @@ private async Task PrepareOLTPMySQLDatabase(EventContext telemetryContext, Cance } } } - - private string GetServerIpAddress() - { - string serverIPAddress = IPAddress.Loopback.ToString(); - - if (this.IsMultiRoleLayout()) - { - ClientInstance serverInstance = this.GetLayoutClientInstances(ClientRole.Server).First(); - IPAddress.TryParse(serverInstance.IPAddress, out IPAddress serverIP); - serverIPAddress = serverIP.ToString(); - } - - return serverIPAddress; - } } } diff --git a/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchConfiguration.cs b/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchConfiguration.cs index b3090ce017..e708179ee1 100644 --- a/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchConfiguration.cs +++ b/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchConfiguration.cs @@ -134,19 +134,5 @@ private async Task PrepareTPCCMySQLDatabase(EventContext telemetryContext, Cance } } } - - private string GetServerIpAddress() - { - string serverIPAddress = IPAddress.Loopback.ToString(); - - if (this.IsMultiRoleLayout()) - { - ClientInstance serverInstance = this.GetLayoutClientInstances(ClientRole.Server).First(); - IPAddress.TryParse(serverInstance.IPAddress, out IPAddress serverIP); - serverIPAddress = serverIP.ToString(); - } - - return serverIPAddress; - } } } diff --git a/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchExecutor.cs b/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchExecutor.cs index bebad56b38..1bb18013a6 100644 --- a/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchExecutor.cs +++ b/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchExecutor.cs @@ -184,7 +184,6 @@ public string Action { this.Parameters.TryGetValue(nameof(this.Action), out IConvertible action); return action?.ToString(); - // return this.Parameters.GetValue<string>(nameof(this.Action), null); } } diff --git a/src/VirtualClient/VirtualClient.Contracts/Extensions/EnvironmentLayoutExtensions.cs b/src/VirtualClient/VirtualClient.Contracts/Extensions/EnvironmentLayoutExtensions.cs index d19eb9f727..066928c9e9 100644 --- a/src/VirtualClient/VirtualClient.Contracts/Extensions/EnvironmentLayoutExtensions.cs +++ b/src/VirtualClient/VirtualClient.Contracts/Extensions/EnvironmentLayoutExtensions.cs @@ -6,6 +6,7 @@ namespace VirtualClient.Contracts using System; using System.Collections.Generic; using System.Linq; + using System.Net; /// <summary> /// Extensions for <see cref="EnvironmentLayout"/> instances. @@ -34,5 +35,23 @@ public static ClientInstance GetClientInstance(this EnvironmentLayout layout, st return clientInstances?.FirstOrDefault(); } + + /// <summary> + /// Returns first ServerIpAddress from the layout file in a client-server scenario. + /// </summary> + /// <param name="component">The component with the environment layout.</param> + /// <returns></returns> + public static string GetServerIpAddress(this VirtualClientComponent component) + { + string serverIPAddress = IPAddress.Loopback.ToString(); + + if (component.IsMultiRoleLayout()) + { + ClientInstance serverInstance = component.GetLayoutClientInstances(ClientRole.Server).First(); + serverIPAddress = serverInstance.IPAddress; + } + + return serverIPAddress; + } } } diff --git a/src/VirtualClient/VirtualClient.Dependencies/MySqlServer/MySqlServerConfiguration.cs b/src/VirtualClient/VirtualClient.Dependencies/MySqlServer/MySqlServerConfiguration.cs index 1331137d65..63b8fc6e82 100644 --- a/src/VirtualClient/VirtualClient.Dependencies/MySqlServer/MySqlServerConfiguration.cs +++ b/src/VirtualClient/VirtualClient.Dependencies/MySqlServer/MySqlServerConfiguration.cs @@ -299,20 +299,6 @@ private async Task<string> GetMySQLInMemoryCapacityAsync(CancellationToken cance return bufferSizeInMegaBytes.ToString(); } - private string GetServerIpAddress() - { - string serverIPAddress = IPAddress.Loopback.ToString(); - - if (this.IsMultiRoleLayout()) - { - ClientInstance serverInstance = this.GetLayoutClientInstances(ClientRole.Server).First(); - IPAddress.TryParse(serverInstance.IPAddress, out IPAddress serverIP); - serverIPAddress = serverIP.ToString(); - } - - return serverIPAddress; - } - private string GetClientIpAddresses() { string clientIpAddresses = string.Empty; diff --git a/src/VirtualClient/VirtualClient.Dependencies/PostgreSQLServer/PostgreSQLServerConfiguration.cs b/src/VirtualClient/VirtualClient.Dependencies/PostgreSQLServer/PostgreSQLServerConfiguration.cs index 3ff111e0e4..ad141601be 100644 --- a/src/VirtualClient/VirtualClient.Dependencies/PostgreSQLServer/PostgreSQLServerConfiguration.cs +++ b/src/VirtualClient/VirtualClient.Dependencies/PostgreSQLServer/PostgreSQLServerConfiguration.cs @@ -272,20 +272,6 @@ private async Task<string> GetPostgreSQLInnodbDirectoriesAsync(CancellationToken return diskPaths; } - private string GetServerIpAddress() - { - string serverIPAddress = IPAddress.Loopback.ToString(); - - if (this.IsMultiRoleLayout()) - { - ClientInstance serverInstance = this.GetLayoutClientInstances(ClientRole.Server).First(); - IPAddress.TryParse(serverInstance.IPAddress, out IPAddress serverIP); - serverIPAddress = serverIP.ToString(); - } - - return serverIPAddress; - } - /// <summary> /// Supported PostgreSQL Server configuration actions. /// </summary> From 7a7b572ed1db61e9f356edb959d2092ce423ab5e Mon Sep 17 00:00:00 2001 From: nikhitha malkapuram <nmalkapuram@microsoft.com> Date: Thu, 19 Dec 2024 11:26:59 +0530 Subject: [PATCH 08/11] adding changes+ extra debugging --- .../SysbenchProfileTests.cs | 2 +- .../VirtualClientComponent.cs | 1 + .../ProfileExpressionEvaluatorTests.cs | 1 + .../ProfileExpressionEvaluator.cs | 16 +++++++++++++--- .../profiles/PERF-MYSQL-SYSBENCH-OLTP.json | 4 +++- 5 files changed, 19 insertions(+), 5 deletions(-) diff --git a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/SysbenchProfileTests.cs b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/SysbenchProfileTests.cs index d803e3f12c..98b5bd24ea 100644 --- a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/SysbenchProfileTests.cs +++ b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/SysbenchProfileTests.cs @@ -63,7 +63,7 @@ public void MySQLSysbenchOLTPWorkloadProfileActionsWillNotBeExecutedIfTheWorkloa [Test] [TestCase("PERF-MYSQL-SYSBENCH-OLTP.json", PlatformID.Unix, Architecture.X64)] - [Ignore("This functional test takes more than a minute to run. There is likely a Task.Delay somewhere that needs to be designed better to avoid long blocking behavior")] + // [Ignore("This functional test takes more than a minute to run. There is likely a Task.Delay somewhere that needs to be designed better to avoid long blocking behavior")] public async Task MySQLSysbenchOLTPWorkloadProfileExecutesTheExpectedWorkloadsOnUnixPlatform(string profile, PlatformID platform, Architecture architecture) { this.fixture.Setup(platform, architecture, this.clientAgentId).SetupLayout( diff --git a/src/VirtualClient/VirtualClient.Contracts/VirtualClientComponent.cs b/src/VirtualClient/VirtualClient.Contracts/VirtualClientComponent.cs index 1b0b879567..84d20d9990 100644 --- a/src/VirtualClient/VirtualClient.Contracts/VirtualClientComponent.cs +++ b/src/VirtualClient/VirtualClient.Contracts/VirtualClientComponent.cs @@ -625,6 +625,7 @@ public async Task ExecuteAsync(CancellationToken cancellationToken) if (this.Parameters?.Any() == true) { + Console.WriteLine("before adding parameters to dictonary"); this.MetadataContract.Add( this.Parameters.Keys.ToDictionary(key => key, entry => this.Parameters[entry] as object).ObscureSecrets(), MetadataContractCategory.Scenario, diff --git a/src/VirtualClient/VirtualClient.Core.UnitTests/ProfileExpressionEvaluatorTests.cs b/src/VirtualClient/VirtualClient.Core.UnitTests/ProfileExpressionEvaluatorTests.cs index 74254ff819..d807196fc1 100644 --- a/src/VirtualClient/VirtualClient.Core.UnitTests/ProfileExpressionEvaluatorTests.cs +++ b/src/VirtualClient/VirtualClient.Core.UnitTests/ProfileExpressionEvaluatorTests.cs @@ -709,6 +709,7 @@ public async Task ProfileExpressionEvaluatorSupportsFunctionReferences() Dictionary<string, string> expressions = new Dictionary<string, string> { + {"{calculate(67419566080 * 800000 / 100)}", "23456"}, { "{calculate(512 + 2)}", "514" }, { "{calculate(512 - 2)}", "510" }, { "{calculate(512 * 2)}", "1024" }, diff --git a/src/VirtualClient/VirtualClient.Core/ProfileExpressionEvaluator.cs b/src/VirtualClient/VirtualClient.Core/ProfileExpressionEvaluator.cs index 78fbc83c9f..2d0d0ee251 100644 --- a/src/VirtualClient/VirtualClient.Core/ProfileExpressionEvaluator.cs +++ b/src/VirtualClient/VirtualClient.Core/ProfileExpressionEvaluator.cs @@ -24,9 +24,13 @@ public class ProfileExpressionEvaluator : IExpressionEvaluator // e.g. // {fn(512 / 16)]} // {fn(512 / {LogicalThreadCount})} - private static readonly Regex CalculateExpression = new Regex( + /*private static readonly Regex CalculateExpression = new Regex( @"\{calculate\(([0-9\*\/\+\-\(\)\s]+)\)\}", - RegexOptions.Compiled | RegexOptions.IgnoreCase); + RegexOptions.Compiled | RegexOptions.IgnoreCase);*/ + + private static readonly Regex CalculateExpression = new Regex( + @"\{calculate\(([0-9L\*\/\+\-\(\)\s]+)\)\}", + RegexOptions.Compiled | RegexOptions.IgnoreCase); // e.g. // {calculate({IsTLSEnabled} ? "Yes" : "No")} @@ -436,13 +440,16 @@ public class ProfileExpressionEvaluator : IExpressionEvaluator if (matches?.Any() == true) { + Console.WriteLine($"just before c# script"); isMatched = true; foreach (Match match in matches) { string function = match.Groups[1].Value; + Console.WriteLine($"just before c# script"); long result = await Microsoft.CodeAnalysis.CSharp.Scripting.CSharpScript.EvaluateAsync<long>(function); - + Console.WriteLine($"the result after csharpscript is {result}"); evaluatedExpression = evaluatedExpression.Replace(match.Value, result.ToString()); + Console.WriteLine($"evaluated expression {evaluatedExpression}"); } } @@ -638,6 +645,7 @@ private static async Task<EvaluationResult> EvaluateExpressionAsync(IServiceColl { isMatched = true; evaluatedExpression = evaluation.Outcome; + Console.WriteLine($"Evaluated expression in evaluateexpressionasync {evaluatedExpression}"); } } } @@ -724,7 +732,9 @@ private static async Task<bool> EvaluateWellKnownExpressionsAsync(IServiceCollec EvaluationResult evaluation = await ProfileExpressionEvaluator.EvaluateExpressionAsync(dependencies, parameters, parameter.Value.ToString(), cancellationToken); if (evaluation.IsMatched) { + Console.WriteLine($"After evaluation in runtime {evaluation.Outcome}"); parameters[parameter.Key] = evaluation.Outcome; + Console.WriteLine($"After assigning it to parameter key in runtime {evaluation.Outcome}"); } } } diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-MYSQL-SYSBENCH-OLTP.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-MYSQL-SYSBENCH-OLTP.json index f2de30a4ee..564e59c024 100644 --- a/src/VirtualClient/VirtualClient.Main/profiles/PERF-MYSQL-SYSBENCH-OLTP.json +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-MYSQL-SYSBENCH-OLTP.json @@ -10,6 +10,7 @@ "DatabaseName": "sbtest", "DatabaseScenario": "Balanced", "DiskFilter": "osdisk:false&sizegreaterthan:256g", + "InnodbBufferPoolSize": "{calculate({SystemMemoryBytes} * 80 / 100)}", "Duration": "00:05:00" }, "Actions": [ @@ -231,7 +232,8 @@ "Action": "SetGlobalVariables", "Benchmark": "OLTP", "DiskFilter": "$.Parameters.DiskFilter", - "Variables": "MAX_PREPARED_STMT_COUNT=1000000;MAX_CONNECTIONS=100000", + "InnodbBufferPoolSize": "$.Parameters.InnodbBufferPoolSize", + "Variables": "MAX_PREPARED_STMT_COUNT=1000000;MAX_CONNECTIONS=100000;innodb_buffer_pool_size={InnodbBufferPoolSize};innodb_lock_wait_timeout=300;innodb_io_capacity=10000;innodb_io_capacity_max=10000;innodb_buffer_pool_dump_at_shutdown=OFF;innodb_change_buffering=0;table_open_cache=20000;skip_log_bin=1;", "PackageName": "mysql-server", "Role": "Server" } From 1982cf4bd6a33e5c3a38d6d31c7e0054cb3f84fb Mon Sep 17 00:00:00 2001 From: nikhitha malkapuram <nmalkapuram@microsoft.com> Date: Mon, 6 Jan 2025 03:51:26 +0530 Subject: [PATCH 09/11] allow striped disk to be used by mysql, updating mysql server package version, resolving comments --- .../CtsTraffic/CtsTrafficClientExecutor.cs | 14 ++++++++++ .../Sysbench/SysbenchExecutor.cs | 17 ++++++++++++ .../Extensions/EnvironmentLayoutExtensions.cs | 18 ------------- .../MySqlServer/MySqlServerConfiguration.cs | 26 ++++++++++++++++++- .../PostgreSQLServerConfiguration.cs | 13 ++++++++++ .../profiles/PERF-MYSQL-SYSBENCH-OLTP.json | 4 +-- 6 files changed, 71 insertions(+), 21 deletions(-) diff --git a/src/VirtualClient/VirtualClient.Actions/CtsTraffic/CtsTrafficClientExecutor.cs b/src/VirtualClient/VirtualClient.Actions/CtsTraffic/CtsTrafficClientExecutor.cs index 35d63645d7..db318a2f70 100644 --- a/src/VirtualClient/VirtualClient.Actions/CtsTraffic/CtsTrafficClientExecutor.cs +++ b/src/VirtualClient/VirtualClient.Actions/CtsTraffic/CtsTrafficClientExecutor.cs @@ -6,6 +6,7 @@ namespace VirtualClient.Actions using System; using System.Collections.Generic; using System.Linq; + using System.Net; using System.Threading; using System.Threading.Tasks; using MathNet.Numerics; @@ -147,5 +148,18 @@ await this.ServerApiClient.PollForExpectedStateAsync<CtsTrafficServerState>( } } + + private string GetServerIpAddress() + { + string serverIPAddress = IPAddress.Loopback.ToString(); + + if (this.IsMultiRoleLayout()) + { + ClientInstance serverInstance = this.GetLayoutClientInstances(ClientRole.Server).First(); + serverIPAddress = serverInstance.IPAddress; + } + + return serverIPAddress; + } } } \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchExecutor.cs b/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchExecutor.cs index 1bb18013a6..b2b57c274f 100644 --- a/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchExecutor.cs +++ b/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchExecutor.cs @@ -392,6 +392,23 @@ protected async Task InitializeExecutablesAsync(EventContext telemetryContext, C await this.stateManager.SaveStateAsync<SysbenchState>(nameof(SysbenchState), state, cancellationToken); } + /// <summary> + /// + /// </summary> + /// <returns></returns> + protected string GetServerIpAddress() + { + string serverIPAddress = IPAddress.Loopback.ToString(); + + if (this.IsMultiRoleLayout()) + { + ClientInstance serverInstance = this.GetLayoutClientInstances(ClientRole.Server).First(); + serverIPAddress = serverInstance.IPAddress; + } + + return serverIPAddress; + } + private async Task CheckDistroSupportAsync(EventContext telemetryContext, CancellationToken cancellationToken) { if (this.Platform == PlatformID.Unix) diff --git a/src/VirtualClient/VirtualClient.Contracts/Extensions/EnvironmentLayoutExtensions.cs b/src/VirtualClient/VirtualClient.Contracts/Extensions/EnvironmentLayoutExtensions.cs index 066928c9e9..4d8ef8cb65 100644 --- a/src/VirtualClient/VirtualClient.Contracts/Extensions/EnvironmentLayoutExtensions.cs +++ b/src/VirtualClient/VirtualClient.Contracts/Extensions/EnvironmentLayoutExtensions.cs @@ -35,23 +35,5 @@ public static ClientInstance GetClientInstance(this EnvironmentLayout layout, st return clientInstances?.FirstOrDefault(); } - - /// <summary> - /// Returns first ServerIpAddress from the layout file in a client-server scenario. - /// </summary> - /// <param name="component">The component with the environment layout.</param> - /// <returns></returns> - public static string GetServerIpAddress(this VirtualClientComponent component) - { - string serverIPAddress = IPAddress.Loopback.ToString(); - - if (component.IsMultiRoleLayout()) - { - ClientInstance serverInstance = component.GetLayoutClientInstances(ClientRole.Server).First(); - serverIPAddress = serverInstance.IPAddress; - } - - return serverIPAddress; - } } } diff --git a/src/VirtualClient/VirtualClient.Dependencies/MySqlServer/MySqlServerConfiguration.cs b/src/VirtualClient/VirtualClient.Dependencies/MySqlServer/MySqlServerConfiguration.cs index 63b8fc6e82..7fbac2255c 100644 --- a/src/VirtualClient/VirtualClient.Dependencies/MySqlServer/MySqlServerConfiguration.cs +++ b/src/VirtualClient/VirtualClient.Dependencies/MySqlServer/MySqlServerConfiguration.cs @@ -63,6 +63,17 @@ public bool SkipInitialize } } + /// <summary> + /// stripeddisk mount point. + /// </summary> + public string StripeDiskMountPoint + { + get + { + return this.Parameters.GetValue<string>(nameof(this.StripeDiskMountPoint), null); + } + } + /// <summary> /// Disk filter specified /// </summary> @@ -164,7 +175,7 @@ await this.SetMySQLGlobalVariableAsync(telemetryContext, cancellationToken) private async Task ConfigureMySQLServerAsync(EventContext telemetryContext, CancellationToken cancellationToken) { string serverIp = this.GetServerIpAddress(); - string innoDbDirs = await this.GetMySQLInnodbDirectoriesAsync(cancellationToken); + string innoDbDirs = this.StripeDiskMountPoint != null ? this.StripeDiskMountPoint : await this.GetMySQLInnodbDirectoriesAsync(cancellationToken); string arguments = $"{this.packageDirectory}/configure.py --serverIp {serverIp} --innoDbDirs \"{innoDbDirs}\""; @@ -299,6 +310,19 @@ private async Task<string> GetMySQLInMemoryCapacityAsync(CancellationToken cance return bufferSizeInMegaBytes.ToString(); } + private string GetServerIpAddress() + { + string serverIPAddress = IPAddress.Loopback.ToString(); + + if (this.IsMultiRoleLayout()) + { + ClientInstance serverInstance = this.GetLayoutClientInstances(ClientRole.Server).First(); + serverIPAddress = serverInstance.IPAddress; + } + + return serverIPAddress; + } + private string GetClientIpAddresses() { string clientIpAddresses = string.Empty; diff --git a/src/VirtualClient/VirtualClient.Dependencies/PostgreSQLServer/PostgreSQLServerConfiguration.cs b/src/VirtualClient/VirtualClient.Dependencies/PostgreSQLServer/PostgreSQLServerConfiguration.cs index ad141601be..7f005ed3e7 100644 --- a/src/VirtualClient/VirtualClient.Dependencies/PostgreSQLServer/PostgreSQLServerConfiguration.cs +++ b/src/VirtualClient/VirtualClient.Dependencies/PostgreSQLServer/PostgreSQLServerConfiguration.cs @@ -272,6 +272,19 @@ private async Task<string> GetPostgreSQLInnodbDirectoriesAsync(CancellationToken return diskPaths; } + private string GetServerIpAddress() + { + string serverIPAddress = IPAddress.Loopback.ToString(); + + if (this.IsMultiRoleLayout()) + { + ClientInstance serverInstance = this.GetLayoutClientInstances(ClientRole.Server).First(); + serverIPAddress = serverInstance.IPAddress; + } + + return serverIPAddress; + } + /// <summary> /// Supported PostgreSQL Server configuration actions. /// </summary> diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-MYSQL-SYSBENCH-OLTP.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-MYSQL-SYSBENCH-OLTP.json index 564e59c024..58c7745c19 100644 --- a/src/VirtualClient/VirtualClient.Main/profiles/PERF-MYSQL-SYSBENCH-OLTP.json +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-MYSQL-SYSBENCH-OLTP.json @@ -170,7 +170,7 @@ "Parameters": { "Scenario": "DownloadMySqlServerPackage", "BlobContainer": "packages", - "BlobName": "mysql-server-8.0.36-v1.zip", + "BlobName": "mysql-server-8.0.36-v2.zip", "PackageName": "mysql-server", "Extract": true, "Role": "Server" @@ -233,7 +233,7 @@ "Benchmark": "OLTP", "DiskFilter": "$.Parameters.DiskFilter", "InnodbBufferPoolSize": "$.Parameters.InnodbBufferPoolSize", - "Variables": "MAX_PREPARED_STMT_COUNT=1000000;MAX_CONNECTIONS=100000;innodb_buffer_pool_size={InnodbBufferPoolSize};innodb_lock_wait_timeout=300;innodb_io_capacity=10000;innodb_io_capacity_max=10000;innodb_buffer_pool_dump_at_shutdown=OFF;innodb_change_buffering=0;table_open_cache=20000;skip_log_bin=1;", + "Variables": "MAX_PREPARED_STMT_COUNT=1000000;MAX_CONNECTIONS=100000;innodb_buffer_pool_size={InnodbBufferPoolSize};innodb_lock_wait_timeout=300;innodb_io_capacity=10000;innodb_io_capacity_max=10000;innodb_buffer_pool_dump_at_shutdown=OFF;innodb_change_buffering=0;table_open_cache=20000;", "PackageName": "mysql-server", "Role": "Server" } From b8b403f38a0421fb88a81b14b0a2d2872768414e Mon Sep 17 00:00:00 2001 From: nikhitha malkapuram <nmalkapuram@microsoft.com> Date: Tue, 4 Feb 2025 13:09:06 +0530 Subject: [PATCH 10/11] adding new parameter while database population and some code clean up --- .../SysbenchProfileTests.cs | 2 +- .../CtsTraffic/CtsTrafficClientExecutor.cs | 18 ++------ .../Sysbench/SysbenchClientExecutor.cs | 40 ++++++++-------- .../Sysbench/SysbenchConfiguration.cs | 25 +++++----- .../Sysbench/SysbenchExecutor.cs | 46 +++++++++++++++---- .../Extensions/EnvironmentLayoutExtensions.cs | 1 - .../VirtualClientComponent.cs | 1 - .../ProfileExpressionEvaluatorTests.cs | 1 - .../ProfileExpressionEvaluator.cs | 11 ----- .../MySqlServer/MySqlServerConfiguration.cs | 29 +++++------- 10 files changed, 83 insertions(+), 91 deletions(-) diff --git a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/SysbenchProfileTests.cs b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/SysbenchProfileTests.cs index 98b5bd24ea..d803e3f12c 100644 --- a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/SysbenchProfileTests.cs +++ b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/SysbenchProfileTests.cs @@ -63,7 +63,7 @@ public void MySQLSysbenchOLTPWorkloadProfileActionsWillNotBeExecutedIfTheWorkloa [Test] [TestCase("PERF-MYSQL-SYSBENCH-OLTP.json", PlatformID.Unix, Architecture.X64)] - // [Ignore("This functional test takes more than a minute to run. There is likely a Task.Delay somewhere that needs to be designed better to avoid long blocking behavior")] + [Ignore("This functional test takes more than a minute to run. There is likely a Task.Delay somewhere that needs to be designed better to avoid long blocking behavior")] public async Task MySQLSysbenchOLTPWorkloadProfileExecutesTheExpectedWorkloadsOnUnixPlatform(string profile, PlatformID platform, Architecture architecture) { this.fixture.Setup(platform, architecture, this.clientAgentId).SetupLayout( diff --git a/src/VirtualClient/VirtualClient.Actions/CtsTraffic/CtsTrafficClientExecutor.cs b/src/VirtualClient/VirtualClient.Actions/CtsTraffic/CtsTrafficClientExecutor.cs index db318a2f70..c00b88be9c 100644 --- a/src/VirtualClient/VirtualClient.Actions/CtsTraffic/CtsTrafficClientExecutor.cs +++ b/src/VirtualClient/VirtualClient.Actions/CtsTraffic/CtsTrafficClientExecutor.cs @@ -100,8 +100,9 @@ await this.ServerApiClient.PollForExpectedStateAsync<CtsTrafficServerState>( cancellationToken, this.PollingInterval); - ClientInstance instance = this.Layout.GetClientInstance(this.AgentId); - string targetIPAddress = (instance.Role == ClientRole.Server) ? "localhost" : this.GetServerIpAddress(); + string targetIPAddress = (this.GetLayoutClientInstances(ClientRole.Server, false) ?? Enumerable.Empty<ClientInstance>()) + .FirstOrDefault()?.IPAddress + ?? "localhost"; string ctsTrafficCommandArgs = $"-Target:{targetIPAddress} -Consoleverbosity:1 -StatusFilename:{this.StatusFileName} " + $@"-ConnectionFilename:{this.ConnectionsFileName} -ErrorFileName:{this.ErrorFileName} -Port:{this.Port} " + @@ -148,18 +149,5 @@ await this.ServerApiClient.PollForExpectedStateAsync<CtsTrafficServerState>( } } - - private string GetServerIpAddress() - { - string serverIPAddress = IPAddress.Loopback.ToString(); - - if (this.IsMultiRoleLayout()) - { - ClientInstance serverInstance = this.GetLayoutClientInstances(ClientRole.Server).First(); - serverIPAddress = serverInstance.IPAddress; - } - - return serverIPAddress; - } } } \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchClientExecutor.cs b/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchClientExecutor.cs index fc695dad0e..9c891a8df0 100644 --- a/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchClientExecutor.cs +++ b/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchClientExecutor.cs @@ -5,17 +5,13 @@ namespace VirtualClient.Actions { using System; using System.Collections.Generic; - using System.Globalization; using System.Linq; using System.Net; - using System.Net.Http; - using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; using Polly; using VirtualClient.Common; - using VirtualClient.Common.Contracts; using VirtualClient.Common.Extensions; using VirtualClient.Common.Telemetry; using VirtualClient.Contracts; @@ -26,7 +22,6 @@ namespace VirtualClient.Actions /// </summary> public class SysbenchClientExecutor : SysbenchExecutor { - private const string PythonCommand = "python3"; private string sysbenchExecutionArguments; private string sysbenchLoggingArguments; private string sysbenchPrepareArguments; @@ -218,7 +213,7 @@ private async Task RunOLTPWorkloadAsync(EventContext telemetryContext, Cancellat string script = $"{this.SysbenchPackagePath}/run-workload.py "; using (IProcessProxy process = await this.ExecuteCommandAsync( - SysbenchClientExecutor.PythonCommand, + SysbenchExecutor.PythonCommand, script + this.sysbenchExecutionArguments, this.SysbenchPackagePath, telemetryContext, @@ -246,7 +241,7 @@ private async Task RunTPCCWorkloadAsync(EventContext telemetryContext, Cancellat string script = $"{this.SysbenchPackagePath}/run-workload.py "; using (IProcessProxy process = await this.ExecuteCommandAsync( - SysbenchClientExecutor.PythonCommand, + SysbenchExecutor.PythonCommand, script + this.sysbenchExecutionArguments, this.SysbenchPackagePath, telemetryContext, @@ -266,15 +261,14 @@ private async Task TruncateMySQLDatabaseAsync(EventContext telemetryContext, Can { string arguments = $"{this.packageDirectory}/truncate-tables.py --dbName {this.DatabaseName}"; - if (this.IsMultiRoleLayout()) - { - ClientInstance instance = this.Layout.GetClientInstance(this.AgentId); - string serverIps = (instance.Role == ClientRole.Server) ? "localhost" : this.GetServerIpAddress(); - arguments += $" --clientIps \"{serverIps}\""; - } + string serverIps = (this.GetLayoutClientInstances(ClientRole.Server, false) ?? Enumerable.Empty<ClientInstance>()) + .FirstOrDefault()?.IPAddress + ?? "localhost"; + + arguments += $" --clientIps \"{serverIps}\""; using (IProcessProxy process = await this.ExecuteCommandAsync( - SysbenchClientExecutor.PythonCommand, + SysbenchExecutor.PythonCommand, arguments, Environment.CurrentDirectory, telemetryContext, @@ -294,19 +288,19 @@ private async Task PrepareOLTPMySQLDatabase(EventContext telemetryContext, Cance int threadCount = GetThreadCount(this.SystemManager, this.DatabaseScenario, this.Threads); int recordCount = GetRecordCount(this.SystemManager, this.DatabaseScenario, this.RecordCount); - this.sysbenchPrepareArguments = $"--dbName {this.DatabaseName} --databaseSystem {this.DatabaseSystem} --benchmark {this.Benchmark} --tableCount {tableCount} --recordCount {recordCount} --threadCount {threadCount} --password {this.SuperUserPassword}"; + this.sysbenchLoggingArguments = $"--dbName {this.DatabaseName} --databaseSystem {this.DatabaseSystem} --benchmark {this.Benchmark} --tableCount {tableCount} --recordCount {recordCount} --threadCount {threadCount}"; + this.sysbenchPrepareArguments = $"{this.sysbenchLoggingArguments} --password {this.SuperUserPassword}"; - if (this.IsMultiRoleLayout()) - { - ClientInstance instance = this.Layout.GetClientInstance(this.AgentId); - string serverIp = (instance.Role == ClientRole.Server) ? "localhost" : this.GetServerIpAddress(); - this.sysbenchPrepareArguments += $" --host \"{serverIp}\""; - } + string serverIp = (this.GetLayoutClientInstances(ClientRole.Server, false) ?? Enumerable.Empty<ClientInstance>()) + .FirstOrDefault()?.IPAddress + ?? "localhost"; + + this.sysbenchPrepareArguments += $" --host \"{serverIp}\""; string arguments = $"{this.SysbenchPackagePath}/populate-database.py "; using (IProcessProxy process = await this.ExecuteCommandAsync( - SysbenchClientExecutor.PythonCommand, + SysbenchExecutor.PythonCommand, arguments + this.sysbenchPrepareArguments, this.SysbenchPackagePath, telemetryContext, @@ -316,6 +310,8 @@ private async Task PrepareOLTPMySQLDatabase(EventContext telemetryContext, Cance { await this.LogProcessDetailsAsync(process, telemetryContext, "Sysbench", logToFile: true); process.ThrowIfErrored<WorkloadException>(process.StandardError.ToString(), ErrorReason.WorkloadUnexpectedAnomaly); + + this.AddMetric(this.sysbenchLoggingArguments, process, telemetryContext, cancellationToken); } } } diff --git a/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchConfiguration.cs b/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchConfiguration.cs index e708179ee1..87587facc3 100644 --- a/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchConfiguration.cs +++ b/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchConfiguration.cs @@ -82,20 +82,17 @@ private async Task PrepareOLTPMySQLDatabase(EventContext telemetryContext, Cance int threadCount = GetThreadCount(this.SystemManager, this.DatabaseScenario, this.Threads); int recordCount = GetRecordCount(this.SystemManager, this.DatabaseScenario, this.RecordCount); - this.sysbenchPrepareArguments = $"--dbName {this.DatabaseName} --databaseSystem {this.DatabaseSystem} --benchmark {this.Benchmark} --tableCount {tableCount} --recordCount {recordCount} --threadCount {threadCount} --password {this.SuperUserPassword}"; + string sysbenchLoggingArguments = $"--dbName {this.DatabaseName} --databaseSystem {this.DatabaseSystem} --benchmark {this.Benchmark} --tableCount {tableCount} --recordCount {recordCount} --threadCount {threadCount}"; + this.sysbenchPrepareArguments = $"{sysbenchLoggingArguments} --password {this.SuperUserPassword}"; - if (this.IsMultiRoleLayout()) - { - ClientInstance instance = this.Layout.GetClientInstance(this.AgentId); - string serverIp = (instance.Role == ClientRole.Server) ? "localhost" : this.GetServerIpAddress(); - this.sysbenchPrepareArguments += $" --host \"{serverIp}\""; - } + string serverIp = "localhost"; + + this.sysbenchPrepareArguments += $" --host \"{serverIp}\""; - string command = $"python3"; string arguments = $"{this.SysbenchPackagePath}/populate-database.py "; using (IProcessProxy process = await this.ExecuteCommandAsync( - command, + SysbenchExecutor.PythonCommand, arguments + this.sysbenchPrepareArguments, this.SysbenchPackagePath, telemetryContext, @@ -105,6 +102,8 @@ private async Task PrepareOLTPMySQLDatabase(EventContext telemetryContext, Cance { await this.LogProcessDetailsAsync(process, telemetryContext, "Sysbench", logToFile: true); process.ThrowIfErrored<WorkloadException>(process.StandardError.ToString(), ErrorReason.WorkloadUnexpectedAnomaly); + + this.AddMetric(sysbenchLoggingArguments, process, telemetryContext, cancellationToken); } } } @@ -115,13 +114,13 @@ private async Task PrepareTPCCMySQLDatabase(EventContext telemetryContext, Cance int threadCount = GetThreadCount(this.SystemManager, this.DatabaseScenario, this.Threads); int warehouseCount = GetWarehouseCount(this.DatabaseScenario, this.WarehouseCount); - this.sysbenchPrepareArguments = $"--dbName {this.DatabaseName} --databaseSystem {this.DatabaseSystem} --benchmark {this.Benchmark} --tableCount {tableCount} --warehouses {warehouseCount} --threadCount {threadCount} --password {this.SuperUserPassword}"; + string sysbenchLoggingArguments = $"--dbName {this.DatabaseName} --databaseSystem {this.DatabaseSystem} --benchmark {this.Benchmark} --tableCount {tableCount} --warehouses {warehouseCount} --threadCount {threadCount}"; + this.sysbenchPrepareArguments = $"{sysbenchLoggingArguments} --password {this.SuperUserPassword}"; - string command = $"python3"; string arguments = $"{this.SysbenchPackagePath}/populate-database.py "; using (IProcessProxy process = await this.ExecuteCommandAsync( - command, + SysbenchExecutor.PythonCommand, arguments + this.sysbenchPrepareArguments, this.SysbenchPackagePath, telemetryContext, @@ -131,6 +130,8 @@ private async Task PrepareTPCCMySQLDatabase(EventContext telemetryContext, Cance { await this.LogProcessDetailsAsync(process, telemetryContext, "Sysbench", logToFile: true); process.ThrowIfErrored<WorkloadException>(process.StandardError.ToString(), ErrorReason.WorkloadUnexpectedAnomaly); + + this.AddMetric(sysbenchLoggingArguments, process, telemetryContext, cancellationToken); } } } diff --git a/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchExecutor.cs b/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchExecutor.cs index b2b57c274f..e020517fc6 100644 --- a/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchExecutor.cs +++ b/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchExecutor.cs @@ -18,6 +18,7 @@ namespace VirtualClient.Actions using VirtualClient.Common.Extensions; using VirtualClient.Common.Telemetry; using VirtualClient.Contracts; + using VirtualClient.Contracts.Metadata; /// <summary> /// The Sysbench workload executor. @@ -35,6 +36,11 @@ public class SysbenchExecutor : VirtualClientComponent /// </summary> public const int SelectWorkloadDefaultTableCount = 1; + /// <summary> + /// const for python command. + /// </summary> + protected const string PythonCommand = "python3"; + private readonly IStateManager stateManager; private static readonly string[] SelectWorkloads = { @@ -393,20 +399,40 @@ protected async Task InitializeExecutablesAsync(EventContext telemetryContext, C } /// <summary> - /// + /// Add metrics to telemtry. /// </summary> - /// <returns></returns> - protected string GetServerIpAddress() + /// <param name="arguments"></param> + /// <param name="process"></param> + /// <param name="telemetryContext"></param> + /// <param name="cancellationToken"></param> + protected void AddMetric(string arguments, IProcessProxy process, EventContext telemetryContext, CancellationToken cancellationToken) { - string serverIPAddress = IPAddress.Loopback.ToString(); - - if (this.IsMultiRoleLayout()) + if (!cancellationToken.IsCancellationRequested) { - ClientInstance serverInstance = this.GetLayoutClientInstances(ClientRole.Server).First(); - serverIPAddress = serverInstance.IPAddress; + this.MetadataContract.AddForScenario( + "Sysbench", + process.FullCommand(), + toolVersion: null); + + this.MetadataContract.Apply(telemetryContext); + + string text = process.StandardOutput.ToString(); + + List<Metric> metrics = new List<Metric>(); + double duration = (process.ExitTime - process.StartTime).TotalMinutes; + metrics.Add(new Metric("PopulateDatabaseTime_Minutes ", duration, "minutes", MetricRelativity.LowerIsBetter)); + + this.Logger.LogMetrics( + toolName: "Sysbench", + scenarioName: this.MetricScenario ?? this.Scenario, + process.StartTime, + process.ExitTime, + metrics, + null, + scenarioArguments: arguments, + this.Tags, + telemetryContext); } - - return serverIPAddress; } private async Task CheckDistroSupportAsync(EventContext telemetryContext, CancellationToken cancellationToken) diff --git a/src/VirtualClient/VirtualClient.Contracts/Extensions/EnvironmentLayoutExtensions.cs b/src/VirtualClient/VirtualClient.Contracts/Extensions/EnvironmentLayoutExtensions.cs index 4d8ef8cb65..d19eb9f727 100644 --- a/src/VirtualClient/VirtualClient.Contracts/Extensions/EnvironmentLayoutExtensions.cs +++ b/src/VirtualClient/VirtualClient.Contracts/Extensions/EnvironmentLayoutExtensions.cs @@ -6,7 +6,6 @@ namespace VirtualClient.Contracts using System; using System.Collections.Generic; using System.Linq; - using System.Net; /// <summary> /// Extensions for <see cref="EnvironmentLayout"/> instances. diff --git a/src/VirtualClient/VirtualClient.Contracts/VirtualClientComponent.cs b/src/VirtualClient/VirtualClient.Contracts/VirtualClientComponent.cs index 84d20d9990..1b0b879567 100644 --- a/src/VirtualClient/VirtualClient.Contracts/VirtualClientComponent.cs +++ b/src/VirtualClient/VirtualClient.Contracts/VirtualClientComponent.cs @@ -625,7 +625,6 @@ public async Task ExecuteAsync(CancellationToken cancellationToken) if (this.Parameters?.Any() == true) { - Console.WriteLine("before adding parameters to dictonary"); this.MetadataContract.Add( this.Parameters.Keys.ToDictionary(key => key, entry => this.Parameters[entry] as object).ObscureSecrets(), MetadataContractCategory.Scenario, diff --git a/src/VirtualClient/VirtualClient.Core.UnitTests/ProfileExpressionEvaluatorTests.cs b/src/VirtualClient/VirtualClient.Core.UnitTests/ProfileExpressionEvaluatorTests.cs index d807196fc1..74254ff819 100644 --- a/src/VirtualClient/VirtualClient.Core.UnitTests/ProfileExpressionEvaluatorTests.cs +++ b/src/VirtualClient/VirtualClient.Core.UnitTests/ProfileExpressionEvaluatorTests.cs @@ -709,7 +709,6 @@ public async Task ProfileExpressionEvaluatorSupportsFunctionReferences() Dictionary<string, string> expressions = new Dictionary<string, string> { - {"{calculate(67419566080 * 800000 / 100)}", "23456"}, { "{calculate(512 + 2)}", "514" }, { "{calculate(512 - 2)}", "510" }, { "{calculate(512 * 2)}", "1024" }, diff --git a/src/VirtualClient/VirtualClient.Core/ProfileExpressionEvaluator.cs b/src/VirtualClient/VirtualClient.Core/ProfileExpressionEvaluator.cs index 2d0d0ee251..86cca7e54f 100644 --- a/src/VirtualClient/VirtualClient.Core/ProfileExpressionEvaluator.cs +++ b/src/VirtualClient/VirtualClient.Core/ProfileExpressionEvaluator.cs @@ -24,10 +24,6 @@ public class ProfileExpressionEvaluator : IExpressionEvaluator // e.g. // {fn(512 / 16)]} // {fn(512 / {LogicalThreadCount})} - /*private static readonly Regex CalculateExpression = new Regex( - @"\{calculate\(([0-9\*\/\+\-\(\)\s]+)\)\}", - RegexOptions.Compiled | RegexOptions.IgnoreCase);*/ - private static readonly Regex CalculateExpression = new Regex( @"\{calculate\(([0-9L\*\/\+\-\(\)\s]+)\)\}", RegexOptions.Compiled | RegexOptions.IgnoreCase); @@ -440,16 +436,12 @@ public class ProfileExpressionEvaluator : IExpressionEvaluator if (matches?.Any() == true) { - Console.WriteLine($"just before c# script"); isMatched = true; foreach (Match match in matches) { string function = match.Groups[1].Value; - Console.WriteLine($"just before c# script"); long result = await Microsoft.CodeAnalysis.CSharp.Scripting.CSharpScript.EvaluateAsync<long>(function); - Console.WriteLine($"the result after csharpscript is {result}"); evaluatedExpression = evaluatedExpression.Replace(match.Value, result.ToString()); - Console.WriteLine($"evaluated expression {evaluatedExpression}"); } } @@ -645,7 +637,6 @@ private static async Task<EvaluationResult> EvaluateExpressionAsync(IServiceColl { isMatched = true; evaluatedExpression = evaluation.Outcome; - Console.WriteLine($"Evaluated expression in evaluateexpressionasync {evaluatedExpression}"); } } } @@ -732,9 +723,7 @@ private static async Task<bool> EvaluateWellKnownExpressionsAsync(IServiceCollec EvaluationResult evaluation = await ProfileExpressionEvaluator.EvaluateExpressionAsync(dependencies, parameters, parameter.Value.ToString(), cancellationToken); if (evaluation.IsMatched) { - Console.WriteLine($"After evaluation in runtime {evaluation.Outcome}"); parameters[parameter.Key] = evaluation.Outcome; - Console.WriteLine($"After assigning it to parameter key in runtime {evaluation.Outcome}"); } } } diff --git a/src/VirtualClient/VirtualClient.Dependencies/MySqlServer/MySqlServerConfiguration.cs b/src/VirtualClient/VirtualClient.Dependencies/MySqlServer/MySqlServerConfiguration.cs index 7fbac2255c..9ad50f9257 100644 --- a/src/VirtualClient/VirtualClient.Dependencies/MySqlServer/MySqlServerConfiguration.cs +++ b/src/VirtualClient/VirtualClient.Dependencies/MySqlServer/MySqlServerConfiguration.cs @@ -64,13 +64,18 @@ public bool SkipInitialize } /// <summary> - /// stripeddisk mount point. + /// stripedisk mount point. /// </summary> public string StripeDiskMountPoint { get { - return this.Parameters.GetValue<string>(nameof(this.StripeDiskMountPoint), null); + if (this.Parameters.TryGetValue(nameof(this.StripeDiskMountPoint), out IConvertible stripediskmountpoint) && stripediskmountpoint != null) + { + return stripediskmountpoint.ToString(); + } + + return string.Empty; } } @@ -174,8 +179,11 @@ await this.SetMySQLGlobalVariableAsync(telemetryContext, cancellationToken) private async Task ConfigureMySQLServerAsync(EventContext telemetryContext, CancellationToken cancellationToken) { - string serverIp = this.GetServerIpAddress(); - string innoDbDirs = this.StripeDiskMountPoint != null ? this.StripeDiskMountPoint : await this.GetMySQLInnodbDirectoriesAsync(cancellationToken); + string serverIp = (this.GetLayoutClientInstances(ClientRole.Server, false) ?? Enumerable.Empty<ClientInstance>()) + .FirstOrDefault()?.IPAddress + ?? IPAddress.Loopback.ToString(); + // string serverIp = this.GetLayoutClientInstances(ClientRole.Server, false).FirstOrDefault()?.IPAddress ?? IPAddress.Loopback.ToString(); + string innoDbDirs = !string.IsNullOrEmpty(this.StripeDiskMountPoint) ? this.StripeDiskMountPoint : await this.GetMySQLInnodbDirectoriesAsync(cancellationToken); string arguments = $"{this.packageDirectory}/configure.py --serverIp {serverIp} --innoDbDirs \"{innoDbDirs}\""; @@ -310,19 +318,6 @@ private async Task<string> GetMySQLInMemoryCapacityAsync(CancellationToken cance return bufferSizeInMegaBytes.ToString(); } - private string GetServerIpAddress() - { - string serverIPAddress = IPAddress.Loopback.ToString(); - - if (this.IsMultiRoleLayout()) - { - ClientInstance serverInstance = this.GetLayoutClientInstances(ClientRole.Server).First(); - serverIPAddress = serverInstance.IPAddress; - } - - return serverIPAddress; - } - private string GetClientIpAddresses() { string clientIpAddresses = string.Empty; From b69150728d387eaf2850e1cd5047806c8a30f21e Mon Sep 17 00:00:00 2001 From: nikhitha malkapuram <nmalkapuram@microsoft.com> Date: Tue, 4 Feb 2025 13:36:19 +0530 Subject: [PATCH 11/11] updating documentation --- .../MySqlServer/MySqlServerConfiguration.cs | 2 +- .../PostgreSQLServerConfiguration.cs | 17 +++-------------- .../workloads/sysbench/sysbench-profiles.md | 5 +++++ 3 files changed, 9 insertions(+), 15 deletions(-) diff --git a/src/VirtualClient/VirtualClient.Dependencies/MySqlServer/MySqlServerConfiguration.cs b/src/VirtualClient/VirtualClient.Dependencies/MySqlServer/MySqlServerConfiguration.cs index 9ad50f9257..4ee17e5f28 100644 --- a/src/VirtualClient/VirtualClient.Dependencies/MySqlServer/MySqlServerConfiguration.cs +++ b/src/VirtualClient/VirtualClient.Dependencies/MySqlServer/MySqlServerConfiguration.cs @@ -182,7 +182,7 @@ private async Task ConfigureMySQLServerAsync(EventContext telemetryContext, Canc string serverIp = (this.GetLayoutClientInstances(ClientRole.Server, false) ?? Enumerable.Empty<ClientInstance>()) .FirstOrDefault()?.IPAddress ?? IPAddress.Loopback.ToString(); - // string serverIp = this.GetLayoutClientInstances(ClientRole.Server, false).FirstOrDefault()?.IPAddress ?? IPAddress.Loopback.ToString(); + string innoDbDirs = !string.IsNullOrEmpty(this.StripeDiskMountPoint) ? this.StripeDiskMountPoint : await this.GetMySQLInnodbDirectoriesAsync(cancellationToken); string arguments = $"{this.packageDirectory}/configure.py --serverIp {serverIp} --innoDbDirs \"{innoDbDirs}\""; diff --git a/src/VirtualClient/VirtualClient.Dependencies/PostgreSQLServer/PostgreSQLServerConfiguration.cs b/src/VirtualClient/VirtualClient.Dependencies/PostgreSQLServer/PostgreSQLServerConfiguration.cs index 7f005ed3e7..876f7d8ca3 100644 --- a/src/VirtualClient/VirtualClient.Dependencies/PostgreSQLServer/PostgreSQLServerConfiguration.cs +++ b/src/VirtualClient/VirtualClient.Dependencies/PostgreSQLServer/PostgreSQLServerConfiguration.cs @@ -177,7 +177,9 @@ await this.DistributePostgreSQLDatabaseAsync(telemetryContext, cancellationToken private async Task ConfigurePostgreSQLServerAsync(EventContext telemetryContext, CancellationToken cancellationToken) { - string serverIp = this.GetServerIpAddress(); + string serverIp = (this.GetLayoutClientInstances(ClientRole.Server, false) ?? Enumerable.Empty<ClientInstance>()) + .FirstOrDefault()?.IPAddress + ?? IPAddress.Loopback.ToString(); string arguments = $"{this.packageDirectory}/configure-server.py --dbName {this.DatabaseName} --serverIp {serverIp} --password {this.SuperUserPassword} --port {this.Port} --inMemory {this.SharedMemoryBuffer}"; @@ -272,19 +274,6 @@ private async Task<string> GetPostgreSQLInnodbDirectoriesAsync(CancellationToken return diskPaths; } - private string GetServerIpAddress() - { - string serverIPAddress = IPAddress.Loopback.ToString(); - - if (this.IsMultiRoleLayout()) - { - ClientInstance serverInstance = this.GetLayoutClientInstances(ClientRole.Server).First(); - serverIPAddress = serverInstance.IPAddress; - } - - return serverIPAddress; - } - /// <summary> /// Supported PostgreSQL Server configuration actions. /// </summary> diff --git a/website/docs/workloads/sysbench/sysbench-profiles.md b/website/docs/workloads/sysbench/sysbench-profiles.md index 5436412b7c..39823ec2fd 100644 --- a/website/docs/workloads/sysbench/sysbench-profiles.md +++ b/website/docs/workloads/sysbench/sysbench-profiles.md @@ -158,6 +158,11 @@ There are a lot of moving parts to this workload that allows for both out-of-box configuration within the same profile or system. VC now supports truncating tables and populating the table again. Following are changes to be made in profile. + + **Note:** Use TruncateDatabase with caution. + + ** **DO NOT CHANGE THE SIZE OF DATABASE FROM ACTION TO ACTION. IF DEALING WITH LARGE DATABASES RE-POPULATING DATABASE WOULD TAKE A LOT OF TIME AND IS NOT ALWAYS RECOMMENDED** ** + ``` bash { "Type": "SysbenchClientExecutor",