From fd4bd849dbc3ccbe8702e8b3d8c3b05614ee48c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stanis=C5=82aw=20Czech?= Date: Mon, 30 Mar 2026 14:35:47 +0200 Subject: [PATCH 1/4] Refactor benchmarking This commit moves the entry points for individual benchmarks to separate files. This change will allows us to easily create new benchmarks, that are parametrized on concurrency level, without the need to create a separate copy of the files. --- benchmark/README.md | 2 + benchmark/logic/batch.js | 73 +++++++++-------- benchmark/logic/benchmark.js | 27 +++++++ benchmark/logic/concurrent_deser.js | 79 +++++++++---------- benchmark/logic/concurrent_insert.js | 67 ++++++++-------- benchmark/logic/concurrent_paging.js | 74 +++++++++-------- benchmark/logic/concurrent_select.js | 66 ++++++++-------- benchmark/logic/concurrent_ser.js | 49 ++++++------ benchmark/logic/deser.js | 39 +++++---- benchmark/logic/insert.js | 55 +++++++------ benchmark/logic/large_select.js | 7 +- benchmark/logic/paging.js | 61 +++++++------- benchmark/logic/parametrized_select.js | 13 +-- benchmark/logic/select.js | 7 +- benchmark/logic/ser.js | 39 +++++---- .../runner-config/cassandra-driver/config.yml | 24 +++--- benchmark/runner-config/config.yml | 10 +++ benchmark/runner-config/run-js-benchmark.sh | 4 +- .../runner-config/scylladb-driver/config.yml | 24 +++--- 19 files changed, 375 insertions(+), 345 deletions(-) create mode 100644 benchmark/logic/benchmark.js diff --git a/benchmark/README.md b/benchmark/README.md index 72fc97730..876d01303 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -63,6 +63,8 @@ Result will be saved to `out.svg`. You can read more in the benchmarker document ## Legacy python script +This script is no longer updated, so it may be broken. + A file that runs all benchmarks: `runner.py` The script compares benchmark results for our driver, [Cassandra driver](https://github.com/apache/cassandra-nodejs-driver) and [Rust driver](https://github.com/scylladb/scylla-rust-driver). Parameters for the benchmarks can be modified inside it. The result is a `graph.png` file that presents a graph of time on a logarithmic scale. The graphs are uploaded to the provided discord webhook. diff --git a/benchmark/logic/batch.js b/benchmark/logic/batch.js index 16a22005a..babacfd0d 100644 --- a/benchmark/logic/batch.js +++ b/benchmark/logic/batch.js @@ -1,50 +1,49 @@ "use strict"; const async = require("async"); -// Possible values of argv[2] (driver) are scylladb-driver-alpha and cassandra-driver. -const cassandra = require(process.argv[2]); const utils = require("./utils"); const { exit } = require("process"); const assert = require("assert"); -const client = new cassandra.Client(utils.getClientArgs()); -const iterCnt = parseInt(process.argv[3]); -// Expectantly determined max batch size, that doesn't cause database error. -const step = 3971; +module.exports = function (cassandra, client, stepCount, _concurrencyLevel) { + // REMEMBER: update benchmark config.yml when changing the constant value. + const iterCnt = stepCount || 3000000; + // Experimentally determined max batch size that doesn't cause database error. + const batchSize = 3971; -async.series( - [ - function initialize(next) { - utils.prepareDatabase(client, utils.tableSchemaBasic, next); - }, - async function insert(next) { - // Limit batch size to step size - for (let z = 0; z * step < iterCnt; z++) { - let queries = []; - for (let i = 0; i < Math.min(iterCnt - (z * step), step); i++) { - queries.push({ - query: 'INSERT INTO benchmarks.basic (id, val) VALUES (?, ?)', - params: [cassandra.types.Uuid.random(), 10] - }); + async.series( + [ + function initialize(next) { + utils.prepareDatabase(client, utils.tableSchemaBasic, next); + }, + async function insert(next) { + for (let z = 0; z * batchSize < iterCnt; z++) { + let queries = []; + for (let i = 0; i < Math.min(iterCnt - (z * batchSize), batchSize); i++) { + queries.push({ + query: 'INSERT INTO benchmarks.basic (id, val) VALUES (?, ?)', + params: [cassandra.types.Uuid.random(), 10] + }); + } + try { + await client.batch(queries, { prepare: true }); + } catch (err) { + return next(err); + } } + next(); + }, + async function test(next) { + const query = "SELECT COUNT(1) FROM benchmarks.basic USING TIMEOUT 120s;"; try { - await client.batch(queries, { prepare: true }); + let res = await client.execute(query); + assert(res.rows[0].count == iterCnt); } catch (err) { return next(err); } + next(); + }, + function r() { + exit(0); } - next(); - }, - async function test(next) { - const query = "SELECT COUNT(1) FROM benchmarks.basic USING TIMEOUT 120s;"; - try { - let res = await client.execute(query); - assert(res.rows[0].count == iterCnt); - } catch (err) { - return next(err); - } - next(); - }, - function r() { - exit(0); - } - ], utils.onError); + ], utils.onError); +}; diff --git a/benchmark/logic/benchmark.js b/benchmark/logic/benchmark.js new file mode 100644 index 000000000..c022ef942 --- /dev/null +++ b/benchmark/logic/benchmark.js @@ -0,0 +1,27 @@ +// Single entry point for all benchmarks. +// This file takes the name of the benchmark, including driver name, step count and concurrency level +// and runs the corresponding benchmark logic, loaded at runtime. + +"use strict"; +const { exit } = require("process"); +const utils = require("./utils"); + +const defaultConcurrencyLevel = 100; + +const driverName = process.argv[2]; +const benchmarkName = process.argv[3]; +// Each individual benchmark has a default step count defined. +// See the explanation in config.yml for more details. +const stepCount = process.argv[4] !== "default" ? parseInt(process.argv[4], 10) : undefined; +const concurrencyLevel = process.argv[5] !== "default" ? parseInt(process.argv[5], 10) : defaultConcurrencyLevel; + +if (!driverName || !benchmarkName) { + console.error("Usage: node benchmark.js [step-count] [concurrency]"); + exit(1); +} + +const cassandra = require(driverName); +const client = new cassandra.Client(utils.getClientArgs()); + +const benchmark = require(`./${benchmarkName}`); +benchmark(cassandra, client, stepCount, concurrencyLevel); diff --git a/benchmark/logic/concurrent_deser.js b/benchmark/logic/concurrent_deser.js index 07dd63b45..d452f92e2 100644 --- a/benchmark/logic/concurrent_deser.js +++ b/benchmark/logic/concurrent_deser.js @@ -1,52 +1,51 @@ "use strict"; const async = require("async"); -// Possible values of argv[2] (driver) are scylladb-driver-alpha and cassandra-driver. -const cassandra = require(process.argv[2]); const utils = require("./utils"); const { exit } = require("process"); const assert = require("assert"); -const client = new cassandra.Client(utils.getClientArgs()); -const iterCnt = parseInt(process.argv[3]); +module.exports = function (cassandra, client, stepCount, concurrencyLevel) { + // REMEMBER: update benchmark config.yml when changing the constant value. + const iterCnt = stepCount || 2000; -async.series( - [ - function initialize(next) { - utils.prepareDatabase(client, utils.tableSchemaDeSer, next); - }, - async function insert(next) { - let allParameters = utils.insertConcurrentDeSer(cassandra, iterCnt); - try { - const _result = await cassandra.concurrent.executeConcurrent(client, allParameters, { prepare: true }); - } catch (err) { - return next(err); - } - next(); - }, - async function select(next) { - let remaining = iterCnt; - while (remaining > 0) { - let currentStep = Math.min(remaining, 500); - remaining -= currentStep; - let allParameters = []; - for (let i = 0; i < currentStep; i++) { - allParameters.push({ - query: 'SELECT * FROM benchmarks.basic', - }); - } + async.series( + [ + function initialize(next) { + utils.prepareDatabase(client, utils.tableSchemaDeSer, next); + }, + async function insert(next) { + let allParameters = utils.insertConcurrentDeSer(cassandra, iterCnt); try { - const result = await cassandra.concurrent.executeConcurrent(client, allParameters, { prepare: true, collectResults: true }); - for (let singleResult of result.resultItems) { - assert.equal(singleResult.rowLength, iterCnt); - } + const _result = await cassandra.concurrent.executeConcurrent(client, allParameters, { prepare: true, concurrencyLevel }); } catch (err) { return next(err); } + next(); + }, + async function select(next) { + let remaining = iterCnt; + while (remaining > 0) { + let currentStep = Math.min(remaining, 500); + remaining -= currentStep; + let allParameters = []; + for (let i = 0; i < currentStep; i++) { + allParameters.push({ + query: 'SELECT * FROM benchmarks.basic', + }); + } + try { + const result = await cassandra.concurrent.executeConcurrent(client, allParameters, { prepare: true, collectResults: true, concurrencyLevel }); + for (let singleResult of result.resultItems) { + assert.equal(singleResult.rowLength, iterCnt); + } + } catch (err) { + return next(err); + } + } + next(); + }, + function r() { + exit(0); } - next(); - }, - function r() { - exit(0); - } - ], utils.onError); - + ], utils.onError); +}; diff --git a/benchmark/logic/concurrent_insert.js b/benchmark/logic/concurrent_insert.js index 5fb1e0ae6..277b9e833 100644 --- a/benchmark/logic/concurrent_insert.js +++ b/benchmark/logic/concurrent_insert.js @@ -1,42 +1,41 @@ "use strict"; const async = require("async"); -// Possible values of argv[2] (driver) are scylladb-driver-alpha and cassandra-driver. -const cassandra = require(process.argv[2]); const utils = require("./utils"); const { exit } = require("process"); -const client = new cassandra.Client(utils.getClientArgs()); -const iterCnt = parseInt(process.argv[3]); +module.exports = function (cassandra, client, stepCount, concurrencyLevel) { + // REMEMBER: update benchmark config.yml when changing the constant value. + const iterCnt = stepCount || 4000000; -async.series( - [ - function initialize(next) { - utils.prepareDatabase(client, utils.tableSchemaBasic, next); - }, - async function insert(next) { - let limited = async function (steps) { - let allParameters = []; - for (let i = 0; i < steps; i++) { - allParameters.push({ - query: 'INSERT INTO benchmarks.basic (id, val) VALUES (?, ?)', - params: [cassandra.types.Uuid.random(), 10] - }); + async.series( + [ + function initialize(next) { + utils.prepareDatabase(client, utils.tableSchemaBasic, next); + }, + async function insert(next) { + let limited = async function (steps) { + let allParameters = []; + for (let i = 0; i < steps; i++) { + allParameters.push({ + query: 'INSERT INTO benchmarks.basic (id, val) VALUES (?, ?)', + params: [cassandra.types.Uuid.random(), 10] + }); + } + try { + const _result = await cassandra.concurrent.executeConcurrent(client, allParameters, { prepare: true, concurrencyLevel }); + } catch (err) { + return next(err); + } } - try { - const _result = await cassandra.concurrent.executeConcurrent(client, allParameters, { prepare: true }); - } catch (err) { - return next(err); - } - } - await utils.repeatCapped(limited, iterCnt); - - next(); - }, - async function test(next) { - utils.checkRowCount(client, iterCnt, next); - }, - function r() { - exit(0); - } - ], utils.onError); + await utils.repeatCapped(limited, iterCnt); + next(); + }, + async function test(next) { + utils.checkRowCount(client, iterCnt, next); + }, + function r() { + exit(0); + } + ], utils.onError); +}; diff --git a/benchmark/logic/concurrent_paging.js b/benchmark/logic/concurrent_paging.js index 2fc95066b..a06dbbee7 100644 --- a/benchmark/logic/concurrent_paging.js +++ b/benchmark/logic/concurrent_paging.js @@ -1,48 +1,46 @@ "use strict"; const async = require("async"); -// Possible values of argv[2] (driver) are scylladb-nodejs-rs-driver and cassandra-driver. -const cassandra = require(process.argv[2]); const utils = require("./utils"); const { exit } = require("process"); -const { assert } = require("console"); +const assert = require("assert"); -const client = new cassandra.Client(utils.getClientArgs()); -const iterCnt = parseInt(process.argv[3]); -const concurrencyLevel = 20; +module.exports = function (cassandra, client, stepCount, concurrencyLevel) { + // REMEMBER: update benchmark config.yml when changing the constant value. + const iterCnt = stepCount || 1280; -async.series( - [ - function initialize(next) { - utils.prepareDatabase(client, utils.tableSchemaBasic, next); - }, - async function insert(next) { - utils.insertSimple(client, 50, next); - }, - async function select(next) { - let limited = async function (steps) { - for (let i = 0; i < steps; i++) { - try { - let s = 0; - let q = await client.execute('SELECT * FROM benchmarks.basic', [], { prepare: true, fetchSize: 1 }); - for await (const row of q) { - s += row['val']; + async.series( + [ + function initialize(next) { + utils.prepareDatabase(client, utils.tableSchemaBasic, next); + }, + async function insert(next) { + utils.insertSimple(client, 50, next); + }, + async function select(next) { + let limited = async function (steps) { + for (let i = 0; i < steps; i++) { + try { + let s = 0; + let q = await client.execute('SELECT * FROM benchmarks.basic', [], { prepare: true, fetchSize: 1 }); + for await (const row of q) { + s += row['val']; + } + assert(s === 5000); + } catch (err) { + return next(err); } - assert(s === 5000); - } catch (err) { - return next(err); } - } + await utils.executeMultipleRepeatCapped(limited, iterCnt, concurrencyLevel); + next(); + }, + function r() { + exit(0); + } + ], function (err) { + if (err) { + console.error("Error: ", err.message, err.stack); + exit(1); } - await utils.executeMultipleRepeatCapped(limited, iterCnt, concurrencyLevel); - next(); - }, - function r() { - exit(0); - } - ], function (err) { - if (err) { - console.error("Error: ", err.message, err.stack); - exit(1); - } - },); + }); +}; diff --git a/benchmark/logic/concurrent_select.js b/benchmark/logic/concurrent_select.js index 69e507b47..7af105f4b 100644 --- a/benchmark/logic/concurrent_select.js +++ b/benchmark/logic/concurrent_select.js @@ -1,43 +1,43 @@ "use strict"; const async = require("async"); -// Possible values of argv[2] (driver) are scylladb-driver-alpha and cassandra-driver. -const cassandra = require(process.argv[2]); const utils = require("./utils"); const { exit } = require("process"); const assert = require("assert"); -const client = new cassandra.Client(utils.getClientArgs()); -const iterCnt = parseInt(process.argv[3]); +module.exports = function (cassandra, client, stepCount, concurrencyLevel) { + // REMEMBER: update benchmark config.yml when changing the constant value. + const iterCnt = stepCount || 400000; -async.series( - [ - function initialize(next) { - utils.prepareDatabase(client, utils.tableSchemaBasic, next); - }, - async function insert(next) { - utils.insertSimple(client, 10, next); - }, - async function select(next) { - let limited = async function (steps) { - let allParameters = []; - for (let i = 0; i < steps; i++) { - allParameters.push({ - query: 'SELECT * FROM benchmarks.basic', - }); - } - try { - const result = await cassandra.concurrent.executeConcurrent(client, allParameters, { prepare: true, collectResults: true }); - for (let singleResult of result.resultItems) { - assert.equal(singleResult.rowLength, 10); + async.series( + [ + function initialize(next) { + utils.prepareDatabase(client, utils.tableSchemaBasic, next); + }, + async function insert(next) { + utils.insertSimple(client, 10, next); + }, + async function select(next) { + let limited = async function (steps) { + let allParameters = []; + for (let i = 0; i < steps; i++) { + allParameters.push({ + query: 'SELECT * FROM benchmarks.basic', + }); + } + try { + const result = await cassandra.concurrent.executeConcurrent(client, allParameters, { prepare: true, collectResults: true, concurrencyLevel }); + for (let singleResult of result.resultItems) { + assert.equal(singleResult.rowLength, 10); + } + } catch (err) { + return next(err); } - } catch (err) { - return next(err); } + await utils.repeatCapped(limited, iterCnt); + next(); + }, + function r() { + exit(0); } - await utils.repeatCapped(limited, iterCnt); - next(); - }, - function r() { - exit(0); - } - ], utils.onError); + ], utils.onError); +}; diff --git a/benchmark/logic/concurrent_ser.js b/benchmark/logic/concurrent_ser.js index 5f5c71b8d..3c30fe975 100644 --- a/benchmark/logic/concurrent_ser.js +++ b/benchmark/logic/concurrent_ser.js @@ -1,32 +1,31 @@ "use strict"; const async = require("async"); -// Possible values of argv[2] (driver) are scylladb-driver-alpha and cassandra-driver. -const cassandra = require(process.argv[2]); const utils = require("./utils"); const { exit } = require("process"); -const client = new cassandra.Client(utils.getClientArgs()); -const iterCnt = parseInt(process.argv[3]); +module.exports = function (cassandra, client, stepCount, concurrencyLevel) { + // REMEMBER: update benchmark config.yml when changing the constant value. + const iterCnt = stepCount || 1200; -async.series( - [ - function initialize(next) { - utils.prepareDatabase(client, utils.tableSchemaDeSer, next); - }, - async function insert(next) { - let allParameters = utils.insertConcurrentDeSer(cassandra, iterCnt * iterCnt); - try { - await cassandra.concurrent.executeConcurrent(client, allParameters, { prepare: true }); - } catch (err) { - return next(err); + async.series( + [ + function initialize(next) { + utils.prepareDatabase(client, utils.tableSchemaDeSer, next); + }, + async function insert(next) { + let allParameters = utils.insertConcurrentDeSer(cassandra, iterCnt * iterCnt); + try { + await cassandra.concurrent.executeConcurrent(client, allParameters, { prepare: true, concurrencyLevel }); + } catch (err) { + return next(err); + } + next(); + }, + async function test(next) { + utils.checkRowCount(client, iterCnt * iterCnt, next); + }, + function r() { + exit(0); } - next(); - }, - async function test(next) { - utils.checkRowCount(client, iterCnt * iterCnt, next); - }, - function r() { - exit(0); - } - ], utils.onError); - + ], utils.onError); +}; diff --git a/benchmark/logic/deser.js b/benchmark/logic/deser.js index da0b9243f..92efa4322 100644 --- a/benchmark/logic/deser.js +++ b/benchmark/logic/deser.js @@ -1,26 +1,25 @@ "use strict"; const async = require("async"); -// Possible values of argv[2] (driver) are scylladb-driver-alpha and cassandra-driver. -const cassandra = require(process.argv[2]); const utils = require("./utils"); const { exit } = require("process"); -const client = new cassandra.Client(utils.getClientArgs()); -const iterCount = parseInt(process.argv[3]); - -async.series( - [ - function initialize(next) { - utils.prepareDatabase(client, utils.tableSchemaDeSer, next); - }, - async function insert(next) { - utils.executeInsertDeSer(client, iterCount, cassandra, next); - }, - async function query(next) { - await utils.queryWithRowCheck(client, iterCount, iterCount, next); - }, - function r() { - exit(0); - } - ], utils.onError); +module.exports = function (cassandra, client, stepCount, _concurrencyLevel) { + // REMEMBER: update benchmark config.yml when changing the constant value. + const iterCount = stepCount || 2000; + async.series( + [ + function initialize(next) { + utils.prepareDatabase(client, utils.tableSchemaDeSer, next); + }, + async function insert(next) { + utils.executeInsertDeSer(client, iterCount, cassandra, next); + }, + async function query(next) { + await utils.queryWithRowCheck(client, iterCount, iterCount, next); + }, + function r() { + exit(0); + } + ], utils.onError); +}; diff --git a/benchmark/logic/insert.js b/benchmark/logic/insert.js index 817c6be42..a41b3c694 100644 --- a/benchmark/logic/insert.js +++ b/benchmark/logic/insert.js @@ -1,36 +1,35 @@ "use strict"; const async = require("async"); -// Possible values of argv[2] (driver) are scylladb-driver-alpha and cassandra-driver. -const cassandra = require(process.argv[2]); const utils = require("./utils"); const { exit } = require("process"); -const client = new cassandra.Client(utils.getClientArgs()); -const iterCnt = parseInt(process.argv[3]); +module.exports = function (cassandra, client, stepCount, _concurrencyLevel) { + // REMEMBER: update benchmark config.yml when changing the constant value. + const iterCnt = stepCount || 400000; -async.series( - [ - function initialize(next) { - utils.prepareDatabase(client, utils.tableSchemaBasic, next); - }, - async function insert(next) { - for (let i = 0; i < iterCnt; i++) { - const id = cassandra.types.Uuid.random(); - const query = - "INSERT INTO benchmarks.basic (id, val) VALUES (?, ?)"; - try { - await client.execute(query, [id, 100], { prepare: true }); - } catch (err) { - return next(err); + async.series( + [ + function initialize(next) { + utils.prepareDatabase(client, utils.tableSchemaBasic, next); + }, + async function insert(next) { + for (let i = 0; i < iterCnt; i++) { + const id = cassandra.types.Uuid.random(); + const query = + "INSERT INTO benchmarks.basic (id, val) VALUES (?, ?)"; + try { + await client.execute(query, [id, 100], { prepare: true }); + } catch (err) { + return next(err); + } } + next(); + }, + async function test(next) { + utils.checkRowCount(client, iterCnt, next); + }, + function r() { + exit(0); } - next(); - }, - async function test(next) { - utils.checkRowCount(client, iterCnt, next); - }, - function r() { - exit(0); - } - ], utils.onError); - + ], utils.onError); +}; diff --git a/benchmark/logic/large_select.js b/benchmark/logic/large_select.js index 1283c062a..7aed913c5 100644 --- a/benchmark/logic/large_select.js +++ b/benchmark/logic/large_select.js @@ -1,5 +1,8 @@ "use strict"; - const selectWithRows = require("./parametrized_select"); -selectWithRows(5000); +module.exports = function (cassandra, client, stepCount, _concurrencyLevel) { + // REMEMBER: update benchmark config.yml when changing the constant value. + stepCount = stepCount || 4000; + selectWithRows(cassandra, client, 5000, stepCount); +}; diff --git a/benchmark/logic/paging.js b/benchmark/logic/paging.js index 40a967b09..13550f0fb 100644 --- a/benchmark/logic/paging.js +++ b/benchmark/logic/paging.js @@ -1,42 +1,41 @@ "use strict"; const async = require("async"); -// Possible values of argv[2] (driver) are scylladb-nodejs-rs-driver and cassandra-driver. -const cassandra = require(process.argv[2]); const utils = require("./utils"); const { exit } = require("process"); const assert = require("assert"); -const client = new cassandra.Client(utils.getClientArgs()); -const iterCnt = parseInt(process.argv[3]); +module.exports = function (cassandra, client, stepCount, _concurrencyLevel) { + // REMEMBER: update benchmark config.yml when changing the constant value. + const iterCnt = stepCount || 4000; -async.series( - [ - function initialize(next) { - utils.prepareDatabase(client, utils.tableSchemaBasic, next); - }, - async function insert(next) { - utils.insertSimple(client, 50, next); - }, - async function select(next) { - let limited = async function (steps) { - for (let i = 0; i < steps; i++) { - try { - let s = 0; - let q = await client.execute('SELECT * FROM benchmarks.basic', [], { prepare: true, fetchSize: 1 }); - for await (const row of q) { - s += row['val']; + async.series( + [ + function initialize(next) { + utils.prepareDatabase(client, utils.tableSchemaBasic, next); + }, + async function insert(next) { + utils.insertSimple(client, 50, next); + }, + async function select(next) { + let limited = async function (steps) { + for (let i = 0; i < steps; i++) { + try { + let s = 0; + let q = await client.execute('SELECT * FROM benchmarks.basic', [], { prepare: true, fetchSize: 1 }); + for await (const row of q) { + s += row['val']; + } + assert.equal(s, 5000); + } catch (err) { + return next(err); } - assert.equal(s, 5000); - } catch (err) { - return next(err); } - } + await utils.repeatCapped(limited, iterCnt); + next(); + }, + function r() { + exit(0); } - await utils.repeatCapped(limited, iterCnt); - next(); - }, - function r() { - exit(0); - } - ], utils.onError); + ], utils.onError); +}; diff --git a/benchmark/logic/parametrized_select.js b/benchmark/logic/parametrized_select.js index caef6876c..695aa1871 100644 --- a/benchmark/logic/parametrized_select.js +++ b/benchmark/logic/parametrized_select.js @@ -1,31 +1,26 @@ "use strict"; const async = require("async"); -// Possible values of argv[2] (driver) are scylladb-driver-alpha and cassandra-driver. -const cassandra = require(process.argv[2]); const utils = require("./utils"); const { exit } = require("process"); -const client = new cassandra.Client(utils.getClientArgs()); -const iterCnt = parseInt(process.argv[3]); +function selectWithRows(cassandra, client, rowCount, stepCount) { + const iterCnt = stepCount; -function selectWithRows(number) { async.series( [ function initialize(next) { utils.prepareDatabase(client, utils.tableSchemaBasic, next); }, async function insert(next) { - utils.insertSimple(client, 10, next); + utils.insertSimple(client, rowCount, next); }, async function query(next) { - await utils.queryWithRowCheck(client, number, iterCnt, next); + await utils.queryWithRowCheck(client, rowCount, iterCnt, next); }, function r() { exit(0); } - ], utils.onError); } - module.exports = selectWithRows; diff --git a/benchmark/logic/select.js b/benchmark/logic/select.js index a0e948a14..09f2b7593 100644 --- a/benchmark/logic/select.js +++ b/benchmark/logic/select.js @@ -1,5 +1,8 @@ "use strict"; - const selectWithRows = require("./parametrized_select"); -selectWithRows(10); +module.exports = function (cassandra, client, stepCount, _concurrencyLevel) { + // REMEMBER: update benchmark config.yml when changing the constant value. + stepCount = stepCount || 100000; + selectWithRows(cassandra, client, 10, stepCount); +}; diff --git a/benchmark/logic/ser.js b/benchmark/logic/ser.js index d8e0aa6f4..f4627a3e8 100644 --- a/benchmark/logic/ser.js +++ b/benchmark/logic/ser.js @@ -1,26 +1,25 @@ "use strict"; const async = require("async"); -// Possible values of argv[2] (driver) are scylladb-driver-alpha and cassandra-driver. -const cassandra = require(process.argv[2]); const utils = require("./utils"); const { exit } = require("process"); -const client = new cassandra.Client(utils.getClientArgs()); -const iterCount = parseInt(process.argv[3]); - -async.series( - [ - function initialize(next) { - utils.prepareDatabase(client, utils.tableSchemaDeSer, next); - }, - async function insert(next) { - utils.executeInsertDeSer(client, iterCount * iterCount, cassandra, next); - }, - async function test(next) { - utils.checkRowCount(client, iterCount * iterCount, next); - }, - function r() { - exit(0); - } - ], utils.onError); +module.exports = function (cassandra, client, stepCount, _concurrencyLevel) { + // REMEMBER: update benchmark config.yml when changing the constant value. + const iterCount = stepCount || 900; + async.series( + [ + function initialize(next) { + utils.prepareDatabase(client, utils.tableSchemaDeSer, next); + }, + async function insert(next) { + utils.executeInsertDeSer(client, iterCount * iterCount, cassandra, next); + }, + async function test(next) { + utils.checkRowCount(client, iterCount * iterCount, next); + }, + function r() { + exit(0); + } + ], utils.onError); +}; diff --git a/benchmark/runner-config/cassandra-driver/config.yml b/benchmark/runner-config/cassandra-driver/config.yml index dbda79fe6..750c197e1 100644 --- a/benchmark/runner-config/cassandra-driver/config.yml +++ b/benchmark/runner-config/cassandra-driver/config.yml @@ -4,37 +4,37 @@ defaults: backends: - benchmark-name: insert - run-command: ./benchmark/runner-config/run-js-benchmark.sh insert.js cassandra-driver + run-command: ./benchmark/runner-config/run-js-benchmark.sh insert cassandra-driver - benchmark-name: concurrent_insert - run-command: ./benchmark/runner-config/run-js-benchmark.sh concurrent_insert.js cassandra-driver + run-command: ./benchmark/runner-config/run-js-benchmark.sh concurrent_insert cassandra-driver - benchmark-name: select - run-command: ./benchmark/runner-config/run-js-benchmark.sh select.js cassandra-driver + run-command: ./benchmark/runner-config/run-js-benchmark.sh select cassandra-driver - benchmark-name: concurrent_select - run-command: ./benchmark/runner-config/run-js-benchmark.sh concurrent_select.js cassandra-driver + run-command: ./benchmark/runner-config/run-js-benchmark.sh concurrent_select cassandra-driver - benchmark-name: batch - run-command: ./benchmark/runner-config/run-js-benchmark.sh batch.js cassandra-driver + run-command: ./benchmark/runner-config/run-js-benchmark.sh batch cassandra-driver - benchmark-name: paging - run-command: ./benchmark/runner-config/run-js-benchmark.sh paging.js cassandra-driver + run-command: ./benchmark/runner-config/run-js-benchmark.sh paging cassandra-driver - benchmark-name: concurrent_paging - run-command: ./benchmark/runner-config/run-js-benchmark.sh concurrent_paging.js cassandra-driver + run-command: ./benchmark/runner-config/run-js-benchmark.sh concurrent_paging cassandra-driver - benchmark-name: large_select - run-command: ./benchmark/runner-config/run-js-benchmark.sh large_select.js cassandra-driver + run-command: ./benchmark/runner-config/run-js-benchmark.sh large_select cassandra-driver - benchmark-name: deser - run-command: ./benchmark/runner-config/run-js-benchmark.sh deser.js cassandra-driver + run-command: ./benchmark/runner-config/run-js-benchmark.sh deser cassandra-driver - benchmark-name: concurrent_deser - run-command: ./benchmark/runner-config/run-js-benchmark.sh concurrent_deser.js cassandra-driver + run-command: ./benchmark/runner-config/run-js-benchmark.sh concurrent_deser cassandra-driver - benchmark-name: ser - run-command: ./benchmark/runner-config/run-js-benchmark.sh ser.js cassandra-driver + run-command: ./benchmark/runner-config/run-js-benchmark.sh ser cassandra-driver - benchmark-name: concurrent_ser - run-command: ./benchmark/runner-config/run-js-benchmark.sh concurrent_ser.js cassandra-driver + run-command: ./benchmark/runner-config/run-js-benchmark.sh concurrent_ser cassandra-driver diff --git a/benchmark/runner-config/config.yml b/benchmark/runner-config/config.yml index 8375dece9..faefdf96c 100644 --- a/benchmark/runner-config/config.yml +++ b/benchmark/runner-config/config.yml @@ -6,6 +6,16 @@ defaults: measure: time timeout: "200s" +# We want to have the benchmark run for about 30-90 seconds at the highest step count. +# If, at some point (ex. due to driver optimizations), the benchmark starts to run faster, +# it's best to increase the default step count, to keep the benchmark results relevant and consistent. +# The last step is defined in each benchmark code file. Ex. for insert benchmark (insert.js): +# const iterCnt = stepCount || 400000; +# At this value the benchmark takes ~40 seconds to run. We then want to define the starting step in this config file. +# This definition should be picked in a way that matches the last step: +# Ex. insert: last step = starting step * (step-progress ** (no-steps - 1)) = 6250 * (4 ** 3) = 400000 +# We chose 6250 as starting step, which is the exact value that sets the last step to 400000. + benchmarks: - name: insert starting-step: 6250 diff --git a/benchmark/runner-config/run-js-benchmark.sh b/benchmark/runner-config/run-js-benchmark.sh index 97569dfc6..087acee0e 100755 --- a/benchmark/runner-config/run-js-benchmark.sh +++ b/benchmark/runner-config/run-js-benchmark.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # Wrapper on the command for benchmark runner -# Usage: run-js-benchmark.sh +# Usage: run-js-benchmark.sh set -euo pipefail BENCHMARK="$1" @@ -10,4 +10,4 @@ N="$3" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" BENCHMARK_DIR="$SCRIPT_DIR/.." -node "$BENCHMARK_DIR/logic/$BENCHMARK" "$DRIVER" "$N" +node "$BENCHMARK_DIR/logic/benchmark.js" "$DRIVER" "$BENCHMARK" "$N" "default" diff --git a/benchmark/runner-config/scylladb-driver/config.yml b/benchmark/runner-config/scylladb-driver/config.yml index 5049fa5d0..91a0e7ce7 100644 --- a/benchmark/runner-config/scylladb-driver/config.yml +++ b/benchmark/runner-config/scylladb-driver/config.yml @@ -4,37 +4,37 @@ defaults: backends: - benchmark-name: insert - run-command: ./benchmark/runner-config/run-js-benchmark.sh insert.js scylladb-driver-alpha + run-command: ./benchmark/runner-config/run-js-benchmark.sh insert scylladb-driver-alpha - benchmark-name: concurrent_insert - run-command: ./benchmark/runner-config/run-js-benchmark.sh concurrent_insert.js scylladb-driver-alpha + run-command: ./benchmark/runner-config/run-js-benchmark.sh concurrent_insert scylladb-driver-alpha - benchmark-name: select - run-command: ./benchmark/runner-config/run-js-benchmark.sh select.js scylladb-driver-alpha + run-command: ./benchmark/runner-config/run-js-benchmark.sh select scylladb-driver-alpha - benchmark-name: concurrent_select - run-command: ./benchmark/runner-config/run-js-benchmark.sh concurrent_select.js scylladb-driver-alpha + run-command: ./benchmark/runner-config/run-js-benchmark.sh concurrent_select scylladb-driver-alpha - benchmark-name: batch - run-command: ./benchmark/runner-config/run-js-benchmark.sh batch.js scylladb-driver-alpha + run-command: ./benchmark/runner-config/run-js-benchmark.sh batch scylladb-driver-alpha - benchmark-name: paging - run-command: ./benchmark/runner-config/run-js-benchmark.sh paging.js scylladb-driver-alpha + run-command: ./benchmark/runner-config/run-js-benchmark.sh paging scylladb-driver-alpha - benchmark-name: concurrent_paging - run-command: ./benchmark/runner-config/run-js-benchmark.sh concurrent_paging.js scylladb-driver-alpha + run-command: ./benchmark/runner-config/run-js-benchmark.sh concurrent_paging scylladb-driver-alpha - benchmark-name: large_select - run-command: ./benchmark/runner-config/run-js-benchmark.sh large_select.js scylladb-driver-alpha + run-command: ./benchmark/runner-config/run-js-benchmark.sh large_select scylladb-driver-alpha - benchmark-name: deser - run-command: ./benchmark/runner-config/run-js-benchmark.sh deser.js scylladb-driver-alpha + run-command: ./benchmark/runner-config/run-js-benchmark.sh deser scylladb-driver-alpha - benchmark-name: concurrent_deser - run-command: ./benchmark/runner-config/run-js-benchmark.sh concurrent_deser.js scylladb-driver-alpha + run-command: ./benchmark/runner-config/run-js-benchmark.sh concurrent_deser scylladb-driver-alpha - benchmark-name: ser - run-command: ./benchmark/runner-config/run-js-benchmark.sh ser.js scylladb-driver-alpha + run-command: ./benchmark/runner-config/run-js-benchmark.sh ser scylladb-driver-alpha - benchmark-name: concurrent_ser - run-command: ./benchmark/runner-config/run-js-benchmark.sh concurrent_ser.js scylladb-driver-alpha + run-command: ./benchmark/runner-config/run-js-benchmark.sh concurrent_ser scylladb-driver-alpha From 221c3438d5922e2be8192c5d8c9048df2e5764e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stanis=C5=82aw=20Czech?= Date: Mon, 30 Mar 2026 12:09:15 +0200 Subject: [PATCH 2/4] Add benchmark that are parametrized on concurrency This adds a new version of the existing benchmarks, that try to parameterize the concurrency level, rather then the number of iterations. --- .../runner-config/cassandra-driver/config.yml | 15 ++++++++++++++ benchmark/runner-config/config.yml | 20 +++++++++++++++++++ .../run-js-concurrency-benchmark.sh | 15 ++++++++++++++ .../runner-config/scylladb-driver/config.yml | 15 ++++++++++++++ 4 files changed, 65 insertions(+) create mode 100755 benchmark/runner-config/run-js-concurrency-benchmark.sh diff --git a/benchmark/runner-config/cassandra-driver/config.yml b/benchmark/runner-config/cassandra-driver/config.yml index 750c197e1..7d49da98c 100644 --- a/benchmark/runner-config/cassandra-driver/config.yml +++ b/benchmark/runner-config/cassandra-driver/config.yml @@ -9,12 +9,18 @@ backends: - benchmark-name: concurrent_insert run-command: ./benchmark/runner-config/run-js-benchmark.sh concurrent_insert cassandra-driver + - benchmark-name: concurrency_parametrized_insert + run-command: ./benchmark/runner-config/run-js-concurrency-benchmark.sh concurrent_insert cassandra-driver + - benchmark-name: select run-command: ./benchmark/runner-config/run-js-benchmark.sh select cassandra-driver - benchmark-name: concurrent_select run-command: ./benchmark/runner-config/run-js-benchmark.sh concurrent_select cassandra-driver + - benchmark-name: concurrency_parametrized_select + run-command: ./benchmark/runner-config/run-js-concurrency-benchmark.sh concurrent_select cassandra-driver + - benchmark-name: batch run-command: ./benchmark/runner-config/run-js-benchmark.sh batch cassandra-driver @@ -24,6 +30,9 @@ backends: - benchmark-name: concurrent_paging run-command: ./benchmark/runner-config/run-js-benchmark.sh concurrent_paging cassandra-driver + - benchmark-name: concurrency_parametrized_paging + run-command: ./benchmark/runner-config/run-js-concurrency-benchmark.sh concurrent_paging cassandra-driver + - benchmark-name: large_select run-command: ./benchmark/runner-config/run-js-benchmark.sh large_select cassandra-driver @@ -33,8 +42,14 @@ backends: - benchmark-name: concurrent_deser run-command: ./benchmark/runner-config/run-js-benchmark.sh concurrent_deser cassandra-driver + - benchmark-name: concurrency_parametrized_deser + run-command: ./benchmark/runner-config/run-js-concurrency-benchmark.sh concurrent_deser cassandra-driver + - benchmark-name: ser run-command: ./benchmark/runner-config/run-js-benchmark.sh ser cassandra-driver - benchmark-name: concurrent_ser run-command: ./benchmark/runner-config/run-js-benchmark.sh concurrent_ser cassandra-driver + + - benchmark-name: concurrency_parametrized_ser + run-command: ./benchmark/runner-config/run-js-concurrency-benchmark.sh concurrent_ser cassandra-driver diff --git a/benchmark/runner-config/config.yml b/benchmark/runner-config/config.yml index faefdf96c..ae67bd092 100644 --- a/benchmark/runner-config/config.yml +++ b/benchmark/runner-config/config.yml @@ -21,23 +21,43 @@ benchmarks: starting-step: 6250 - name: concurrent_insert starting-step: 62500 + - name: concurrency_parametrized_insert + starting-step: 16 + step-progress: 2 + no-steps: 8 - name: select starting-step: 1563 - name: concurrent_select starting-step: 6250 + - name: concurrency_parametrized_select + starting-step: 16 + step-progress: 2 + no-steps: 8 - name: batch starting-step: 46875 - name: paging starting-step: 63 - name: concurrent_paging starting-step: 20 + - name: concurrency_parametrized_paging + starting-step: 16 + step-progress: 2 + no-steps: 8 - name: large_select starting-step: 63 - name: deser starting-step: 32 - name: concurrent_deser starting-step: 32 + - name: concurrency_parametrized_deser + starting-step: 16 + step-progress: 2 + no-steps: 8 - name: ser starting-step: 15 - name: concurrent_ser starting-step: 19 + - name: concurrency_parametrized_ser + starting-step: 16 + step-progress: 2 + no-steps: 8 diff --git a/benchmark/runner-config/run-js-concurrency-benchmark.sh b/benchmark/runner-config/run-js-concurrency-benchmark.sh new file mode 100755 index 000000000..05f50ad11 --- /dev/null +++ b/benchmark/runner-config/run-js-concurrency-benchmark.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# Runner for scenarios where we parametrize concurrency level, rather than number of queries. +# Wrapper for fixed-query-count benchmarks where step count controls concurrency. +# Usage: run-js-concurrency-benchmark.sh +set -euo pipefail + +BENCHMARK="$1" +DRIVER="$2" +CONCURRENCY="$3" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +BENCHMARK_DIR="$SCRIPT_DIR/.." + +# Step count omitted (uses module default); N passed as concurrency level. +node "$BENCHMARK_DIR/logic/benchmark.js" "$DRIVER" "$BENCHMARK" "default" "$CONCURRENCY" diff --git a/benchmark/runner-config/scylladb-driver/config.yml b/benchmark/runner-config/scylladb-driver/config.yml index 91a0e7ce7..ed1871dbd 100644 --- a/benchmark/runner-config/scylladb-driver/config.yml +++ b/benchmark/runner-config/scylladb-driver/config.yml @@ -9,12 +9,18 @@ backends: - benchmark-name: concurrent_insert run-command: ./benchmark/runner-config/run-js-benchmark.sh concurrent_insert scylladb-driver-alpha + - benchmark-name: concurrency_parametrized_insert + run-command: ./benchmark/runner-config/run-js-concurrency-benchmark.sh concurrent_insert scylladb-driver-alpha + - benchmark-name: select run-command: ./benchmark/runner-config/run-js-benchmark.sh select scylladb-driver-alpha - benchmark-name: concurrent_select run-command: ./benchmark/runner-config/run-js-benchmark.sh concurrent_select scylladb-driver-alpha + - benchmark-name: concurrency_parametrized_select + run-command: ./benchmark/runner-config/run-js-concurrency-benchmark.sh concurrent_select scylladb-driver-alpha + - benchmark-name: batch run-command: ./benchmark/runner-config/run-js-benchmark.sh batch scylladb-driver-alpha @@ -24,6 +30,9 @@ backends: - benchmark-name: concurrent_paging run-command: ./benchmark/runner-config/run-js-benchmark.sh concurrent_paging scylladb-driver-alpha + - benchmark-name: concurrency_parametrized_paging + run-command: ./benchmark/runner-config/run-js-concurrency-benchmark.sh concurrent_paging scylladb-driver-alpha + - benchmark-name: large_select run-command: ./benchmark/runner-config/run-js-benchmark.sh large_select scylladb-driver-alpha @@ -33,8 +42,14 @@ backends: - benchmark-name: concurrent_deser run-command: ./benchmark/runner-config/run-js-benchmark.sh concurrent_deser scylladb-driver-alpha + - benchmark-name: concurrency_parametrized_deser + run-command: ./benchmark/runner-config/run-js-concurrency-benchmark.sh concurrent_deser scylladb-driver-alpha + - benchmark-name: ser run-command: ./benchmark/runner-config/run-js-benchmark.sh ser scylladb-driver-alpha - benchmark-name: concurrent_ser run-command: ./benchmark/runner-config/run-js-benchmark.sh concurrent_ser scylladb-driver-alpha + + - benchmark-name: concurrency_parametrized_ser + run-command: ./benchmark/runner-config/run-js-concurrency-benchmark.sh concurrent_ser scylladb-driver-alpha From a3a795c023585de66c5a0f1539591757a11d5e80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stanis=C5=82aw=20Czech?= Date: Thu, 9 Apr 2026 11:00:45 +0200 Subject: [PATCH 3/4] Update batch benchmark to use common utils This benchmark was still using it's own row check count, instead of the one from utils --- benchmark/logic/batch.js | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/benchmark/logic/batch.js b/benchmark/logic/batch.js index babacfd0d..c27b4762f 100644 --- a/benchmark/logic/batch.js +++ b/benchmark/logic/batch.js @@ -2,7 +2,6 @@ const async = require("async"); const utils = require("./utils"); const { exit } = require("process"); -const assert = require("assert"); module.exports = function (cassandra, client, stepCount, _concurrencyLevel) { // REMEMBER: update benchmark config.yml when changing the constant value. @@ -33,14 +32,7 @@ module.exports = function (cassandra, client, stepCount, _concurrencyLevel) { next(); }, async function test(next) { - const query = "SELECT COUNT(1) FROM benchmarks.basic USING TIMEOUT 120s;"; - try { - let res = await client.execute(query); - assert(res.rows[0].count == iterCnt); - } catch (err) { - return next(err); - } - next(); + utils.checkRowCount(client, iterCnt, next); }, function r() { exit(0); From b90d2fd0ecd6b966a10009f0c93c64f0e117cca1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stanis=C5=82aw=20Czech?= Date: Fri, 10 Apr 2026 11:30:32 +0200 Subject: [PATCH 4/4] Add rust version of concurrency parameterized benchmarks --- benchmark/logic_rust/common.rs | 17 ++++++++++++++++ benchmark/logic_rust/concurrent_deser.rs | 20 ++++++++++--------- benchmark/logic_rust/concurrent_insert.rs | 13 ++++++------ benchmark/logic_rust/concurrent_paging.rs | 13 ++++++------ benchmark/logic_rust/concurrent_select.rs | 13 ++++++------ benchmark/logic_rust/concurrent_ser.rs | 13 ++++++------ .../run-rust-concurrency-benchmark.sh | 13 ++++++++++++ .../runner-config/rust-driver/config.yml | 20 +++++++++++++++++++ 8 files changed, 88 insertions(+), 34 deletions(-) create mode 100755 benchmark/runner-config/run-rust-concurrency-benchmark.sh diff --git a/benchmark/logic_rust/common.rs b/benchmark/logic_rust/common.rs index 57c11798e..e3276ede4 100644 --- a/benchmark/logic_rust/common.rs +++ b/benchmark/logic_rust/common.rs @@ -106,6 +106,7 @@ pub(crate) fn get_deser_data() -> ( (id, 100, tuuid, ip, date, time, tuple, udt, set, duration) } +#[allow(dead_code)] pub(crate) fn get_cnt() -> i32 { env::var("CNT") .ok() @@ -113,6 +114,22 @@ pub(crate) fn get_cnt() -> i32 { .expect("CNT parameter is required.") } +#[allow(dead_code)] +pub(crate) fn get_cnt_with_default(default: i32) -> i32 { + env::var("CNT") + .ok() + .and_then(|s: String| s.parse::().ok()) + .unwrap_or(default) +} + +#[allow(dead_code)] +pub(crate) fn get_concurrency(default: usize) -> usize { + env::var("CONCURRENCY") + .ok() + .and_then(|s: String| s.parse::().ok()) + .unwrap_or(default) +} + // This may be imported by binaries that use only one of the helpers #[allow(dead_code)] pub(crate) async fn check_row_cnt( diff --git a/benchmark/logic_rust/concurrent_deser.rs b/benchmark/logic_rust/concurrent_deser.rs index ad25e9de4..0e25eee28 100644 --- a/benchmark/logic_rust/concurrent_deser.rs +++ b/benchmark/logic_rust/concurrent_deser.rs @@ -8,12 +8,11 @@ use crate::common::DESER_INSERT_QUERY; mod common; -const CONCURRENCY: usize = 100; - async fn insert_data( session: Arc, start_index: usize, n: i32, + concurrency: usize, insert_query: &PreparedStatement, ) -> Result<(), Box> { let mut index = start_index; @@ -22,7 +21,7 @@ async fn insert_data( session .execute_unpaged(insert_query, common::get_deser_data()) .await?; - index += CONCURRENCY; + index += concurrency; } Ok(()) @@ -32,6 +31,7 @@ async fn select_data( session: Arc, start_index: usize, n: i32, + concurrency: usize, select_query: &PreparedStatement, ) -> Result<(), Box> { let mut index = start_index; @@ -44,7 +44,7 @@ async fn select_data( .rows::()? .collect::>(); assert_eq!(r.len() as i32, n); - index += CONCURRENCY; + index += concurrency; } Ok(()) @@ -52,7 +52,9 @@ async fn select_data( #[tokio::main] async fn main() -> Result<(), Box> { - let n: i32 = common::get_cnt(); + // REMEMBER: update benchmark config.yml when changing the constant value. + let n: i32 = common::get_cnt_with_default(2_048); + let concurrency = common::get_concurrency(100); let session = common::init_deser_table().await?; @@ -61,11 +63,11 @@ async fn main() -> Result<(), Box> { let mut handles = vec![]; let session = Arc::new(session); - for i in 0..CONCURRENCY { + for i in 0..concurrency { let session_clone = Arc::clone(&session); let insert_query_clone = insert_query.clone(); handles.push(tokio::spawn(async move { - insert_data(session_clone, i, n, &insert_query_clone) + insert_data(session_clone, i, n, concurrency, &insert_query_clone) .await .unwrap(); })); @@ -81,11 +83,11 @@ async fn main() -> Result<(), Box> { let mut handles = vec![]; - for i in 0..CONCURRENCY { + for i in 0..concurrency { let session_clone = Arc::clone(&session); let select_query_clone = select_query.clone(); handles.push(tokio::spawn(async move { - select_data(session_clone, i, n, &select_query_clone) + select_data(session_clone, i, n, concurrency, &select_query_clone) .await .unwrap(); })); diff --git a/benchmark/logic_rust/concurrent_insert.rs b/benchmark/logic_rust/concurrent_insert.rs index 443ce2b46..ea21437a4 100644 --- a/benchmark/logic_rust/concurrent_insert.rs +++ b/benchmark/logic_rust/concurrent_insert.rs @@ -8,12 +8,11 @@ use crate::common::SIMPLE_INSERT_QUERY; mod common; -const CONCURRENCY: usize = 100; - async fn insert_data( session: Arc, start_index: usize, n: i32, + concurrency: usize, insert_query: &PreparedStatement, ) -> Result<(), Box> { let mut index = start_index; @@ -21,7 +20,7 @@ async fn insert_data( while index < n as usize { let id = Uuid::new_v4(); session.execute_unpaged(insert_query, (id, 100)).await?; - index += CONCURRENCY; + index += concurrency; } Ok(()) @@ -29,7 +28,9 @@ async fn insert_data( #[tokio::main] async fn main() -> Result<(), Box> { - let n: i32 = common::get_cnt(); + // REMEMBER: update benchmark config.yml when changing the constant value. + let n: i32 = common::get_cnt_with_default(4_000_000); + let concurrency = common::get_concurrency(100); let session = common::init_simple_table().await?; @@ -38,11 +39,11 @@ async fn main() -> Result<(), Box> { let mut handles = vec![]; let session = Arc::new(session); - for i in 0..CONCURRENCY { + for i in 0..concurrency { let session_clone = Arc::clone(&session); let insert_query_clone = insert_query.clone(); handles.push(tokio::spawn(async move { - insert_data(session_clone, i, n, &insert_query_clone) + insert_data(session_clone, i, n, concurrency, &insert_query_clone) .await .unwrap(); })); diff --git a/benchmark/logic_rust/concurrent_paging.rs b/benchmark/logic_rust/concurrent_paging.rs index c099cdeb9..b62dd54e2 100644 --- a/benchmark/logic_rust/concurrent_paging.rs +++ b/benchmark/logic_rust/concurrent_paging.rs @@ -1,18 +1,17 @@ use scylla::{response::PagingState, statement::Statement}; -use std::{env, ops::ControlFlow, sync::Arc}; +use std::{ops::ControlFlow, sync::Arc}; use uuid::Uuid; use crate::common::SIMPLE_INSERT_QUERY; mod common; -const CONCURRENCY_LEVEL: usize = 20; #[tokio::main] async fn main() -> Result<(), Box> { - let n: i32 = env::var("CNT") - .ok() - .and_then(|s: String| s.parse::().ok()) - .expect("CNT parameter is required."); + // REMEMBER: update benchmark config.yml when changing the constant value. + let n: i32 = common::get_cnt_with_default(1_280); + + let concurrency = common::get_concurrency(20); let session = Arc::new(common::init_simple_table_caching().await?); @@ -29,7 +28,7 @@ async fn main() -> Result<(), Box> { .await?; } let mut tasks = vec![]; - for _ in 0..(CONCURRENCY_LEVEL) { + for _ in 0..concurrency { let session = session.clone(); tasks.push(tokio::task::spawn(async move { let mut select_query = Statement::new("SELECT * FROM benchmarks.basic"); diff --git a/benchmark/logic_rust/concurrent_select.rs b/benchmark/logic_rust/concurrent_select.rs index 57db140bc..e7165fdca 100644 --- a/benchmark/logic_rust/concurrent_select.rs +++ b/benchmark/logic_rust/concurrent_select.rs @@ -9,12 +9,11 @@ use crate::common::SIMPLE_INSERT_QUERY; mod common; -const CONCURRENCY: usize = 100; - async fn select_data( session: Arc, start_index: usize, n: i32, + concurrency: usize, select_query: &PreparedStatement, ) -> Result<(), Box> { let mut index = start_index; @@ -27,7 +26,7 @@ async fn select_data( .rows::()? .collect::>(); assert_eq!(r.len() as i32, 10); - index += CONCURRENCY; + index += concurrency; } Ok(()) @@ -35,7 +34,9 @@ async fn select_data( #[tokio::main] async fn main() -> Result<(), Box> { - let n: i32 = common::get_cnt(); + // REMEMBER: update benchmark config.yml when changing the constant value. + let n: i32 = common::get_cnt_with_default(400_000); + let concurrency = common::get_concurrency(100); let session = common::init_simple_table().await?; @@ -50,11 +51,11 @@ async fn main() -> Result<(), Box> { let mut handles = vec![]; let session = Arc::new(session); - for i in 0..CONCURRENCY { + for i in 0..concurrency { let session_clone = Arc::clone(&session); let select_query_clone = select_query.clone(); handles.push(tokio::spawn(async move { - select_data(session_clone, i, n, &select_query_clone) + select_data(session_clone, i, n, concurrency, &select_query_clone) .await .unwrap(); })); diff --git a/benchmark/logic_rust/concurrent_ser.rs b/benchmark/logic_rust/concurrent_ser.rs index 6aea16d35..cf989d001 100644 --- a/benchmark/logic_rust/concurrent_ser.rs +++ b/benchmark/logic_rust/concurrent_ser.rs @@ -7,12 +7,11 @@ use crate::common::DESER_INSERT_QUERY; mod common; -const CONCURRENCY: usize = 100; - async fn insert_data( session: Arc, start_index: usize, n: i32, + concurrency: usize, insert_query: &PreparedStatement, ) -> Result<(), Box> { let mut index = start_index; @@ -21,7 +20,7 @@ async fn insert_data( session .execute_unpaged(insert_query, common::get_deser_data()) .await?; - index += CONCURRENCY; + index += concurrency; } Ok(()) @@ -29,7 +28,9 @@ async fn insert_data( #[tokio::main] async fn main() -> Result<(), Box> { - let n: i32 = common::get_cnt(); + // REMEMBER: update benchmark config.yml when changing the constant value. + let n: i32 = common::get_cnt_with_default(1_216); + let concurrency = common::get_concurrency(100); let session = common::init_deser_table().await?; @@ -38,11 +39,11 @@ async fn main() -> Result<(), Box> { let mut handles = vec![]; let session = Arc::new(session); - for i in 0..CONCURRENCY { + for i in 0..concurrency { let session_clone = Arc::clone(&session); let insert_query_clone = insert_query.clone(); handles.push(tokio::spawn(async move { - insert_data(session_clone, i, n * n, &insert_query_clone) + insert_data(session_clone, i, n * n, concurrency, &insert_query_clone) .await .unwrap(); })); diff --git a/benchmark/runner-config/run-rust-concurrency-benchmark.sh b/benchmark/runner-config/run-rust-concurrency-benchmark.sh new file mode 100755 index 000000000..63f51d8ad --- /dev/null +++ b/benchmark/runner-config/run-rust-concurrency-benchmark.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Runner for scenarios where we parametrize concurrency level, rather than number of queries. +# Step count omitted (uses binary default); N passed as concurrency level. +# Usage: run-rust-concurrency-benchmark.sh +set -euo pipefail + +BINARY="$1" +CONCURRENCY="$2" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$SCRIPT_DIR/../.." + +CONCURRENCY="$CONCURRENCY" "$REPO_ROOT/target/release/$BINARY" diff --git a/benchmark/runner-config/rust-driver/config.yml b/benchmark/runner-config/rust-driver/config.yml index 6db88eb03..0df7f1dda 100644 --- a/benchmark/runner-config/rust-driver/config.yml +++ b/benchmark/runner-config/rust-driver/config.yml @@ -10,6 +10,10 @@ backends: build-command: cargo build -p benchmark --bin concurrent_insert_benchmark -r run-command: ./benchmark/runner-config/run-rust-benchmark.sh concurrent_insert_benchmark + - benchmark-name: concurrency_parametrized_insert + build-command: cargo build -p benchmark --bin concurrent_insert_benchmark -r + run-command: ./benchmark/runner-config/run-rust-concurrency-benchmark.sh concurrent_insert_benchmark + - benchmark-name: select build-command: cargo build -p benchmark --bin select_benchmark -r run-command: ./benchmark/runner-config/run-rust-benchmark.sh select_benchmark @@ -18,6 +22,10 @@ backends: build-command: cargo build -p benchmark --bin concurrent_select_benchmark -r run-command: ./benchmark/runner-config/run-rust-benchmark.sh concurrent_select_benchmark + - benchmark-name: concurrency_parametrized_select + build-command: cargo build -p benchmark --bin concurrent_select_benchmark -r + run-command: ./benchmark/runner-config/run-rust-concurrency-benchmark.sh concurrent_select_benchmark + - benchmark-name: batch build-command: cargo build -p benchmark --bin batch_benchmark -r run-command: ./benchmark/runner-config/run-rust-benchmark.sh batch_benchmark @@ -30,6 +38,10 @@ backends: build-command: cargo build -p benchmark --bin concurrent_paging_benchmark -r run-command: ./benchmark/runner-config/run-rust-benchmark.sh concurrent_paging_benchmark + - benchmark-name: concurrency_parametrized_paging + build-command: cargo build -p benchmark --bin concurrent_paging_benchmark -r + run-command: ./benchmark/runner-config/run-rust-concurrency-benchmark.sh concurrent_paging_benchmark + - benchmark-name: large_select build-command: cargo build -p benchmark --bin large_select_benchmark -r run-command: ./benchmark/runner-config/run-rust-benchmark.sh large_select_benchmark @@ -42,6 +54,10 @@ backends: build-command: cargo build -p benchmark --bin concurrent_deser_benchmark -r run-command: ./benchmark/runner-config/run-rust-benchmark.sh concurrent_deser_benchmark + - benchmark-name: concurrency_parametrized_deser + build-command: cargo build -p benchmark --bin concurrent_deser_benchmark -r + run-command: ./benchmark/runner-config/run-rust-concurrency-benchmark.sh concurrent_deser_benchmark + - benchmark-name: ser build-command: cargo build -p benchmark --bin ser_benchmark -r run-command: ./benchmark/runner-config/run-rust-benchmark.sh ser_benchmark @@ -49,3 +65,7 @@ backends: - benchmark-name: concurrent_ser build-command: cargo build -p benchmark --bin concurrent_ser_benchmark -r run-command: ./benchmark/runner-config/run-rust-benchmark.sh concurrent_ser_benchmark + + - benchmark-name: concurrency_parametrized_ser + build-command: cargo build -p benchmark --bin concurrent_ser_benchmark -r + run-command: ./benchmark/runner-config/run-rust-concurrency-benchmark.sh concurrent_ser_benchmark