From f7aea7a11a45057328a4b877f40434339c72112b Mon Sep 17 00:00:00 2001 From: Anshgrover23 Date: Tue, 2 Jul 2024 21:52:33 +0530 Subject: [PATCH 1/4] Add analyze.js and update run_benchmarks.sh for testing --- analyze.js | 116 ++++++++++++++++++++++++++++++++++++++++++++++ package-lock.json | 0 run_benchmarks.sh | 4 ++ 3 files changed, 120 insertions(+) create mode 100644 analyze.js create mode 100644 package-lock.json diff --git a/analyze.js b/analyze.js new file mode 100644 index 00000000..fef07d0c --- /dev/null +++ b/analyze.js @@ -0,0 +1,116 @@ +const fs = require('fs'); +const path = require('path'); +const exec = require('child_process').execSync; + +const extractMetric = (file, metric) => { + const content = fs.readFileSync(file, 'utf-8'); + const match = content.match(new RegExp(`${metric}: (\\d+\\.?\\d*)`)); + return match ? parseFloat(match[1]) : null; +}; + +const average = (values) => { + const sum = values.reduce((acc, val) => acc + val, 0); + return sum / values.length; +}; + +const formattedServerNames = { + tailcall: 'Tailcall', + gqlgen: 'Gqlgen', + apollo: 'Apollo GraphQL', + netflixdgs: 'Netflix DGS', + caliban: 'Caliban', + async_graphql: 'async-graphql' +}; + +const servers = ['apollo', 'caliban', 'netflixdgs', 'gqlgen', 'tailcall', 'async_graphql']; +const resultFiles = process.argv.slice(2); +const avgReqSecs = {}; +const avgLatencies = {}; + +servers.forEach((server, idx) => { + const reqSecVals = []; + const latencyVals = []; + for (let j = 0; j < 3; j++) { + const fileIdx = idx * 3 + j; + reqSecVals.push(extractMetric(resultFiles[fileIdx], 'Requests/sec')); + latencyVals.push(extractMetric(resultFiles[fileIdx], 'Latency')); + } + avgReqSecs[server] = average(reqSecVals); + avgLatencies[server] = average(latencyVals); +}); + +// Generating data files for gnuplot +const reqSecData = '/tmp/reqSec.dat'; +const latencyData = '/tmp/latency.dat'; + +const writeDataFile = (filePath, data) => { + const content = 'Server Value\n' + data.map(([server, value]) => `${server} ${value}`).join('\n'); + fs.writeFileSync(filePath, content); +}; + +writeDataFile(reqSecData, servers.map(server => [server, avgReqSecs[server]])); +writeDataFile(latencyData, servers.map(server => [server, avgLatencies[server]])); + +const whichBench = resultFiles[0].includes('bench2') ? 2 : 1; +const reqSecHistogramFile = `req_sec_histogram${whichBench}.png`; +const latencyHistogramFile = `latency_histogram${whichBench}.png`; + +// Plotting using gnuplot +const gnuplotScript = ` +set term pngcairo size 1280,720 enhanced font "Courier,12" +set output "${reqSecHistogramFile}" +set style data histograms +set style histogram cluster gap 1 +set style fill solid border -1 +set xtics rotate by -45 +set boxwidth 0.9 +set title "Requests/Sec" +stats "${reqSecData}" using 2 nooutput +set yrange [0:STATS_max*1.2] +set key outside right top +plot "${reqSecData}" using 2:xtic(1) title "Req/Sec" + +set output "${latencyHistogramFile}" +set title "Latency (in ms)" +stats "${latencyData}" using 2 nooutput +set yrange [0:STATS_max*1.2] +plot "${latencyData}" using 2:xtic(1) title "Latency" +`; + +exec(`echo "${gnuplotScript}" | gnuplot`); + +const assetsDir = 'assets'; +if (!fs.existsSync(assetsDir)) { + fs.mkdirSync(assetsDir); +} + +fs.renameSync(reqSecHistogramFile, path.join(assetsDir, reqSecHistogramFile)); +fs.renameSync(latencyHistogramFile, path.join(assetsDir, latencyHistogramFile)); + +const resultsTable = [ + ``, + '', + '| Server | Requests/sec | Latency (ms) |', + '|--------:|--------------:|--------------:|', + ...servers.map(server => { + const formattedReqSecs = avgReqSecs[server].toLocaleString(undefined, { minimumFractionDigits: 2, maximumFractionDigits: 2 }); + const formattedLatencies = avgLatencies[server].toLocaleString(undefined, { minimumFractionDigits: 2, maximumFractionDigits: 2 }); + return `| ${formattedServerNames[server]} | \`${formattedReqSecs}\` | \`${formattedLatencies}\` |`; + }), + '', + `` +].join('\n'); + +const updateReadme = (table, whichBench) => { + const readmePath = 'README.md'; + const readmeContent = fs.readFileSync(readmePath, 'utf-8'); + const newContent = readmeContent.replace( + new RegExp(`[\\s\\S]*`, 'm'), + table + ); + fs.writeFileSync(readmePath, newContent); +}; + +updateReadme(resultsTable, whichBench); + +console.log(resultsTable.replace(``, '').replace(``, '')); diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 00000000..e69de29b diff --git a/run_benchmarks.sh b/run_benchmarks.sh index 3151eec1..39088df6 100755 --- a/run_benchmarks.sh +++ b/run_benchmarks.sh @@ -71,3 +71,7 @@ done bash analyze.sh "${bench1Results[@]}" bash analyze.sh "${bench2Results[@]}" + +# Run the new analyze.js script +node wrk/analyze.js "${bench1Results[@]}" +node wrk/analyze.js "${bench2Results[@]}" From 8c22bad4ec8b5424c4c8462a0b70174b8df60279 Mon Sep 17 00:00:00 2001 From: Anshgrover23 <168731971+Anshgrover23@users.noreply.github.com> Date: Wed, 3 Jul 2024 14:23:39 +0530 Subject: [PATCH 2/4] Update run_benchmarks.sh --- run_benchmarks.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/run_benchmarks.sh b/run_benchmarks.sh index 39088df6..64d99adf 100755 --- a/run_benchmarks.sh +++ b/run_benchmarks.sh @@ -73,5 +73,5 @@ bash analyze.sh "${bench1Results[@]}" bash analyze.sh "${bench2Results[@]}" # Run the new analyze.js script -node wrk/analyze.js "${bench1Results[@]}" -node wrk/analyze.js "${bench2Results[@]}" +node analyze.js "${bench1Results[@]}" +node analyze.js "${bench2Results[@]}" From be8e6b427504d29c37c87fb6aec4552cdcf2b6af Mon Sep 17 00:00:00 2001 From: Anshgrover23 <168731971+Anshgrover23@users.noreply.github.com> Date: Wed, 3 Jul 2024 14:29:55 +0530 Subject: [PATCH 3/4] Update run_benchmarks.sh --- run_benchmarks.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/run_benchmarks.sh b/run_benchmarks.sh index 64d99adf..8d8cd6a9 100755 --- a/run_benchmarks.sh +++ b/run_benchmarks.sh @@ -69,9 +69,6 @@ for service in "apollo_server" "caliban" "netflix_dgs" "gqlgen" "tailcall" "asyn fi done -bash analyze.sh "${bench1Results[@]}" -bash analyze.sh "${bench2Results[@]}" - # Run the new analyze.js script node analyze.js "${bench1Results[@]}" node analyze.js "${bench2Results[@]}" From 6158b983b5da9c45a81ff49ece1ac572676545ab Mon Sep 17 00:00:00 2001 From: Anshgrover23 <168731971+Anshgrover23@users.noreply.github.com> Date: Wed, 3 Jul 2024 15:10:45 +0530 Subject: [PATCH 4/4] Update analyze.js --- analyze.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/analyze.js b/analyze.js index fef07d0c..78e0c6f6 100644 --- a/analyze.js +++ b/analyze.js @@ -3,6 +3,7 @@ const path = require('path'); const exec = require('child_process').execSync; const extractMetric = (file, metric) => { + console.log(`Reading file: ${file}`); const content = fs.readFileSync(file, 'utf-8'); const match = content.match(new RegExp(`${metric}: (\\d+\\.?\\d*)`)); return match ? parseFloat(match[1]) : null; @@ -32,6 +33,7 @@ servers.forEach((server, idx) => { const latencyVals = []; for (let j = 0; j < 3; j++) { const fileIdx = idx * 3 + j; + console.log(`Processing server: ${server}, file index: ${fileIdx}, file path: ${resultFiles[fileIdx]}`); reqSecVals.push(extractMetric(resultFiles[fileIdx], 'Requests/sec')); latencyVals.push(extractMetric(resultFiles[fileIdx], 'Latency')); } @@ -69,7 +71,6 @@ stats "${reqSecData}" using 2 nooutput set yrange [0:STATS_max*1.2] set key outside right top plot "${reqSecData}" using 2:xtic(1) title "Req/Sec" - set output "${latencyHistogramFile}" set title "Latency (in ms)" stats "${latencyData}" using 2 nooutput