From be0b9ee30386fb643bdb5ca0631731c1e2aca333 Mon Sep 17 00:00:00 2001 From: vijay Date: Thu, 11 Jul 2024 15:34:13 +0530 Subject: [PATCH 1/4] Fic : rewrite to analyze.js --- analyze.js | 127 +++++++++++++++++++++++++++++++++++ analyze.sh | 164 ---------------------------------------------- run_benchmarks.sh | 6 +- 3 files changed, 130 insertions(+), 167 deletions(-) create mode 100755 analyze.js delete mode 100755 analyze.sh diff --git a/analyze.js b/analyze.js new file mode 100755 index 00000000..f621f0b9 --- /dev/null +++ b/analyze.js @@ -0,0 +1,127 @@ +#!/usr/bin/env node + +const fs = require("fs"); +const { execSync } = require("child_process"); + +const average = (arr) => arr.reduce((a, b) => a + b, 0) / arr.length; + +const servers = ["apollo", "netflixdgs", "gqlgen", "tailcall"]; + +const resultFiles = process.argv + .slice(2) + .filter((arg) => arg.startsWith("result") && fs.existsSync(arg)); + +const extractMetrics = (files) => { + return files.map((file) => { + const { summary } = JSON.parse(fs.readFileSync(file, "utf8")); + return { + requestsPerSec: summary.requestsPerSec, + latency: summary.average * 1000, + }; + }); +}; + +const calculateAverages = (metrics) => { + return servers.reduce((acc, server, index) => { + const serverMetrics = metrics.slice(index * 3, (index + 1) * 3); + acc[server] = { + avgReqSec: average(serverMetrics.map((m) => m.requestsPerSec)), + avgLatency: average(serverMetrics.map((m) => m.latency)), + }; + return acc; + }, {}); +}; + +const writeDataFile = (filename, data) => { + fs.writeFileSync(filename, "Server Value\n"); + Object.entries(data).forEach(([server, value]) => { + fs.appendFileSync(filename, `${server} ${value}\n`); + }); +}; + +const generateGnuplotScript = (reqSecData, latencyData) => ` +set term pngcairo size 1280,720 enhanced font "Courier,12" +set style data histograms +set style histogram cluster gap 1 +set style fill solid border -1 +set xtics rotate by -45 +set boxwidth 0.9 +set key outside right top +set output "req_sec_histogram.png" +set title "Requests/Sec" +stats "${reqSecData}" using 2 nooutput +set yrange [0:*] +plot "${reqSecData}" using 2:xtic(1) title "Req/Sec" +set output "latency_histogram.png" +set title "Latency (in ms)" +stats "${latencyData}" using 2 nooutput +set yrange [0:*] +plot "${latencyData}" using 2:xtic(1) title "Latency" +`; + +const generateResultsTable = (averages) => { + let table = + "\n| Server | Requests/sec | Latency (ms) |\n|--------|--------------|--------------|"; + Object.entries(averages).forEach(([server, { avgReqSec, avgLatency }]) => { + table += `\n| ${server} | ${avgReqSec} | ${avgLatency} |`; + }); + return table + "\n"; +}; + +const updateReadme = (resultsTable) => { + const readmePath = "README.md"; + let content = fs.readFileSync(readmePath, "utf8"); + const regex = + /[\s\S]*/; + content = content.includes("PERFORMANCE_RESULTS_START") + ? content.replace(regex, resultsTable) + : content + "\n\n" + resultsTable; + fs.writeFileSync(readmePath, content); +}; + +const main = () => { + const metrics = extractMetrics(resultFiles); + const averages = calculateAverages(metrics); + + resultFiles.forEach(fs.unlinkSync); + + const reqSecData = "/tmp/reqSec.dat"; + const latencyData = "/tmp/latency.dat"; + + writeDataFile( + reqSecData, + Object.fromEntries( + Object.entries(averages).map(([k, v]) => [k, v.avgReqSec]) + ) + ); + writeDataFile( + latencyData, + Object.fromEntries( + Object.entries(averages).map(([k, v]) => [k, v.avgLatency]) + ) + ); + + const gnuplotScript = generateGnuplotScript(reqSecData, latencyData); + fs.writeFileSync("/tmp/gnuplot_script", gnuplotScript); + execSync("gnuplot /tmp/gnuplot_script"); + + if (!fs.existsSync("assets")) fs.mkdirSync("assets"); + ["req_sec_histogram.png", "latency_histogram.png"].forEach((file) => + fs.renameSync(file, `assets/${file}`) + ); + + const resultsTable = generateResultsTable(averages); + updateReadme(resultsTable); + + console.log(resultsTable.replace(/\n?/g, "")); + + execSync( + "git add README.md assets/req_sec_histogram.png assets/latency_histogram.png" + ); + execSync('git commit -m "Updated performance results in README.md"'); + execSync("git push"); + + resultFiles.forEach(fs.unlinkSync); +}; + +main(); diff --git a/analyze.sh b/analyze.sh deleted file mode 100755 index 17bbbce1..00000000 --- a/analyze.sh +++ /dev/null @@ -1,164 +0,0 @@ -#!/bin/bash - -function extractMetric() { - local file="$1" - local metric="$2" - grep "$metric" "$file" | awk '{print $2}' | sed 's/ms//' -} - -function average() { - echo "$@" | awk '{for(i=1;i<=NF;i++) s+=$i; print s/NF}' -} - -declare -A formattedServerNames -formattedServerNames=( - ["tailcall"]="Tailcall" - ["gqlgen"]="Gqlgen" - ["apollo"]="Apollo GraphQL" - ["netflixdgs"]="Netflix DGS" - ["caliban"]="Caliban" - ["async_graphql"]="async-graphql" - ["hasura"]="Hasura" - ["graphql_jit"]="GraphQL JIT" -) - -servers=("apollo" "caliban" "netflixdgs" "gqlgen" "tailcall" "async_graphql" "hasura" "graphql_jit") -resultFiles=("$@") -declare -A avgReqSecs -declare -A avgLatencies - -# Extract metrics and calculate averages -for idx in "${!servers[@]}"; do - startIdx=$((idx * 3)) - reqSecVals=() - latencyVals=() - for j in 0 1 2; do - fileIdx=$((startIdx + j)) - reqSecVals+=($(extractMetric "${resultFiles[$fileIdx]}" "Requests/sec")) - latencyVals+=($(extractMetric "${resultFiles[$fileIdx]}" "Latency")) - done - avgReqSecs[${servers[$idx]}]=$(average "${reqSecVals[@]}") - avgLatencies[${servers[$idx]}]=$(average "${latencyVals[@]}") -done - -# Generating data files for gnuplot -reqSecData="/tmp/reqSec.dat" -latencyData="/tmp/latency.dat" - -echo "Server Value" >"$reqSecData" -for server in "${servers[@]}"; do - echo "$server ${avgReqSecs[$server]}" >>"$reqSecData" -done - -echo "Server Value" >"$latencyData" -for server in "${servers[@]}"; do - echo "$server ${avgLatencies[$server]}" >>"$latencyData" -done - -whichBench=1 -if [[ $1 == bench2* ]]; then - whichBench=2 -elif [[ $1 == bench3* ]]; then - whichBench=3 -fi - -reqSecHistogramFile="req_sec_histogram${whichBench}.png" -latencyHistogramFile="latency_histogram${whichBench}.png" - -# Plotting using gnuplot -gnuplot <<-EOF - set term pngcairo size 1280,720 enhanced font "Courier,12" - set output "$reqSecHistogramFile" - set style data histograms - set style histogram cluster gap 1 - set style fill solid border -1 - set xtics rotate by -45 - set boxwidth 0.9 - set title "Requests/Sec" - stats "$reqSecData" using 2 nooutput - set yrange [0:STATS_max*1.2] - set key outside right top - plot "$reqSecData" using 2:xtic(1) title "Req/Sec" - - set output "$latencyHistogramFile" - set title "Latency (in ms)" - stats "$latencyData" using 2 nooutput - set yrange [0:STATS_max*1.2] - plot "$latencyData" using 2:xtic(1) title "Latency" -EOF - -# Move PNGs to assets -mkdir -p assets -mv $reqSecHistogramFile assets/ -mv $latencyHistogramFile assets/ - -# Declare an associative array for server RPS -declare -A serverRPS - -# Populate the serverRPS array -for server in "${servers[@]}"; do - serverRPS[$server]=${avgReqSecs[$server]} -done - -# Get the servers sorted by RPS in descending order -IFS=$'\n' sortedServers=($(for server in "${!serverRPS[@]}"; do echo "$server ${serverRPS[$server]}"; done | sort -rn -k2 | cut -d' ' -f1)) - -echo "Sorted servers: ${sortedServers[@]}" -lastServer="${sortedServers[-1]}" -lastServerReqSecs=${avgReqSecs[$lastServer]} - -# Start building the resultsTable -if [[ $whichBench == 1 ]]; then - resultsTable="\n\n| Query | Server | Requests/sec | Latency (ms) | Relative |\n|-------:|--------:|--------------:|--------------:|---------:|\n| $whichBench | \`{ posts { id userId title user { id name email }}}\` |" -elif [[ $whichBench == 2 ]]; then - resultsTable="| $whichBench | \`{ posts { title }}\` |" -elif [[ $whichBench == 3 ]]; then - resultsTable="| $whichBench | \`{ greet }\` |" -fi - -# Build the resultsTable with sorted servers and formatted numbers -for server in "${sortedServers[@]}"; do - formattedReqSecs=$(printf "%.2f" ${avgReqSecs[$server]} | perl -pe 's/(?<=\d)(?=(\d{3})+(\.\d*)?$)/,/g') - formattedLatencies=$(printf "%.2f" ${avgLatencies[$server]} | perl -pe 's/(?<=\d)(?=(\d{3})+(\.\d*)?$)/,/g') - # Calculate the relative performance - relativePerformance=$(echo "${avgReqSecs[$server]} $lastServerReqSecs" | awk '{printf "%.2f", $1 / $2}') - - resultsTable+="\n|| [${formattedServerNames[$server]}] | \`${formattedReqSecs}\` | \`${formattedLatencies}\` | \`${relativePerformance}x\` |" -done - -if [[ $whichBench == 3 ]]; then - resultsTable+="\n\n" -fi - -echo "resultsTable: $resultsTable" - -# Print the results table in a new file -resultsFile="results.md" -echo -e $resultsTable >> $resultsFile - - -if [[ $whichBench == 3 ]]; then - finalResults=$(printf '%s\n' "$(cat $resultsFile)" | sed 's/$/\\n/'| tr -d '\n') - # Remove the last newline character - finalResults=${finalResults::-2} - - # Print the results as a table in the terminal - echo -e $finalResults | sed "s///;s///" - # Check if the markers are present - if grep -q "PERFORMANCE_RESULTS_START" README.md; then - # Replace the old results with the new results - sed -i "/PERFORMANCE_RESULTS_START/,/PERFORMANCE_RESULTS_END/c\\$finalResults" README.md - else - # Append the results at the end of the README.md file - echo -e "\n$finalResults" >> README.md - fi -fi - -# Move the generated images to the assets folder -mv $reqSecHistogramFile assets/ -mv $latencyHistogramFile assets/ - -# Delete the result TXT files -for file in "${resultFiles[@]}"; do - rm "$file" -done diff --git a/run_benchmarks.sh b/run_benchmarks.sh index 263e6f69..e665e6f1 100755 --- a/run_benchmarks.sh +++ b/run_benchmarks.sh @@ -83,6 +83,6 @@ for service in "apollo_server" "caliban" "netflix_dgs" "gqlgen" "tailcall" "asyn fi done -bash analyze.sh "${bench1Results[@]}" -bash analyze.sh "${bench2Results[@]}" -bash analyze.sh "${bench3Results[@]}" +bash analyze.js "${bench1Results[@]}" +bash analyze.js "${bench2Results[@]}" +bash analyze.js "${bench3Results[@]}" From 51cb4a0722d251f1f3c7a9998d088a88e92a7eb3 Mon Sep 17 00:00:00 2001 From: vijay Date: Thu, 11 Jul 2024 15:48:42 +0530 Subject: [PATCH 2/4] add other fservernames --- analyze.js | 279 ++++++++++++++++++++++++++++++++--------------------- 1 file changed, 169 insertions(+), 110 deletions(-) diff --git a/analyze.js b/analyze.js index f621f0b9..6f936323 100755 --- a/analyze.js +++ b/analyze.js @@ -1,127 +1,186 @@ #!/usr/bin/env node const fs = require("fs"); -const { execSync } = require("child_process"); - -const average = (arr) => arr.reduce((a, b) => a + b, 0) / arr.length; - -const servers = ["apollo", "netflixdgs", "gqlgen", "tailcall"]; - -const resultFiles = process.argv - .slice(2) - .filter((arg) => arg.startsWith("result") && fs.existsSync(arg)); - -const extractMetrics = (files) => { - return files.map((file) => { - const { summary } = JSON.parse(fs.readFileSync(file, "utf8")); - return { - requestsPerSec: summary.requestsPerSec, - latency: summary.average * 1000, - }; - }); -}; - -const calculateAverages = (metrics) => { - return servers.reduce((acc, server, index) => { - const serverMetrics = metrics.slice(index * 3, (index + 1) * 3); - acc[server] = { - avgReqSec: average(serverMetrics.map((m) => m.requestsPerSec)), - avgLatency: average(serverMetrics.map((m) => m.latency)), - }; - return acc; - }, {}); +const { exec } = require("child_process"); +const path = require("path"); + +// Helper functions +const extractMetric = (file, metric) => { + const content = fs.readFileSync(file, "utf-8"); + const regex = new RegExp(`${metric}\\s+(\\d+\\.?\\d*)`); + const match = content.match(regex); + return match ? parseFloat(match[1]) : null; }; -const writeDataFile = (filename, data) => { - fs.writeFileSync(filename, "Server Value\n"); - Object.entries(data).forEach(([server, value]) => { - fs.appendFileSync(filename, `${server} ${value}\n`); - }); +const average = (values) => + values.reduce((sum, value) => sum + value, 0) / values.length; + +const formattedServerNames = { + tailcall: "Tailcall", + gqlgen: "Gqlgen", + apollo: "Apollo GraphQL", + netflixdgs: "Netflix DGS", + caliban: "Caliban", + async_graphql: "async-graphql", + hasura: "Hasura", + graphql_jit: "GraphQL JIT", }; -const generateGnuplotScript = (reqSecData, latencyData) => ` -set term pngcairo size 1280,720 enhanced font "Courier,12" -set style data histograms -set style histogram cluster gap 1 -set style fill solid border -1 -set xtics rotate by -45 -set boxwidth 0.9 -set key outside right top -set output "req_sec_histogram.png" -set title "Requests/Sec" -stats "${reqSecData}" using 2 nooutput -set yrange [0:*] -plot "${reqSecData}" using 2:xtic(1) title "Req/Sec" -set output "latency_histogram.png" -set title "Latency (in ms)" -stats "${latencyData}" using 2 nooutput -set yrange [0:*] -plot "${latencyData}" using 2:xtic(1) title "Latency" -`; - -const generateResultsTable = (averages) => { - let table = - "\n| Server | Requests/sec | Latency (ms) |\n|--------|--------------|--------------|"; - Object.entries(averages).forEach(([server, { avgReqSec, avgLatency }]) => { - table += `\n| ${server} | ${avgReqSec} | ${avgLatency} |`; - }); - return table + "\n"; +const servers = [ + "apollo", + "caliban", + "netflixdgs", + "gqlgen", + "tailcall", + "async_graphql", + "hasura", + "graphql_jit", +]; +const resultFiles = process.argv.slice(2); +const avgReqSecs = {}; +const avgLatencies = {}; + +// Extract metrics and calculate averages +servers.forEach((server, idx) => { + const reqSecVals = []; + const latencyVals = []; + for (let j = 0; j < 3; j++) { + const fileIdx = idx * 3 + j; + reqSecVals.push(extractMetric(resultFiles[fileIdx], "Requests/sec")); + latencyVals.push(extractMetric(resultFiles[fileIdx], "Latency")); + } + avgReqSecs[server] = average(reqSecVals); + avgLatencies[server] = average(latencyVals); +}); + +// Generating data files for gnuplot +const reqSecData = "/tmp/reqSec.dat"; +const latencyData = "/tmp/latency.dat"; + +const writeDataFile = (filePath, data) => { + const content = [ + "Server Value", + ...data.map(({ server, value }) => `${server} ${value}`), + ].join("\n"); + fs.writeFileSync(filePath, content); }; -const updateReadme = (resultsTable) => { - const readmePath = "README.md"; - let content = fs.readFileSync(readmePath, "utf8"); - const regex = - /[\s\S]*/; - content = content.includes("PERFORMANCE_RESULTS_START") - ? content.replace(regex, resultsTable) - : content + "\n\n" + resultsTable; - fs.writeFileSync(readmePath, content); +writeDataFile( + reqSecData, + servers.map((server) => ({ server, value: avgReqSecs[server] })) +); +writeDataFile( + latencyData, + servers.map((server) => ({ server, value: avgLatencies[server] })) +); + +// Determine which benchmark to use +let whichBench = 1; +if (resultFiles[0].startsWith("bench2")) { + whichBench = 2; +} else if (resultFiles[0].startsWith("bench3")) { + whichBench = 3; +} + +const reqSecHistogramFile = `req_sec_histogram${whichBench}.png`; +const latencyHistogramFile = `latency_histogram${whichBench}.png`; + +// Plotting using gnuplot +const plotWithGnuplot = (outputFile, title, dataFile) => { + const script = ` + set term pngcairo size 1280,720 enhanced font "Courier,12" + set output "${outputFile}" + set style data histograms + set style histogram cluster gap 1 + set style fill solid border -1 + set xtics rotate by -45 + set boxwidth 0.9 + set title "${title}" + stats "${dataFile}" using 2 nooutput + set yrange [0:STATS_max*1.2] + set key outside right top + plot "${dataFile}" using 2:xtic(1) title "${title.split(" ")[0]}" + `; + exec(`gnuplot -e '${script}'`); }; -const main = () => { - const metrics = extractMetrics(resultFiles); - const averages = calculateAverages(metrics); - - resultFiles.forEach(fs.unlinkSync); - - const reqSecData = "/tmp/reqSec.dat"; - const latencyData = "/tmp/latency.dat"; - - writeDataFile( - reqSecData, - Object.fromEntries( - Object.entries(averages).map(([k, v]) => [k, v.avgReqSec]) - ) - ); - writeDataFile( - latencyData, - Object.fromEntries( - Object.entries(averages).map(([k, v]) => [k, v.avgLatency]) - ) - ); - - const gnuplotScript = generateGnuplotScript(reqSecData, latencyData); - fs.writeFileSync("/tmp/gnuplot_script", gnuplotScript); - execSync("gnuplot /tmp/gnuplot_script"); - - if (!fs.existsSync("assets")) fs.mkdirSync("assets"); - ["req_sec_histogram.png", "latency_histogram.png"].forEach((file) => - fs.renameSync(file, `assets/${file}`) +plotWithGnuplot(reqSecHistogramFile, "Requests/Sec", reqSecData); +plotWithGnuplot(latencyHistogramFile, "Latency (in ms)", latencyData); + +// Move PNGs to assets +const assetsDir = path.join(__dirname, "assets"); +if (!fs.existsSync(assetsDir)) { + fs.mkdirSync(assetsDir); +} +fs.renameSync(reqSecHistogramFile, path.join(assetsDir, reqSecHistogramFile)); +fs.renameSync(latencyHistogramFile, path.join(assetsDir, latencyHistogramFile)); + +// Calculate relative performance and build the results table +const serverRPS = {}; +servers.forEach((server) => { + serverRPS[server] = avgReqSecs[server]; +}); + +const sortedServers = Object.keys(serverRPS).sort( + (a, b) => serverRPS[b] - serverRPS[a] +); +const lastServer = sortedServers[sortedServers.length - 1]; +const lastServerReqSecs = avgReqSecs[lastServer]; + +let resultsTable = ""; + +if (whichBench === 1) { + resultsTable += `\n\n| Query | Server | Requests/sec | Latency (ms) | Relative |\n|-------:|--------:|--------------:|--------------:|---------:|\n| ${whichBench} | \`{ posts { id userId title user { id name email }}}\` |`; +} else if (whichBench === 2) { + resultsTable += `| ${whichBench} | \`{ posts { title }}\` |`; +} else if (whichBench === 3) { + resultsTable += `| ${whichBench} | \`{ greet }\` |`; +} + +sortedServers.forEach((server) => { + const formattedReqSecs = avgReqSecs[server].toLocaleString(undefined, { + minimumFractionDigits: 2, + maximumFractionDigits: 2, + }); + const formattedLatencies = avgLatencies[server].toLocaleString(undefined, { + minimumFractionDigits: 2, + maximumFractionDigits: 2, + }); + const relativePerformance = (avgReqSecs[server] / lastServerReqSecs).toFixed( + 2 ); - const resultsTable = generateResultsTable(averages); - updateReadme(resultsTable); + resultsTable += `\n|| [${formattedServerNames[server]}] | \`${formattedReqSecs}\` | \`${formattedLatencies}\` | \`${relativePerformance}x\` |`; +}); - console.log(resultsTable.replace(/\n?/g, "")); +if (whichBench === 3) { + resultsTable += `\n\n`; +} - execSync( - "git add README.md assets/req_sec_histogram.png assets/latency_histogram.png" - ); - execSync('git commit -m "Updated performance results in README.md"'); - execSync("git push"); +const resultsFile = "results.md"; +fs.writeFileSync(resultsFile, resultsTable); - resultFiles.forEach(fs.unlinkSync); -}; +if (whichBench === 3) { + const finalResults = fs + .readFileSync(resultsFile, "utf-8") + .replace(/(\r\n|\n|\r)/gm, "\\n"); -main(); + const readmePath = "README.md"; + let readmeContent = fs.readFileSync(readmePath, "utf-8"); + const performanceResultsRegex = + /[\s\S]*/; + if (performanceResultsRegex.test(readmeContent)) { + readmeContent = readmeContent.replace( + performanceResultsRegex, + finalResults + ); + } else { + readmeContent += `\n${finalResults}`; + } + fs.writeFileSync(readmePath, readmeContent); +} + +// Delete the result TXT files +resultFiles.forEach((file) => { + fs.unlinkSync(file); +}); From b2ad4198de199622fdb342147250434f5d39a3a1 Mon Sep 17 00:00:00 2001 From: vijay Date: Fri, 19 Jul 2024 15:30:13 +0530 Subject: [PATCH 3/4] reimplement the logic in js file --- analyze.js | 162 +++++++++++++++++++++++++++++++++-------------------- 1 file changed, 102 insertions(+), 60 deletions(-) diff --git a/analyze.js b/analyze.js index 6f936323..d455bfc6 100755 --- a/analyze.js +++ b/analyze.js @@ -1,19 +1,24 @@ #!/usr/bin/env node const fs = require("fs"); -const { exec } = require("child_process"); +const { execSync } = require("child_process"); const path = require("path"); -// Helper functions -const extractMetric = (file, metric) => { - const content = fs.readFileSync(file, "utf-8"); - const regex = new RegExp(`${metric}\\s+(\\d+\\.?\\d*)`); - const match = content.match(regex); - return match ? parseFloat(match[1]) : null; -}; +function extractMetric(file, metric) { + try { + const command = `grep "${metric}" "${file}" | awk '{print $2}' | sed 's/ms//'`; + const result = execSync(command, { encoding: "utf-8" }).trim(); + return result; + } catch (error) { + console.error(`Error extracting metric from ${file}: ${error.message}`); + return null; + } +} -const average = (values) => - values.reduce((sum, value) => sum + value, 0) / values.length; +function average(values) { + const sum = values.reduce((a, b) => parseFloat(a) + parseFloat(b), 0); + return sum / values.length; +} const formattedServerNames = { tailcall: "Tailcall", @@ -40,14 +45,16 @@ const resultFiles = process.argv.slice(2); const avgReqSecs = {}; const avgLatencies = {}; -// Extract metrics and calculate averages servers.forEach((server, idx) => { + const startIdx = idx * 3; const reqSecVals = []; const latencyVals = []; for (let j = 0; j < 3; j++) { - const fileIdx = idx * 3 + j; - reqSecVals.push(extractMetric(resultFiles[fileIdx], "Requests/sec")); - latencyVals.push(extractMetric(resultFiles[fileIdx], "Latency")); + const fileIdx = startIdx + j; + const reqSec = extractMetric(resultFiles[fileIdx], "Requests/sec"); + const latency = extractMetric(resultFiles[fileIdx], "Latency"); + if (reqSec !== null) reqSecVals.push(reqSec); + if (latency !== null) latencyVals.push(latency); } avgReqSecs[server] = average(reqSecVals); avgLatencies[server] = average(latencyVals); @@ -57,24 +64,17 @@ servers.forEach((server, idx) => { const reqSecData = "/tmp/reqSec.dat"; const latencyData = "/tmp/latency.dat"; -const writeDataFile = (filePath, data) => { - const content = [ - "Server Value", - ...data.map(({ server, value }) => `${server} ${value}`), - ].join("\n"); - fs.writeFileSync(filePath, content); -}; - -writeDataFile( +fs.writeFileSync( reqSecData, - servers.map((server) => ({ server, value: avgReqSecs[server] })) + "Server Value\n" + + servers.map((server) => `${server} ${avgReqSecs[server]}`).join("\n") ); -writeDataFile( +fs.writeFileSync( latencyData, - servers.map((server) => ({ server, value: avgLatencies[server] })) + "Server Value\n" + + servers.map((server) => `${server} ${avgLatencies[server]}`).join("\n") ); -// Determine which benchmark to use let whichBench = 1; if (resultFiles[0].startsWith("bench2")) { whichBench = 2; @@ -85,37 +85,68 @@ if (resultFiles[0].startsWith("bench2")) { const reqSecHistogramFile = `req_sec_histogram${whichBench}.png`; const latencyHistogramFile = `latency_histogram${whichBench}.png`; -// Plotting using gnuplot -const plotWithGnuplot = (outputFile, title, dataFile) => { - const script = ` - set term pngcairo size 1280,720 enhanced font "Courier,12" - set output "${outputFile}" - set style data histograms - set style histogram cluster gap 1 - set style fill solid border -1 - set xtics rotate by -45 - set boxwidth 0.9 - set title "${title}" - stats "${dataFile}" using 2 nooutput - set yrange [0:STATS_max*1.2] - set key outside right top - plot "${dataFile}" using 2:xtic(1) title "${title.split(" ")[0]}" - `; - exec(`gnuplot -e '${script}'`); -}; +function getMaxValue(data) { + return Math.max( + ...data + .split("\n") + .slice(1) + .map((line) => parseFloat(line.split(" ")[1])) + ); +} -plotWithGnuplot(reqSecHistogramFile, "Requests/Sec", reqSecData); -plotWithGnuplot(latencyHistogramFile, "Latency (in ms)", latencyData); +const reqSecMax = getMaxValue(fs.readFileSync(reqSecData, "utf-8")) * 1.2; +const latencyMax = getMaxValue(fs.readFileSync(latencyData, "utf-8")) * 1.2; + +const gnuplotScript = ` +set term pngcairo size 1280,720 enhanced font 'Courier,12' +set output '${reqSecHistogramFile}' +set style data histograms +set style histogram cluster gap 1 +set style fill solid border -1 +set xtics rotate by -45 +set boxwidth 0.9 +set title 'Requests/Sec' +set yrange [0:${reqSecMax}] +set key outside right top +plot '${reqSecData}' using 2:xtic(1) title 'Req/Sec' +set output '${latencyHistogramFile}' +set title 'Latency (in ms)' +set yrange [0:${latencyMax}] +plot '${latencyData}' using 2:xtic(1) title 'Latency' +`; + +const gnuplotScriptFile = "/tmp/gnuplot_script.gp"; +fs.writeFileSync(gnuplotScriptFile, gnuplotScript); + +try { + execSync(`gnuplot ${gnuplotScriptFile}`, { stdio: "inherit" }); + console.log("Gnuplot executed successfully"); +} catch (error) { + console.error("Error executing gnuplot:", error.message); + process.exit(1); +} -// Move PNGs to assets const assetsDir = path.join(__dirname, "assets"); if (!fs.existsSync(assetsDir)) { fs.mkdirSync(assetsDir); } -fs.renameSync(reqSecHistogramFile, path.join(assetsDir, reqSecHistogramFile)); -fs.renameSync(latencyHistogramFile, path.join(assetsDir, latencyHistogramFile)); -// Calculate relative performance and build the results table +function moveFile(source, destination) { + try { + if (fs.existsSync(source)) { + fs.renameSync(source, destination); + console.log(`Moved ${source} to ${destination}`); + } else { + console.log(`Source file ${source} does not exist`); + } + } catch (error) { + console.error(`Error moving file ${source}: ${error.message}`); + } +} + +moveFile(reqSecHistogramFile, path.join(assetsDir, reqSecHistogramFile)); +moveFile(latencyHistogramFile, path.join(assetsDir, latencyHistogramFile)); + const serverRPS = {}; servers.forEach((server) => { serverRPS[server] = avgReqSecs[server]; @@ -127,14 +158,28 @@ const sortedServers = Object.keys(serverRPS).sort( const lastServer = sortedServers[sortedServers.length - 1]; const lastServerReqSecs = avgReqSecs[lastServer]; +const resultsFile = "results.md"; + +if ( + !fs.existsSync(resultsFile) || + fs.readFileSync(resultsFile, "utf8").trim() === "" +) { + fs.writeFileSync( + resultsFile, + ` +| Query | Server | Requests/sec | Latency (ms) | Relative | +|-------:|--------:|--------------:|--------------:|---------:|` + ); +} + let resultsTable = ""; if (whichBench === 1) { - resultsTable += `\n\n| Query | Server | Requests/sec | Latency (ms) | Relative |\n|-------:|--------:|--------------:|--------------:|---------:|\n| ${whichBench} | \`{ posts { id userId title user { id name email }}}\` |`; + resultsTable += `\n| ${whichBench} | \`{ posts { id userId title user { id name email }}}\` |`; } else if (whichBench === 2) { - resultsTable += `| ${whichBench} | \`{ posts { title }}\` |`; + resultsTable += `\n| ${whichBench} | \`{ posts { title }}\` |`; } else if (whichBench === 3) { - resultsTable += `| ${whichBench} | \`{ greet }\` |`; + resultsTable += `\n| ${whichBench} | \`{ greet }\` |`; } sortedServers.forEach((server) => { @@ -153,14 +198,11 @@ sortedServers.forEach((server) => { resultsTable += `\n|| [${formattedServerNames[server]}] | \`${formattedReqSecs}\` | \`${formattedLatencies}\` | \`${relativePerformance}x\` |`; }); -if (whichBench === 3) { - resultsTable += `\n\n`; -} - -const resultsFile = "results.md"; -fs.writeFileSync(resultsFile, resultsTable); +fs.appendFileSync(resultsFile, resultsTable); if (whichBench === 3) { + fs.appendFileSync(resultsFile, "\n\n"); + const finalResults = fs .readFileSync(resultsFile, "utf-8") .replace(/(\r\n|\n|\r)/gm, "\\n"); From 85fc350e3323f2b39478d38052b18defce80d95d Mon Sep 17 00:00:00 2001 From: vijay Date: Fri, 19 Jul 2024 16:24:51 +0530 Subject: [PATCH 4/4] Change commands to node --- run_benchmarks.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/run_benchmarks.sh b/run_benchmarks.sh index e665e6f1..09d625e3 100755 --- a/run_benchmarks.sh +++ b/run_benchmarks.sh @@ -83,6 +83,6 @@ for service in "apollo_server" "caliban" "netflix_dgs" "gqlgen" "tailcall" "asyn fi done -bash analyze.js "${bench1Results[@]}" -bash analyze.js "${bench2Results[@]}" -bash analyze.js "${bench3Results[@]}" +node analyze.js "${bench1Results[@]}" +node analyze.js "${bench2Results[@]}" +node analyze.js "${bench3Results[@]}"