chore: Reimplement analyze.sh in JS

README.md

@@ -42,14 +42,10 @@ Get started with the benchmarks:
 ## Benchmark Results
 
 <!-- PERFORMANCE_RESULTS_START -->
-
-| Server           | Requests/sec | Latency (ms) |
-| ---------------- | -----------: | -----------: |
-| [Tailcall]       |   `2,890.68` |      `34.69` |
-| [Gqlgen]         |     `935.00` |     `115.73` |
-| [Apollo GraphQL] |     `793.37` |     `128.22` |
-| [Netflix DGS]    |     `597.39` |     `191.85` |
-
+| Server | Requests/sec | Latency (ms) |
+|--------|--------------|--------------|
+| apollo | 6543.885258627874 | 15.464760673819313 |
+| tailcall | 121400.78711951857 | 0.8161994372989355 |
 <!-- PERFORMANCE_RESULTS_END -->
 
 ### Throughput (Higher is better)

analyze.js

@@ -0,0 +1,129 @@
+#!/usr/bin/env node
+
+const fs = require("fs");
+const { execSync } = require("child_process");
+
+const average = (arr) => arr.reduce((a, b) => a + b, 0) / arr.length;
+
+const servers = ["apollo", "netflixdgs", "gqlgen", "tailcall"];
+
+const resultFiles = process.argv
+  .slice(2)
+  .filter((arg) => arg.startsWith("result") && fs.existsSync(arg));
+
+const extractMetrics = (files) => {
+  return files.map((file) => {
+    const { summary } = JSON.parse(fs.readFileSync(file, "utf8"));
+    return {
+      requestsPerSec: summary.requestsPerSec,
+      latency: summary.average * 1000,
+    };
+  });
+};
+
+const calculateAverages = (metrics) => {
+  return servers.reduce((acc, server, index) => {
+    const serverMetrics = metrics.slice(index * 3, (index + 1) * 3);
+    acc[server] = {
+      avgReqSec: average(serverMetrics.map((m) => m.requestsPerSec)),
+      avgLatency: average(serverMetrics.map((m) => m.latency)),
+    };
+    return acc;
+  }, {});
+};
+
+const writeDataFile = (filename, data) => {
+  fs.writeFileSync(filename, "Server Value\n");
+  Object.entries(data).forEach(([server, value]) => {
+    fs.appendFileSync(filename, `${server} ${value}\n`);
+  });
+};
+
+const generateGnuplotScript = (reqSecData, latencyData) => `
+set term pngcairo size 1280,720 enhanced font "Courier,12"
+set style data histograms
+set style histogram cluster gap 1
+set style fill solid border -1
+set xtics rotate by -45
+set boxwidth 0.9
+set key outside right top
+
+set output "req_sec_histogram.png"
+set title "Requests/Sec"
+stats "${reqSecData}" using 2 nooutput
+set yrange [0:*]
+plot "${reqSecData}" using 2:xtic(1) title "Req/Sec"
+
+set output "latency_histogram.png"
+set title "Latency (in ms)"
+stats "${latencyData}" using 2 nooutput
+set yrange [0:*]
+plot "${latencyData}" using 2:xtic(1) title "Latency"
+`;
+
+const generateResultsTable = (averages) => {
+  let table =
+    "<!-- PERFORMANCE_RESULTS_START -->\n| Server | Requests/sec | Latency (ms) |\n|--------|--------------|--------------|";
+  Object.entries(averages).forEach(([server, { avgReqSec, avgLatency }]) => {
+    table += `\n| ${server} | ${avgReqSec} | ${avgLatency} |`;
+  });
+  return table + "\n<!-- PERFORMANCE_RESULTS_END -->";
+};
+
+const updateReadme = (resultsTable) => {
+  const readmePath = "README.md";
+  let content = fs.readFileSync(readmePath, "utf8");
+  const regex =
+    /<!-- PERFORMANCE_RESULTS_START -->[\s\S]*<!-- PERFORMANCE_RESULTS_END -->/;
+  content = content.includes("PERFORMANCE_RESULTS_START")
+    ? content.replace(regex, resultsTable)
+    : content + "\n\n" + resultsTable;
+  fs.writeFileSync(readmePath, content);
+};
+
+const main = () => {
+  const metrics = extractMetrics(resultFiles);
+  const averages = calculateAverages(metrics);
+
+  resultFiles.forEach(fs.unlinkSync);
+
+  const reqSecData = "/tmp/reqSec.dat";
+  const latencyData = "/tmp/latency.dat";
+
+  writeDataFile(
+    reqSecData,
+    Object.fromEntries(
+      Object.entries(averages).map(([k, v]) => [k, v.avgReqSec])
+    )
+  );
+  writeDataFile(
+    latencyData,
+    Object.fromEntries(
+      Object.entries(averages).map(([k, v]) => [k, v.avgLatency])
+    )
+  );
+
+  const gnuplotScript = generateGnuplotScript(reqSecData, latencyData);
+  fs.writeFileSync("/tmp/gnuplot_script", gnuplotScript);
+  execSync("gnuplot /tmp/gnuplot_script");
+
+  if (!fs.existsSync("assets")) fs.mkdirSync("assets");
+  ["req_sec_histogram.png", "latency_histogram.png"].forEach((file) =>
+    fs.renameSync(file, `assets/${file}`)
+  );
+
+  const resultsTable = generateResultsTable(averages);
+  updateReadme(resultsTable);
+
+  console.log(resultsTable.replace(/<!--.*?-->\n?/g, ""));
+
+  execSync(
+    "git add README.md assets/req_sec_histogram.png assets/latency_histogram.png"
+  );
+  execSync('git commit -m "Updated performance results in README.md"');
+  execSync("git push");
+
+  resultFiles.forEach(fs.unlinkSync);
+};
+
+main();

generate_load.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+PORT=$1
+
+oha -z 10s -c 100 -m POST \
+    -H "Connection: keep-alive" \
+    -H "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36" \
+    -H "Content-Type: application/json" \
+    -d '{"operationName":null,"variables":{},"query":"{posts{title}}"}' \
+    --json \
+    http://localhost:$PORT/graphql

graphql/tailcall/benchmark.graphql

@@ -1,31 +1,23 @@
 schema
-  @server(
-    port: 8083
-    baseURL: "http://jsonplaceholder.typicode.com"
-    enableHttpCache: false
-    enableGraphiql: "/graphiql"
-    proxy: {url: "http://localhost:3000"}
-  ) {
+  @server(port: 8083)
+  @upstream(baseURL: "http://jsonplaceholder.typicode.com") {
   query: Query
 }
 
 type Query {
-  posts: [Post] @http(path: "/posts")
+  users: [User] @http(path: "/users")
 }
 
 type User {
   id: Int!
   name: String!
   username: String!
   email: String!
-  phone: String
-  website: String
+  posts: [Post] @http(path: "/users/{{.value.id}}/posts")
 }
 
 type Post {
   id: Int!
-  userId: Int!
   title: String!
   body: String!
-  user: User @http(path: "/users/{{value.userId}}")
 }

graphql/tailcall/run.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 current_dir=$(pwd)
-$HOME/.tailcall/bin/tc start $current_dir/graphql/tailcall/benchmark.graphql
+$HOME/.tailcall/bin/tailcall start $current_dir/graphql/tailcall/benchmark.graphql

run_benchmarks.sh

@@ -18,42 +18,42 @@ sh nginx/run.sh
 
 function runBenchmark() {
     local serviceScript="$1"
-    local benchmarkScript="$2"
+    local port="$2"
 
     # Replace / with _
     local sanitizedServiceScriptName=$(echo "$serviceScript" | tr '/' '_')
 
-    local resultFiles=("result1_${sanitizedServiceScriptName}.txt" "result2_${sanitizedServiceScriptName}.txt" "result3_${sanitizedServiceScriptName}.txt")
+    local resultFiles=("result1_${sanitizedServiceScriptName}.json" "result2_${sanitizedServiceScriptName}.json" "result3_${sanitizedServiceScriptName}.json")
 
     bash "$serviceScript" &   # Run in daemon mode
     sleep 15   # Give some time for the service to start up
 
     # Warmup run
-    bash "$benchmarkScript" > /dev/null
+    bash "./generate_load.sh" "$port" > /dev/null
 
     # 3 benchmark runs
     for resultFile in "${resultFiles[@]}"; do
-        bash "$benchmarkScript" > "$resultFile"
+        bash "./generate_load.sh" "$port" > "$resultFile"
         allResults+=("$resultFile")
     done
 }
 
-runBenchmark "graphql/apollo_server/run.sh" "wrk/apollo_bench.sh"
+runBenchmark "graphql/apollo_server/run.sh" 8080
 cd graphql/apollo_server/
 npm stop
 cd ../../
 
-killServerOnPort 8082
-runBenchmark "graphql/netflix_dgs/run.sh" "wrk/dgs_bench.sh"
-killServerOnPort 8082
+# killServerOnPort 8082
+# runBenchmark "graphql/netflix_dgs/run.sh" 8082
+# killServerOnPort 8082
 
-killServerOnPort 8081
-runBenchmark "graphql/gqlgen/run.sh" "wrk/gqlgen_bench.sh"
-killServerOnPort 8081
+# killServerOnPort 8081
+# runBenchmark "graphql/gqlgen/run.sh" 8081
+# killServerOnPort 8081
 
 killServerOnPort 8083
-runBenchmark "graphql/tailcall/run.sh" "wrk/tc_bench.sh"
+runBenchmark "graphql/tailcall/run.sh" 8083
 killServerOnPort 8083
 
 # Now, analyze all results together
-bash analyze.sh "${allResults[@]}"
+node analyze.js "${allResults[@]}"

setup.sh

@@ -15,5 +15,7 @@ cd graphql/netflix_dgs
 cd ../../
 
 # For tailcall:
-curl -sSL https://raw.githubusercontent.com/tailcallhq/tailcall/main/install.sh | bash -s -- v0.10.0
+curl -sSL https://raw.githubusercontent.com/tailcallhq/tailcall/main/install.sh | bash -s
 export PATH=$PATH:/root/.tailcall/bin
+
+cargo install oha