Skip to content
This repository was archived by the owner on Mar 17, 2025. It is now read-only.

Commit f5c7121

Browse files
authored
Benchmarks in parallel (#357)
1 parent 1814866 commit f5c7121

File tree

4 files changed

+98
-28
lines changed

4 files changed

+98
-28
lines changed

.github/workflows/bench.yml

Lines changed: 40 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,50 +1,81 @@
11
name: "Run benchmark"
2-
32
on:
43
pull_request_target:
54
types: [assigned, opened, synchronize, reopened, edited]
65
push:
76
branches:
87
- main
9-
108
permissions:
119
contents: write
1210
pull-requests: write
1311
issues: write
14-
1512
jobs:
1613
build:
1714
runs-on: benchmarking-runner
1815
if: github.event.head_commit.message != 'Update performance results in README.md'
1916
env:
2017
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
18+
strategy:
19+
matrix:
20+
service: [apollo_server, caliban, netflix_dgs, gqlgen, tailcall, async_graphql, hasura, graphql_jit]
2121
steps:
2222
- name: Checkout (GitHub)
2323
uses: actions/checkout@v4
2424
with:
2525
token: ${{ secrets.GITHUB_TOKEN }}
2626
ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || github.sha }}
27-
28-
- name: Build devcontainer and run benchmarks
27+
28+
- name: Build devcontainer and run setup and benchmark
2929
uses: devcontainers/[email protected]
3030
with:
3131
imageName: graphql-benchmarks
3232
push: never
3333
runCmd: |
34-
bash ./setup.sh
35-
bash ./run_benchmarks.sh
34+
bash ./graphql/${{ matrix.service }}/setup.sh
35+
bash run_benchmarks.sh ${{ matrix.service }}
36+
37+
- name: List benchmark files
38+
run: |
39+
ls -la bench*.txt || echo "No matching files found"
40+
41+
- name: Upload benchmark results
42+
uses: actions/upload-artifact@v3
43+
with:
44+
name: benchmark-results
45+
path: bench*.txt
46+
47+
48+
analyze:
49+
needs: benchmark
50+
runs-on: benchmarking-runner
51+
steps:
52+
- name: Checkout (GitHub)
53+
uses: actions/checkout@v4
54+
55+
- name: Download all benchmark results
56+
uses: actions/download-artifact@v3
57+
with:
58+
name: benchmark-results
59+
path: .
60+
61+
- name: List downloaded artifacts
62+
run: ls -la bench*.txt || echo "No matching files found"
63+
64+
- name: Analyze results
65+
run: |
66+
bash run_analyze_script.sh
3667
3768
- name: Print benchmark results
3869
run: cat ./results.md
39-
70+
4071
- name: Comment benchmark results on PR
4172
if: github.event_name == 'pull_request_target'
4273
uses: peter-evans/commit-comment@v3
4374
with:
4475
sha: ${{ github.event.pull_request.head.sha }}
4576
body-path: "results.md"
4677
reactions: eyes
47-
78+
4879
- name: Commit and push changes (on main branch)
4980
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
5081
uses: stefanzweifel/git-auto-commit-action@v5

analyze.sh

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
#!/bin/bash
22

3+
# Install gnuplot
4+
sudo apt-get update && sudo apt-get install -y gnuplot
5+
36
function extractMetric() {
47
local file="$1"
58
local metric="$2"
@@ -161,4 +164,4 @@ mv $latencyHistogramFile assets/
161164
# Delete the result TXT files
162165
for file in "${resultFiles[@]}"; do
163166
rm "$file"
164-
done
167+
done

run_analyze_script.sh

Lines changed: 31 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,36 @@
11
#!/bin/bash
22

3-
rm results.md
3+
# Update and install gnuplot
4+
sudo apt-get update && sudo apt-get install -y gnuplot
45

6+
# Remove existing results file
7+
rm -f results.md
8+
9+
services=("apollo" "caliban" "netflixdgs" "gqlgen" "tailcall" "async_graphql" "hasura" "graphql_jit")
10+
11+
# Loop through each benchmark (1, 2, 3)
512
for bench in 1 2 3; do
6-
if ls bench${bench}*.txt &> /dev/null; then
7-
echo "Processing files for bench${bench}:"
8-
bash analyze.sh bench${bench}*.txt
9-
echo "Files processed: $(ls bench${bench}*.txt)"
10-
else
11-
echo "No matching files found for bench${bench}*.txt"
12-
fi
13+
echo "Processing files for bench${bench}:"
14+
15+
# Construct the command for each benchmark
16+
cmd="bash analyze.sh"
17+
18+
# Loop through each service
19+
for service in "${services[@]}"; do
20+
# Convert service name to match file naming convention
21+
case $service in
22+
"apollo") file_service="apollo_server" ;;
23+
"netflixdgs") file_service="netflix_dgs" ;;
24+
*) file_service=$service ;;
25+
esac
26+
27+
# Add files for current service to the command
28+
for run in 1 2 3; do
29+
cmd+=" bench${bench}_result${run}_graphql_${file_service}_run.sh.txt"
30+
done
31+
done
32+
33+
# Execute the command
34+
echo "Executing: $cmd"
35+
eval $cmd
1336
done

run_benchmarks.sh

Lines changed: 23 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ function killServerOnPort() {
1212
echo "No process found running on port $port"
1313
fi
1414
}
15+
1516
bench1Results=()
1617
bench2Results=()
1718
bench3Results=()
@@ -72,17 +73,29 @@ function runBenchmark() {
7273

7374
rm "results.md"
7475

75-
for service in "apollo_server" "caliban" "netflix_dgs" "gqlgen" "tailcall" "async_graphql" "hasura" "graphql_jit"; do
76-
runBenchmark "graphql/${service}/run.sh"
77-
if [ "$service" == "apollo_server" ]; then
76+
# Main script
77+
if [ $# -eq 0 ]; then
78+
echo "Usage: $0 <service_name>"
79+
echo "Available services: apollo_server, caliban, netflix_dgs, gqlgen, tailcall, async_graphql, hasura, graphql_jit"
80+
exit 1
81+
fi
82+
83+
service="$1"
84+
valid_services=("apollo_server" "caliban" "netflix_dgs" "gqlgen" "tailcall" "async_graphql" "hasura" "graphql_jit")
85+
86+
if [[ ! " ${valid_services[@]} " =~ " ${service} " ]]; then
87+
echo "Invalid service name. Available services: ${valid_services[*]}"
88+
exit 1
89+
fi
90+
91+
92+
93+
runBenchmark "graphql/${service}/run.sh"
94+
95+
if [ "$service" == "apollo_server" ]; then
7896
cd graphql/apollo_server/
7997
npm stop
8098
cd ../../
81-
elif [ "$service" == "hasura" ]; then
99+
elif [ "$service" == "hasura" ]; then
82100
bash "graphql/hasura/kill.sh"
83-
fi
84-
done
85-
86-
bash analyze.sh "${bench1Results[@]}"
87-
bash analyze.sh "${bench2Results[@]}"
88-
bash analyze.sh "${bench3Results[@]}"
101+
fi

0 commit comments

Comments
 (0)