Repository navigation
Expand file tree
/
Copy pathbm
More file actions
executable file
·371 lines (352 loc) · 13.9 KB
/
Copy pathbm
File metadata and controls
executable file
·371 lines (352 loc) · 13.9 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
#!/usr/bin/env bash
# Compare release-build CLI throughput across LuaJIT, Zig/C, Rust, Lean, and Roc.
# Seeded outputs are proved byte-identical before any timing is recorded.
set -u
set -o pipefail
here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
cd "$here" || exit 1
IFS=$' \t\n'
mode=full
case "${1-}" in
"") ;;
--simd) shift; exec "$here/experiments/bm-fixed-simd" "$@" ;;
--ziggurat) shift; exec "$here/experiments/bm-ziggurat" "$@" ;;
--bend2) shift; exec "$here/experiments/bm-bend2" "$@" ;;
--quick) mode=quick ;;
--check) mode=check ;;
--help|-h)
printf '%s\n' \
'Usage: ./bm [--quick|--check|--simd [--check]|--ziggurat|--bend2 [profile options]]' \
'' \
'Benchmarks byte-identical deterministic work through random (LuaJIT),' \
'randomz (Zig/C), randomr (Rust), randoml (Lean), and randomroc (Roc, Linux).' \
'Rust timing bypasses only the' \
'Nix self-test wrapper; the packaged CLI remains in the parity check.' \
'Parity payloads stream through SHA-256 and are never written to disk.' \
'Hyperfine executes each payload directly, without an intermediate shell,' \
'and large single-process batches amortize unavoidable program startup.' \
'Builds happen before timing.' \
'' \
' --quick Use smaller workloads and three measured runs' \
' --check Build and prove every benchmark workload identical; do not time' \
' --simd Opt-in exact fixed-point Zig/Rust AVX2 experiments (x86_64)' \
' --ziggurat Opt-in fixed-point normal sampler experiment (Linux)' \
' --bend2 Opt-in Bend 2 native/JS core comparison (x86_64 Linux)' \
' --full: nine exact sampler workloads, six backends, CPU/wall, 1/12 cores' \
' --full-cli: 13 workloads across all six CLIs, 1/12 cores' \
'' \
'Environment: BM_SKIP_BUILD=1, BM_RUNS, BM_WARMUP, BM_COUNT,' \
'BM_RAW_BYTES, BM_MACHINE_ID, BM_RESULTS_FILE, BM_RUST_TIMING_CLI,' \
'BM_ROC_CLI, BM_SIMD_RESULTS_FILE (for --simd).'
exit 0
;;
*)
echo "Unknown argument: $1" >&2
exit 1
;;
esac
require_command() {
if ! command -v "$1" >/dev/null 2>&1; then
echo "./bm requires $1" >&2
exit 1
fi
}
if [ "${BM_SKIP_BUILD:-0}" != 1 ]; then
for implementation in luajit rust lean; do
"$here/build" --impl "$implementation" || exit 1
done
# Keep the existing native-GNU benchmark ABI explicit; the portable
# canonical Linux command defaults to musl instead.
zig_options=()
[ "$(uname -s)" != Linux ] || zig_options=(--variant gnu)
"$here/build" --impl zig "${zig_options[@]}" || exit 1
[ "$(uname -s)" != Linux ] || "$here/build" --impl roc || exit 1
fi
lua_product="$("$here/build-support/product-path" luajit)" || exit 1
zig_options=()
[ "$(uname -s)" != Linux ] || zig_options=(--variant gnu)
zig_product="$("$here/build-support/product-path" zig "${zig_options[@]}")" || exit 1
rust_product="$("$here/build-support/product-path" rust)" || exit 1
lean_product="$("$here/build-support/product-path" lean)" || exit 1
lua_cli="$lua_product/bin/random"
zig_cli="$zig_product/bin/randomz"
rust_cli="$rust_product/bin/randomr"
lean_cli="$lean_product/bin/randoml"
rust_timing_cli="${BM_RUST_TIMING_CLI:-$rust_product/libexec/randomr}"
implementations=(luajit zig rust lean)
parity_executables=("$lua_cli" "$zig_cli" "$rust_cli" "$lean_cli")
timing_executables=("$lua_cli" "$zig_cli" "$rust_timing_cli" "$lean_cli")
if [ "$(uname -s)" = Linux ] || [ -n "${BM_ROC_CLI:-}" ]; then
roc_cli="${BM_ROC_CLI:-}"
if [ -z "$roc_cli" ] && [ "${BM_SKIP_BUILD:-0}" != 1 ]; then
roc_cli="$("$here/build-support/product-path" roc)/bin/randomroc"
fi
if [ -z "$roc_cli" ]; then
root="$here"
source "$here/tests/roc_native_setup.sh"
roc_native_setup || exit 1
roc_cli="$RANDOM_ROC_NATIVE_DIR/bin/randomroc"
fi
implementations+=(roc)
parity_executables+=("$roc_cli")
timing_executables+=("$roc_cli")
fi
for executable in "${parity_executables[@]}" "$rust_timing_cli"; do
if [ ! -x "$executable" ]; then
echo "./bm requires a release executable: $executable" >&2
echo "Unset BM_SKIP_BUILD to build each requested implementation." >&2
exit 1
fi
done
if [ "$mode" = quick ]; then
default_runs=3
default_warmup=1
default_count=5000
default_raw_bytes=1048576
else
default_runs=7
default_warmup=2
default_count=20000
default_raw_bytes=8388608
fi
runs="${BM_RUNS:-$default_runs}"
warmup="${BM_WARMUP:-$default_warmup}"
count="${BM_COUNT:-$default_count}"
raw_bytes="${BM_RAW_BYTES:-$default_raw_bytes}"
for numeric_setting in "$runs" "$warmup" "$count" "$raw_bytes"; do
if ! [[ "$numeric_setting" =~ ^[0-9]+$ ]]; then
echo "Benchmark counts and run settings must be non-negative integers" >&2
exit 1
fi
done
if [ "$runs" -eq 0 ] || [ "$count" -eq 0 ] || [ "$raw_bytes" -eq 0 ]; then
echo "BM_RUNS, BM_COUNT, and BM_RAW_BYTES must be positive" >&2
exit 1
fi
case_names=(
raw_bytes
normalized_bytes
hex_bytes
base64_bytes
uniform_small
uniform_wide
normal
exponential
poisson
geometric
geometric_sparse
log_normal
beta
)
case_arguments=(
"-d --seed 42 --binaryoutput --count $raw_bytes"
"-d --seed 42 --binaryoutput --normalized --count $count"
"-d --seed 42 --binaryoutput --hex --count $raw_bytes"
"-d --seed 42 --binaryoutput --base64 --count $raw_bytes"
"-d --seed 42 --count $count 0..99"
"-d --seed 42 --count $count -1000000000000..1000000000000"
"-d --seed 42 --normalized --mean=-0.25 --stddev=1.75 --count $count"
"-d --seed 42 --exponential --rate=1.75 --count $count"
"-d --seed 42 --poisson --lambda=5 --count $count"
"-d --seed 42 --geometric --probability=0.25 --count $count"
"-d --seed 42 --geometric --probability=1e-20 --count $count"
"-d --seed 42 --log-normal --mean=-0.25 --stddev=1.75 --count $count"
"-d --seed 42 --beta=3 --alpha=1 --count $count"
)
require_command sha256sum
digest_output() {
local case_name="$1"
local implementation="$2"
local executable="$3"
local digest
shift 3
# Continuation metadata is deliberately not part of the benchmark payload;
# suppress its JSON stderr while pipefail still preserves producer failure.
digest="$("$executable" "$@" 2>/dev/null | sha256sum)" || {
echo "Benchmark producer or digest failed: $case_name/$implementation" >&2
return 1
}
digest="${digest%% *}"
if ! [[ "$digest" =~ ^[[:xdigit:]]{64}$ ]]; then
echo "Invalid SHA-256 digest for benchmark workload: $case_name/$implementation" >&2
return 1
fi
printf '%s' "$digest"
}
parity_labels=("${implementations[@]}" rust-timing)
parity_digest_executables=("${parity_executables[@]}" "$rust_timing_cli")
echo "Checking benchmark workload equivalence..."
for case_index in "${!case_names[@]}"; do
case_name="${case_names[$case_index]}"
read -r -a args <<<"${case_arguments[$case_index]}"
parity_digests=()
for implementation_index in "${!parity_labels[@]}"; do
parity_digests+=("$(digest_output "$case_name" \
"${parity_labels[$implementation_index]}" \
"${parity_digest_executables[$implementation_index]}" "${args[@]}")") || exit 1
done
parity_mismatch=false
for digest in "${parity_digests[@]:1}"; do
if [ "${parity_digests[0]}" != "$digest" ]; then
parity_mismatch=true
break
fi
done
if "$parity_mismatch"; then
echo "Benchmark workload differs across implementations: $case_name" >&2
for implementation_index in "${!parity_labels[@]}"; do
printf ' %s %s\n' "${parity_digests[$implementation_index]}" \
"${parity_labels[$implementation_index]}" >&2
done
exit 1
fi
printf ' ✓ %s\n' "$case_name"
done
echo "All ${#case_names[@]} benchmark workloads are byte-identical."
if [ "$mode" = check ]; then
exit 0
fi
require_command hyperfine
require_command jq
machine_raw="${BM_MACHINE_ID:-}"
if [ -z "$machine_raw" ] && [ -r /etc/machine-id ]; then
machine_raw="$(tr -d '\n' </etc/machine-id)"
fi
if [ -z "$machine_raw" ]; then
machine_raw="$(hostname 2>/dev/null || uname -n)"
fi
machine_id="$(printf '%s' "$machine_raw" | tr -c 'A-Za-z0-9._-' '_')"
history_file="${BM_RESULTS_FILE:-$here/benchmarks/$machine_id.ndjson}"
mkdir -p "$(dirname "$history_file")" || exit 1
touch "$history_file" || exit 1
timestamp="$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
git_commit="$(git rev-parse HEAD 2>/dev/null || printf unknown)"
if [ -n "$(git status --porcelain 2>/dev/null)" ]; then
git_dirty=true
else
git_dirty=false
fi
platform="$(uname -srm)"
cpu_model="$(lscpu 2>/dev/null | sed -n 's/^Model name:[[:space:]]*//p' | head -n 1)"
lua_shebang="$(head -n 1 "$lua_cli")"
case "$lua_shebang" in
'#!'/*)
lua_interpreter="${lua_shebang#\#!}"
luajit_version="$("$lua_interpreter" -v 2>&1 | head -n 1)"
;;
*) luajit_version="$(luajit -v 2>&1 | head -n 1)" ;;
esac
zig_version="$(zig version 2>/dev/null || printf unknown)"
rust_version="$(rustc --version 2>/dev/null || printf unknown)"
lean_version="$(lean --version 2>/dev/null | head -n 1 || printf unknown)"
roc_compiler="${RANDOM_ROC_COMPILER:-roc}"
roc_version="$("$roc_compiler" version 2>/dev/null | head -n 1 || printf unknown)"
hyperfine_version="$(hyperfine --version 2>/dev/null || printf unknown)"
echo
echo "Benchmarking release executables ($runs runs, $warmup warmups)..."
echo "Execution: direct argv (Hyperfine --shell=none); startup amortized over each batch"
echo "Rust timing payload: $rust_timing_cli (the packaged CLI remains in the parity preflight)"
echo "History: $history_file"
timing_surface="implementation-payload-v1"
for case_index in "${!case_names[@]}"; do
case_name="${case_names[$case_index]}"
arguments="${case_arguments[$case_index]}"
hyperfine_args=(
-N --warmup "$warmup" --runs "$runs" --style basic --sort command
--output null --export-json /dev/fd/3
)
for implementation_index in "${!implementations[@]}"; do
hyperfine_args+=(
-n "${implementations[$implementation_index]}"
"${timing_executables[$implementation_index]} $arguments"
)
done
echo
echo "== $case_name: $arguments =="
# Keep Hyperfine's report visible on stderr while its JSON export travels
# through fd 3 into this in-memory text string. No temporary result
# file is created.
result_json="$(hyperfine "${hyperfine_args[@]}" 3>&1 1>&2)" || exit 1
# /dev/fd/3 is a pipe: successive exports append cumulative snapshots.
# Parse and validate before the loop so a failed jq cannot silently pass.
expected_results="$(printf '%s\n' "${implementations[@]}" | jq -Rsc 'split("\n")[:-1]')" || exit 1
parsed_results="$(jq -cse --argjson expected "$expected_results" \
-f "$here/tools/hyperfine_results.jq" <<<"$result_json")" || exit 1
while IFS= read -r result; do
implementation="$(jq -r '.command' <<<"$result")"
mean_seconds="$(jq -r '.mean' <<<"$result")"
user_seconds="$(jq -r '.user' <<<"$result")"
system_seconds="$(jq -r '.system' <<<"$result")"
cpu_seconds="$(jq -r '.user + .system' <<<"$result")"
median_seconds="$(jq -r '.median' <<<"$result")"
stddev_seconds="$(jq -r '.stddev' <<<"$result")"
baseline="$(jq -r --arg case "$case_name" --arg implementation "$implementation" \
--arg arguments "$arguments" --arg timing_surface "$timing_surface" \
'select(.case == $case and .implementation == $implementation and
.arguments == $arguments and .output_sink == "null" and
.timing_surface == $timing_surface) | .cpu_seconds' \
"$history_file" 2>/dev/null | tail -n 1)"
if [ -n "$baseline" ] && [ "$baseline" != null ] && [ "$baseline" != 0 ]; then
awk -v current="$cpu_seconds" -v baseline="$baseline" \
-v label="$case_name/$implementation" 'BEGIN {
ratio = current / baseline
if (ratio > 1.15) {
printf " WARNING: %s CPU time regressed %.1f%% versus prior machine baseline\n", label, (ratio - 1) * 100
} else if (ratio < 0.85) {
printf " REVIEW: %s CPU time improved %.1f%%; verify no work was lost\n", label, (1 - ratio) * 100
}
}'
fi
case "$implementation" in
luajit) measured_executable="$lua_cli" ;;
zig) measured_executable="$zig_cli" ;;
rust) measured_executable="$rust_timing_cli" ;;
lean) measured_executable="$lean_cli" ;;
roc) measured_executable="$roc_cli" ;;
*) echo "Unknown hyperfine result: $implementation" >&2; exit 1 ;;
esac
binary_sha256="$(sha256sum "$measured_executable" | awk '{print $1}')"
jq -cn \
--arg timestamp "$timestamp" \
--arg machine_id "$machine_id" \
--arg git_commit "$git_commit" \
--argjson git_dirty "$git_dirty" \
--arg platform "$platform" \
--arg cpu_model "$cpu_model" \
--arg case "$case_name" \
--arg implementation "$implementation" \
--arg arguments "$arguments" \
--arg timing_surface "$timing_surface" \
--arg binary_sha256 "$binary_sha256" \
--arg luajit_version "$luajit_version" \
--arg zig_version "$zig_version" \
--arg rust_version "$rust_version" \
--arg lean_version "$lean_version" \
--arg roc_version "$roc_version" \
--arg hyperfine_version "$hyperfine_version" \
--argjson runs "$runs" \
--argjson warmup "$warmup" \
--argjson mean_seconds "$mean_seconds" \
--argjson median_seconds "$median_seconds" \
--argjson stddev_seconds "$stddev_seconds" \
--argjson user_seconds "$user_seconds" \
--argjson system_seconds "$system_seconds" \
--argjson cpu_seconds "$cpu_seconds" \
'{schema_version: 2, timestamp: $timestamp, machine_id: $machine_id,
git_commit: $git_commit, git_dirty: $git_dirty, platform: $platform,
cpu_model: $cpu_model, case: $case, implementation: $implementation,
arguments: $arguments, output_sink: "null", timing_surface: $timing_surface,
runs: $runs, warmup: $warmup,
mean_seconds: $mean_seconds, median_seconds: $median_seconds,
stddev_seconds: $stddev_seconds, user_seconds: $user_seconds,
system_seconds: $system_seconds, cpu_seconds: $cpu_seconds,
binary_sha256: $binary_sha256,
luajit_version: $luajit_version, zig_version: $zig_version,
rust_version: $rust_version, lean_version: $lean_version, roc_version: $roc_version,
hyperfine_version: $hyperfine_version}' \
>>"$history_file" || exit 1
done <<<"$parsed_results"
done
echo
echo "Benchmark results appended to $history_file"
echo "Two-sided ±15% CPU-time changes are reported for review, not auto-accepted."