Skip to content

Commit 32cdac7

Browse files
richiemcilroyrichiemcilroy
committed
ci: camera fan-out copy A/B bench
1 parent d16fd19 commit 32cdac7

3 files changed

Lines changed: 509 additions & 0 deletions

File tree

‎.github/workflows/camera-fanout-ab.yml‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
name: camera-fanout-ab
2+
3+
# Scratch A/B: camera feed fan-out copies, main vs perf/recording-camera-fanout-copies. Not for main.
4+
5+
on:
6+
push:
7+
branches: [ci/camera-fanout-ab]
8+
workflow_dispatch:
9+
10+
env:
11+
MAIN_SHA: d16fd199bbe8bd2a1c52fb19bcb3d383e9a2ff61
12+
FIX_REF: perf/recording-camera-fanout-copies
13+
14+
jobs:
15+
bench:
16+
runs-on: blacksmith-8vcpu-ubuntu-2404
17+
timeout-minutes: 120
18+
env:
19+
RUST_TARGET_TRIPLE: x86_64-unknown-linux-gnu
20+
CARGO_PROFILE_RELEASE_LTO: "false"
21+
CARGO_PROFILE_RELEASE_CODEGEN_UNITS: "16"
22+
CARGO_PROFILE_RELEASE_DEBUG: "0"
23+
steps:
24+
- uses: actions/checkout@v4
25+
with:
26+
fetch-depth: 0
27+
28+
- name: Stage bench and check out main
29+
run: |
30+
cp -R scripts/camera-fanout-ab "$RUNNER_TEMP/camera-fanout-ab"
31+
git checkout -f "$MAIN_SHA"
32+
33+
- name: Install Linux desktop dependencies
34+
uses: ./.github/actions/install-desktop-deps
35+
36+
- uses: dtolnay/rust-toolchain@1.88.0
37+
38+
- uses: ./.github/actions/setup-rust-cache
39+
with:
40+
target: x86_64-unknown-linux-gnu
41+
prefix-key: v1-rust-camera-fanout-ab
42+
save-cache: "true"
43+
44+
- name: Setup
45+
run: node scripts/setup.js
46+
47+
- name: Build both variants
48+
run: |
49+
set +e
50+
build() {
51+
local variant=$1
52+
cp "$RUNNER_TEMP/camera-fanout-ab/camera-fanout-bench.rs" crates/recording/examples/camera-fanout-bench.rs
53+
cargo build --release --locked -p cap-recording --example camera-fanout-bench --features test-utils || exit 1
54+
mkdir -p "$RUNNER_TEMP/bin-$variant"
55+
cp target/release/examples/camera-fanout-bench "$RUNNER_TEMP/bin-$variant/" || exit 1
56+
}
57+
build main
58+
git fetch origin "$FIX_REF" || exit 1
59+
git diff --stat "$MAIN_SHA" FETCH_HEAD -- crates/recording
60+
git checkout -f FETCH_HEAD -- crates/recording || exit 1
61+
build fix
62+
63+
- name: Measure
64+
run: |
65+
set +e
66+
export LD_LIBRARY_PATH="$GITHUB_WORKSPACE/target/native-deps/lib:$GITHUB_WORKSPACE/target/release${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
67+
out="$RUNNER_TEMP/camera-fanout-out"
68+
mkdir -p "$out"
69+
nproc; lscpu | grep -E "Model name|^CPU\(s\)" || true
70+
status=0
71+
for round in 1 2 3 4 5; do
72+
for res in "1280 720" "1920 1080"; do
73+
for senders in 2 3; do
74+
for variant in main fix; do
75+
"$RUNNER_TEMP/bin-$variant/camera-fanout-bench" cpu $res $senders 600 $variant "$out" | tee -a "$out/results.txt" || status=1
76+
done
77+
done
78+
done
79+
done
80+
for res in "1280 720" "1920 1080"; do
81+
for variant in main fix; do
82+
"$RUNNER_TEMP/bin-$variant/camera-fanout-bench" verify $res 4 240 $variant "$out" | tee -a "$out/verify.txt" || status=1
83+
done
84+
done
85+
python3 -I "$RUNNER_TEMP/camera-fanout-ab/aggregate.py" "$out/results.txt" | tee -a "$GITHUB_STEP_SUMMARY"
86+
exit $status
87+
88+
- name: Compare outputs
89+
run: |
90+
set +e
91+
sudo apt-get install -y --no-install-recommends ffmpeg > /dev/null
92+
out="$RUNNER_TEMP/camera-fanout-out"
93+
status=0
94+
{
95+
echo "### Output identity"
96+
echo
97+
for res in 1280x720 1920x1080; do
98+
if cmp -s "$out/main-$res-digests.txt" "$out/fix-$res-digests.txt"; then
99+
echo "- $res consumer digests (raw, encoder input, BGRA preview): identical ($(wc -l < "$out/fix-$res-digests.txt") frame digests)"
100+
else
101+
echo "- $res consumer digests: DIFFER"; status=1
102+
fi
103+
ffmpeg -v error -i "$out/main-$res.mp4" -f framemd5 - > "$out/main-$res.framemd5"
104+
ffmpeg -v error -i "$out/fix-$res.mp4" -f framemd5 - > "$out/fix-$res.framemd5"
105+
frames=$(grep -vc '^#' "$out/fix-$res.framemd5")
106+
if cmp -s "$out/main-$res.framemd5" "$out/fix-$res.framemd5" && [ "$frames" -gt 0 ]; then
107+
echo "- $res decoded recording frames (framemd5): identical ($frames frames)"
108+
else
109+
echo "- $res decoded recording frames: DIFFER"; status=1
110+
fi
111+
echo "- $res mp4 md5: main $(md5sum < "$out/main-$res.mp4" | cut -c1-12), fix $(md5sum < "$out/fix-$res.mp4" | cut -c1-12)"
112+
done
113+
echo
114+
echo '```'
115+
cat "$out/verify.txt"
116+
echo '```'
117+
} | tee -a "$GITHUB_STEP_SUMMARY"
118+
exit $status
119+
120+
- uses: actions/upload-artifact@v4
121+
if: always()
122+
with:
123+
name: camera-fanout-ab
124+
path: ${{ runner.temp }}/camera-fanout-out

‎scripts/camera-fanout-ab/aggregate.py‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
import statistics
2+
import sys
3+
from collections import defaultdict
4+
5+
rows = defaultdict(list)
6+
for line in open(sys.argv[1]):
7+
if not line.startswith("RESULT "):
8+
continue
9+
fields = dict(part.split("=", 1) for part in line.split()[1:])
10+
rows[(fields["res"], int(fields["senders"]), fields["variant"])].append(fields)
11+
12+
metrics = [
13+
("fanout_p50_us", "fan-out p50 us"),
14+
("fanout_p95_us", "fan-out p95 us"),
15+
("cpu_us_per_frame", "CPU us/frame"),
16+
("minflt_per_frame", "page faults/frame"),
17+
("copied_mb_per_frame", "MB copied/frame"),
18+
("maxrss_mb", "max RSS MB"),
19+
]
20+
21+
print("### Camera fan-out A/B (median of rounds)")
22+
print()
23+
print("| res | senders | metric | main | fix | change |")
24+
print("|---|---|---|---|---|---|")
25+
for res, senders in sorted({(k[0], k[1]) for k in rows}, key=lambda k: (int(k[0].split("x")[0]), k[1])):
26+
main = rows.get((res, senders, "main"), [])
27+
fix = rows.get((res, senders, "fix"), [])
28+
if not main or not fix:
29+
continue
30+
for key, label in metrics:
31+
a = statistics.median(float(r[key]) for r in main)
32+
b = statistics.median(float(r[key]) for r in fix)
33+
change = f"{(b - a) / a * 100:+.0f}%" if a else "n/a"
34+
print(f"| {res} | {senders} | {label} | {a:.2f} | {b:.2f} | {change} |")
35+
rounds = len(fix)
36+
copied = sum(int(r["copied_receipts"]) for r in fix)
37+
shared = sum(int(r["shared_receipts"]) for r in fix)
38+
print(f"| {res} | {senders} | fix receipts copied/shared ({rounds} rounds) | | {copied}/{shared} | |")

0 commit comments

Comments
 (0)