Measure bench baselines on the same runner #116
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Bench | |
| # Criterion regression gate for the in-process dispatch hot path. | |
| # | |
| # - push to main: runs the gated bench groups as an informational smoke run. | |
| # - pull_request: measures the PR merge base as the `main` baseline, then | |
| # measures the PR head in the same job on the same runner. The job FAILS | |
| # when any bench regresses by more than 10% mean change AND the 95% | |
| # confidence interval lower bound exceeds +5%. | |
| # - Cargo.toml remains a trigger so dependency/profile changes are covered, | |
| # but a PR whose only relevant edits are version assignments is skipped. | |
| # - Cargo build outputs are reused between the two measurements within the job; | |
| # target/criterion is never restored from a cross-run cache. | |
| # | |
| # Gated groups are the stable per-request paths (wire_path, | |
| # headers_path, request_headers_path, resolve_path, dispatch_path). The | |
| # streaming and contended groups are noisier (spawn_blocking / scheduler | |
| # timing) and the router_path setup micro-bench is low-signal, so those | |
| # are validated locally instead — see PERF_REPORT.md. | |
| # | |
| # This TIMING gate fires only at a loose ±10% (residual same-runner noise), so it | |
| # catches BIG regressions. Small, deterministic ALLOCATION regressions are | |
| # caught noise-free by the `alloc_budget` integration test (a counting | |
| # global allocator asserting exact per-dispatch allocation budgets) in the | |
| # normal `cargo test` job — the two gates are complementary. | |
| on: | |
| push: | |
| branches: | |
| - main | |
| paths: | |
| - 'crates/**' | |
| - 'Cargo.toml' | |
| - 'Cargo.lock' | |
| - '.github/workflows/bench.yml' | |
| pull_request: | |
| paths: | |
| - 'crates/**' | |
| - 'Cargo.toml' | |
| - 'Cargo.lock' | |
| - '.github/workflows/bench.yml' | |
| concurrency: | |
| group: bench-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| BENCH_FILTER: 'wire_path|headers_path|request_headers_path|resolve_path|dispatch_path' | |
| jobs: | |
| bench: | |
| name: Criterion regression gate | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v7 | |
| with: | |
| fetch-depth: 0 | |
| - uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| - name: Determine PR benchmark scope | |
| if: github.event_name == 'pull_request' | |
| id: scope | |
| shell: bash | |
| env: | |
| BASE_SHA: ${{ github.event.pull_request.base.sha }} | |
| HEAD_SHA: ${{ github.event.pull_request.head.sha }} | |
| run: | | |
| merge_base=$(git merge-base "$BASE_SHA" "$HEAD_SHA") | |
| echo "merge_base=$merge_base" >> "$GITHUB_OUTPUT" | |
| should_run=false | |
| while IFS= read -r path; do | |
| case "$path" in | |
| crates/*|Cargo.lock|.github/workflows/bench.yml) | |
| should_run=true | |
| ;; | |
| esac | |
| done < <(git diff --name-only "$merge_base" "$HEAD_SHA") | |
| # Compare parsed TOML after removing only changepacks-managed release | |
| # versions. External dependency versions, profiles, features, and any | |
| # unparseable change fail safe by keeping the gate enabled. | |
| if ! python3 - "$merge_base" "$HEAD_SHA" <<'PY' | |
| import copy | |
| import subprocess | |
| import sys | |
| import tomllib | |
| def cargo_toml_at(revision): | |
| contents = subprocess.check_output( | |
| ["git", "show", f"{revision}:Cargo.toml"], | |
| text=True, | |
| encoding="utf-8", | |
| ) | |
| return tomllib.loads(contents) | |
| def without_release_versions(document): | |
| normalized = copy.deepcopy(document) | |
| workspace = normalized.get("workspace") | |
| if not isinstance(workspace, dict): | |
| return normalized | |
| package = workspace.get("package") | |
| if isinstance(package, dict): | |
| package.pop("version", None) | |
| dependencies = workspace.get("dependencies") | |
| if isinstance(dependencies, dict): | |
| for name, spec in dependencies.items(): | |
| if not isinstance(spec, dict): | |
| continue | |
| if spec.get("path") == f"crates/{name}": | |
| spec.pop("version", None) | |
| return normalized | |
| base = without_release_versions(cargo_toml_at(sys.argv[1])) | |
| head = without_release_versions(cargo_toml_at(sys.argv[2])) | |
| version_only = base == head | |
| print(f"cargo_version_only={str(version_only).lower()}") | |
| raise SystemExit(0 if version_only else 1) | |
| PY | |
| then | |
| echo "::notice::Could not prove Cargo.toml changes are version-only; keeping the performance gate enabled." | |
| should_run=true | |
| fi | |
| echo "should_run=$should_run" >> "$GITHUB_OUTPUT" | |
| - name: Skip version-only PR | |
| if: github.event_name == 'pull_request' && steps.scope.outputs.should_run != 'true' | |
| run: echo "::notice::The only benchmark-relevant edits are Cargo.toml version assignments; skipping the performance gate." | |
| - name: Run benchmark smoke test on main | |
| if: github.event_name == 'push' | |
| run: cargo bench -p vespera_inprocess --bench dispatch -- "${BENCH_FILTER}" | |
| - name: Measure merge-base baseline | |
| if: github.event_name == 'pull_request' && steps.scope.outputs.should_run == 'true' | |
| env: | |
| MERGE_BASE: ${{ steps.scope.outputs.merge_base }} | |
| run: | | |
| git checkout --detach "$MERGE_BASE" | |
| rm -rf target/criterion | |
| cargo bench -p vespera_inprocess --bench dispatch -- \ | |
| --save-baseline main "${BENCH_FILTER}" | |
| - name: Compare PR head against merge-base baseline | |
| if: github.event_name == 'pull_request' && steps.scope.outputs.should_run == 'true' | |
| env: | |
| HEAD_SHA: ${{ github.event.pull_request.head.sha }} | |
| run: | | |
| git checkout --detach "$HEAD_SHA" | |
| cargo bench -p vespera_inprocess --bench dispatch -- \ | |
| --baseline main "${BENCH_FILTER}" | |
| - name: Enforce regression gate | |
| if: github.event_name == 'pull_request' && steps.scope.outputs.should_run == 'true' | |
| run: | | |
| shopt -s nullglob | |
| fail=0 | |
| found=0 | |
| while IFS= read -r f; do | |
| found=1 | |
| mean=$(jq -r '.mean.point_estimate' "$f") | |
| lower=$(jq -r '.mean.confidence_interval.lower_bound' "$f") | |
| bench=$(dirname "$(dirname "$f")") | |
| bench=${bench#target/criterion/} | |
| printf '%s: mean %+.2f%% (CI lower %+.2f%%)\n' \ | |
| "$bench" "$(awk -v v="$mean" 'BEGIN{print v*100}')" \ | |
| "$(awk -v v="$lower" 'BEGIN{print v*100}')" | |
| if awk -v m="$mean" -v l="$lower" 'BEGIN{exit !(m > 0.10 && l > 0.05)}'; then | |
| echo "::error::Performance regression: ${bench} mean change exceeds +10% with CI lower bound > +5%" | |
| fail=1 | |
| fi | |
| done < <(find target/criterion -path '*/change/estimates.json') | |
| if [ "$found" -eq 0 ]; then | |
| echo "::notice::No change estimates found (first run against this baseline?) — nothing to gate." | |
| fi | |
| exit $fail |