Skip to content

Commit 5b67cc3

Browse files
richiemcilroyrichiemcilroy
authored
Merge branch 'main' into fix/caption-unicode
2 parents 7733322 + ada2be5 commit 5b67cc3

73 files changed

Lines changed: 5699 additions & 662 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/baseline-diff.yml‎

Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
name: Baseline diff
2+
3+
# Report artifacts expire after 90 days and the baseline branch never
4+
# changes, so the baseline is re-run weekly.
5+
6+
on:
7+
workflow_run:
8+
workflows: ["Engine verify", "Verify recording (real capture)"]
9+
types: [completed]
10+
branches: [main]
11+
schedule:
12+
- cron: "17 2 * * 1"
13+
workflow_dispatch:
14+
inputs:
15+
main-run:
16+
description: "Run id on main to compare (Engine verify or real capture)"
17+
required: true
18+
baseline-run:
19+
description: "Baseline run id (default: the latest of the same workflow on scratch/prod-baseline)"
20+
default: ""
21+
22+
permissions:
23+
contents: read
24+
25+
concurrency:
26+
group: baseline-diff-${{ github.event.workflow_run.id || inputs.main-run || 'refresh' }}
27+
cancel-in-progress: false
28+
29+
env:
30+
BASELINE_BRANCH: scratch/prod-baseline
31+
32+
jobs:
33+
refresh:
34+
if: github.event_name == 'schedule'
35+
runs-on: blacksmith-2vcpu-ubuntu-2404
36+
permissions:
37+
actions: write
38+
steps:
39+
- name: Re-run the baseline
40+
env:
41+
GH_TOKEN: ${{ github.token }}
42+
run: |
43+
gh workflow run engine-verify.yml --repo "$GITHUB_REPOSITORY" --ref "$BASELINE_BRANCH" --raw-field tier=full
44+
gh workflow run verify-recording.yml --repo "$GITHUB_REPOSITORY" --ref "$BASELINE_BRANCH" \
45+
--raw-field tier=full --raw-field repeat=3 --raw-field runners=both
46+
47+
diff:
48+
permissions:
49+
contents: read
50+
actions: read
51+
if: >-
52+
github.event_name == 'workflow_dispatch' ||
53+
(github.event_name == 'workflow_run' && github.event.workflow_run.event == 'schedule')
54+
runs-on: blacksmith-2vcpu-ubuntu-2404
55+
steps:
56+
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
57+
with:
58+
persist-credentials: false
59+
60+
- name: Runs
61+
id: runs
62+
env:
63+
GH_TOKEN: ${{ github.token }}
64+
MAIN_RUN: ${{ github.event.workflow_run.id || inputs.main-run }}
65+
BASELINE_INPUT: ${{ inputs.baseline-run }}
66+
run: |
67+
set -euo pipefail
68+
workflow=$(gh run view "$MAIN_RUN" --repo "$GITHUB_REPOSITORY" --json workflowDatabaseId --jq .workflowDatabaseId)
69+
baseline="$BASELINE_INPUT"
70+
if [[ -z "$baseline" ]]; then
71+
# Baseline runs fail (0.6.0 has known bugs), but a run with a
72+
# cancelled or timed-out job stopped before reporting every case.
73+
for candidate in $(gh run list --repo "$GITHUB_REPOSITORY" --workflow "$workflow" --branch "$BASELINE_BRANCH" \
74+
--limit 20 --json databaseId,conclusion \
75+
--jq '.[] | select(.conclusion == "success" or .conclusion == "failure") | .databaseId'); do
76+
stopped=$(gh run view "$candidate" --repo "$GITHUB_REPOSITORY" --json conclusion,jobs \
77+
--jq '[.conclusion, .jobs[].conclusion] | map(select(. == "cancelled" or . == "timed_out")) | length')
78+
if [[ "$stopped" == 0 ]]; then
79+
baseline="$candidate"
80+
break
81+
fi
82+
done
83+
fi
84+
if [[ -z "$baseline" ]]; then
85+
echo "::error::no completed baseline run of workflow $workflow on $BASELINE_BRANCH"
86+
exit 1
87+
fi
88+
echo "main=$MAIN_RUN" >> "$GITHUB_OUTPUT"
89+
echo "baseline=$baseline" >> "$GITHUB_OUTPUT"
90+
echo "Comparing main run $MAIN_RUN with baseline run $baseline" | tee -a "$GITHUB_STEP_SUMMARY"
91+
92+
- name: Download reports
93+
env:
94+
GH_TOKEN: ${{ github.token }}
95+
MAIN_RUN: ${{ steps.runs.outputs.main }}
96+
BASELINE_RUN: ${{ steps.runs.outputs.baseline }}
97+
run: |
98+
set -euo pipefail
99+
for side in main baseline; do
100+
run=$([[ $side == main ]] && echo "$MAIN_RUN" || echo "$BASELINE_RUN")
101+
dest="$RUNNER_TEMP/reports/$side"
102+
mkdir -p "$dest"
103+
# Real capture also uploads its reports alone; runs from before
104+
# that only have the artifact with the recordings.
105+
names=$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$run/artifacts" --paginate --jq '.artifacts[].name')
106+
if grep -q '^verify-report-' <<< "$names"; then
107+
pattern='verify-report-*'
108+
elif grep -q '^verify-recording-report-' <<< "$names"; then
109+
pattern='verify-recording-report-*'
110+
elif grep -q '^verify-recording-' <<< "$names"; then
111+
pattern='verify-recording-*'
112+
else
113+
echo "::error::run $run has no verification reports"
114+
exit 1
115+
fi
116+
gh run download "$run" --repo "$GITHUB_REPOSITORY" --dir "$dest" --pattern "$pattern"
117+
find "$dest" -type d -name verify-scratch -prune -exec rm -r {} +
118+
echo "$side: $(find "$dest" -name '*.jsonl' | wc -l) report files"
119+
done
120+
121+
- name: Baseline covers main's runners
122+
shell: bash
123+
run: |
124+
set -euo pipefail
125+
# Lanes as diff.py names them: shard numbers and tiers dropped.
126+
lanes() {
127+
local dir
128+
for dir in "$1"/verify-*/; do
129+
if [[ -d "$dir" ]]; then basename "$dir"; fi
130+
done | sed -E 's/^verify-recording-report-/verify-recording-/; s/-(fast|full)(-[0-9]+)?$//' | sort -u
131+
}
132+
main_lanes=$(lanes "$RUNNER_TEMP/reports/main")
133+
baseline_lanes=$(lanes "$RUNNER_TEMP/reports/baseline")
134+
if [[ -z "$main_lanes" ]]; then
135+
echo "::error::main's run has no report directories"
136+
exit 1
137+
fi
138+
missing=$(comm -23 <(echo "$main_lanes") <(echo "$baseline_lanes"))
139+
if [[ -n "$missing" ]]; then
140+
echo "::error::the baseline run has no reports for: $(echo $missing)"
141+
exit 1
142+
fi
143+
144+
- name: Diff
145+
run: |
146+
python3 scripts/verify/diff.py \
147+
"$RUNNER_TEMP/reports/baseline" "$RUNNER_TEMP/reports/main" \
148+
--na scripts/verify/baseline-na.tsv \
149+
--title "cap-v0.6.0 baseline vs main" \
150+
--fail-on-worse > "$RUNNER_TEMP/baseline-diff.md"
151+
status=$?
152+
cat "$RUNNER_TEMP/baseline-diff.md" >> "$GITHUB_STEP_SUMMARY"
153+
exit "$status"
154+
shell: bash {0}
155+
156+
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
157+
if: always()
158+
with:
159+
name: baseline-diff
160+
path: ${{ runner.temp }}/baseline-diff.md
161+
if-no-files-found: ignore

‎.github/workflows/engine-verify.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,7 @@ jobs:
8888
run: |
8989
python3 scripts/verify/plan.py test > /dev/null
9090
python3 scripts/verify/test_diff.py
91+
python3 scripts/verify/test_perf_build.py
9192
if grep -rn --include='expectations*.toml' '^reason = "TODO' crates; then
9293
echo "::error::blessed expectations still have TODO reasons: describe each failure (and link its fix) before merging"
9394
exit 1

‎.github/workflows/perf-budget.yml‎

Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
name: Perf budget
2+
3+
# Head and base are built and timed on one machine, interleaved, because
4+
# Blacksmith VMs have no hardware instruction counters and machine state
5+
# drifts between jobs. "Perf budget" is the job to require.
6+
7+
on:
8+
pull_request:
9+
types: [opened, synchronize, reopened, ready_for_review]
10+
workflow_dispatch:
11+
inputs:
12+
base:
13+
description: "Ref to compare against (default: the merge base with main)"
14+
default: ""
15+
16+
concurrency:
17+
group: perf-budget-${{ github.event.pull_request.number || github.ref }}
18+
cancel-in-progress: true
19+
20+
permissions:
21+
contents: read
22+
23+
jobs:
24+
scope:
25+
runs-on: blacksmith-2vcpu-ubuntu-2404
26+
outputs:
27+
run: ${{ steps.scope.outputs.run }}
28+
steps:
29+
- uses: actions/checkout@v4
30+
with:
31+
persist-credentials: false
32+
- name: Scope
33+
id: scope
34+
env:
35+
GH_TOKEN: ${{ github.token }}
36+
EVENT: ${{ github.event_name }}
37+
PR: ${{ github.event.pull_request.number }}
38+
DRAFT: ${{ github.event.pull_request.draft }}
39+
SAME_REPO: ${{ github.event.pull_request.head.repo.full_name == github.repository }}
40+
RUSTUP_TOOLCHAIN: stable
41+
run: |
42+
set -euo pipefail
43+
if [[ "$EVENT" == workflow_dispatch ]]; then
44+
echo "run=true" >> "$GITHUB_OUTPUT"; exit 0
45+
fi
46+
if [[ "$DRAFT" == true ]]; then
47+
echo "Draft: the perf budget runs once the pull request is ready." | tee -a "$GITHUB_STEP_SUMMARY"
48+
echo "run=false" >> "$GITHUB_OUTPUT"; exit 0
49+
fi
50+
gh api "repos/$GITHUB_REPOSITORY/pulls/$PR/files" --paginate --jq '.[].filename' > "$RUNNER_TEMP/changed.txt"
51+
python3 scripts/verify/plan.py risk --files "$RUNNER_TEMP/changed.txt" --output "$RUNNER_TEMP/risk.txt"
52+
# A change to the budget itself is measured only from this
53+
# repository's branches, so a fork cannot queue the macOS job by
54+
# editing these files.
55+
own_files=false
56+
if [[ "$SAME_REPO" == true ]] && grep -qE '^(\.github/workflows/perf-budget\.yml|scripts/verify/perf-(ab|build)\.py)$' "$RUNNER_TEMP/changed.txt"; then
57+
own_files=true
58+
fi
59+
if grep -qx 'escalate=true' "$RUNNER_TEMP/risk.txt" || grep -qx 'engine=true' "$RUNNER_TEMP/risk.txt" || [[ "$own_files" == true ]]; then
60+
echo "run=true" >> "$GITHUB_OUTPUT"
61+
else
62+
echo "No engine or high-risk code changed: nothing to time." | tee -a "$GITHUB_STEP_SUMMARY"
63+
echo "run=false" >> "$GITHUB_OUTPUT"
64+
fi
65+
66+
measure:
67+
needs: scope
68+
if: needs.scope.outputs.run == 'true'
69+
runs-on: blacksmith-6vcpu-macos-26
70+
timeout-minutes: 120
71+
steps:
72+
- uses: actions/checkout@v4
73+
with:
74+
persist-credentials: false
75+
fetch-depth: 0
76+
77+
- name: Setup
78+
uses: ./.github/actions/engine-verify-setup
79+
with:
80+
target: aarch64-apple-darwin
81+
tier: fast
82+
83+
- name: Perf build cache
84+
uses: actions/cache@v4
85+
with:
86+
path: target/perf-verify/
87+
key: perf-verify-v1-${{ hashFiles('Cargo.lock', 'rust-toolchain.toml') }}-${{ github.run_id }}
88+
restore-keys: |
89+
perf-verify-v1-${{ hashFiles('Cargo.lock', 'rust-toolchain.toml') }}-
90+
perf-verify-v1-
91+
92+
- name: Base worktree
93+
env:
94+
BASE_INPUT: ${{ inputs.base }}
95+
PR_BASE: ${{ github.event.pull_request.base.sha }}
96+
run: |
97+
set -euo pipefail
98+
base="${BASE_INPUT:-}"
99+
if [[ -z "$base" ]]; then
100+
base="$(git merge-base HEAD "${PR_BASE:-origin/main}")"
101+
fi
102+
echo "Comparing $(git rev-parse --short HEAD) with base $(git rev-parse --short "$base")" | tee -a "$GITHUB_STEP_SUMMARY"
103+
git worktree add --detach "$RUNNER_TEMP/base" "$base"
104+
# The base builds against the head's native dependencies: the same
105+
# target dir and the FFMPEG_DIR that scripts/setup.js writes to the
106+
# (git-ignored) .cargo/config.toml.
107+
ln -s "$GITHUB_WORKSPACE/target" "$RUNNER_TEMP/base/target"
108+
mkdir -p "$RUNNER_TEMP/base/.cargo"
109+
cp .cargo/config.toml "$RUNNER_TEMP/base/.cargo/config.toml"
110+
111+
- name: Build head
112+
run: python3 scripts/verify/perf-build.py . "$RUNNER_TEMP/perf/head"
113+
114+
- name: Build base
115+
run: python3 scripts/verify/perf-build.py "$RUNNER_TEMP/base" "$RUNNER_TEMP/perf/base"
116+
117+
- name: Measure
118+
run: |
119+
python3 scripts/verify/perf-ab.py \
120+
--base "$RUNNER_TEMP/perf/base/binaries.json" \
121+
--head "$RUNNER_TEMP/perf/head/binaries.json" \
122+
--fixtures "$RUNNER_TEMP/perf/fixtures" \
123+
--out "$RUNNER_TEMP/perf/runs" \
124+
--summary "$GITHUB_STEP_SUMMARY"
125+
126+
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
127+
if: always()
128+
with:
129+
name: perf-budget
130+
path: ${{ runner.temp }}/perf/runs/**/perf-budget.md
131+
if-no-files-found: ignore
132+
133+
verdict:
134+
name: Perf budget
135+
needs: [scope, measure]
136+
if: always()
137+
runs-on: blacksmith-2vcpu-ubuntu-2404
138+
steps:
139+
- env:
140+
RUN: ${{ needs.scope.outputs.run }}
141+
SCOPE: ${{ needs.scope.result }}
142+
MEASURE: ${{ needs.measure.result }}
143+
run: |
144+
if [[ "$SCOPE" != success ]]; then echo "scope failed"; exit 1; fi
145+
if [[ "$RUN" != true ]]; then echo "not measured"; exit 0; fi
146+
[[ "$MEASURE" == success ]]

0 commit comments

Comments
 (0)