Skip to content

Commit d394f61

Browse files
richiemcilroyrichiemcilroy
authored
Merge branch 'main' into fix/cap-muxer-survives-recorder-crash
2 parents 109aa5e + fa92b85 commit d394f61

176 files changed

Lines changed: 47465 additions & 1447 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.config/hakari.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ platforms = [
3535
# unified tree into every `cargo build -p cap-muxer`. Any drift it causes is
3636
# bounded by its own tiny graph.
3737
[traversal-excludes]
38-
workspace-members = ["cap-muxer", "cap-muxer-protocol"]
38+
workspace-members = ["cap-muxer", "cap-muxer-protocol", "cap-verify-expect"]
3939

4040
# hakari unifies dev-dependencies as well, so left alone it pins tauri with its
4141
# `test` feature and tokio `test-util` into every release binary and pulls the

‎.config/nextest.toml‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
# cargo-nextest settings. The engine verification suite (scripts/verify/run.sh)
2+
# runs with the `default` profile locally and `ci` in engine-verify.yml.
3+
4+
[profile.default]
5+
retries = 0
6+
fail-fast = false
7+
slow-timeout = { period = "60s" }
8+
9+
# The decoder model fuzzers run 3 to 4 worker threads each.
10+
[[profile.default.overrides]]
11+
filter = 'binary_id(cap-rendering) and test(/^decoder::model::/)'
12+
threads-required = 3
13+
14+
[profile.ci]
15+
# No retries: a flaky case is a bug. run.sh reruns failures once afterwards
16+
# only to label them flaky in the job summary; the job still fails.
17+
retries = 0
18+
fail-fast = false
19+
failure-output = "immediate-final"
20+
success-output = "never"
21+
status-level = "pass"
22+
final-status-level = "slow"
23+
slow-timeout = { period = "60s", terminate-after = 15 }
24+
25+
[profile.ci.junit]
26+
path = "junit.xml"
27+
store-success-output = false
28+
store-failure-output = true
29+
30+
[[profile.ci.overrides]]
31+
filter = 'binary_id(cap-rendering) and test(/^decoder::model::/)'
32+
threads-required = 3
33+
34+
35+
# The FULL-tier seek storms (3000 seeks per decoder) are the slowest tests:
36+
# on the macos-15 runner one storm fixture takes 240 to 570 s per decoder,
37+
# up to about 900 s for both decoders on one shard. Twice that, so a real
38+
# hang still fails.
39+
[[profile.ci.overrides]]
40+
filter = 'binary_id(cap-verify::seek) and test(/^storm_/)'
41+
slow-timeout = { period = "60s", terminate-after = 30 }

‎.github/actionlint.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
self-hosted-runner:
2+
labels:
3+
- blacksmith-8vcpu-ubuntu-2404
4+
- blacksmith-6vcpu-macos-15
5+
- blacksmith-6vcpu-macos-26
6+
- blacksmith-8vcpu-windows-2025

‎.github/actions/engine-verify-setup/action.yml‎

Lines changed: 168 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,168 @@
1+
name: Engine verify setup
2+
description: Toolchain, cargo-nextest, Rust cache, native deps and fixture cache for the engine verification suite
3+
inputs:
4+
target:
5+
description: Rust target triple
6+
required: true
7+
tier:
8+
description: fast or full (fixture cache key)
9+
required: true
10+
rust-cache:
11+
description: Restore the Rust build cache (build jobs only)
12+
default: "true"
13+
outputs:
14+
build-cache-key:
15+
description: Key to save the build cache under
16+
value: ${{ steps.build-key.outputs.key }}
17+
build-cache-hit:
18+
description: Whether the build cache was restored exactly
19+
value: ${{ steps.build-cache.outputs.cache-hit }}
20+
fixtures-hit:
21+
description: Whether the fixture cache was restored exactly
22+
value: ${{ steps.fixtures.outputs.cache-hit }}
23+
fixtures-key:
24+
description: Fixture cache key
25+
value: ${{ steps.fixtures.outputs.cache-primary-key }}
26+
runs:
27+
using: composite
28+
steps:
29+
# Only what the suite's dependency closure links against (no webkit/gtk),
30+
# plus lavapipe for software Vulkan rendering.
31+
- name: Linux packages
32+
if: runner.os == 'Linux'
33+
shell: bash
34+
run: |
35+
sudo apt-get update -qq
36+
sudo apt-get install -y -qq --no-install-recommends \
37+
libasound2-dev libasound2-plugins libpipewire-0.3-dev libspa-0.2-dev libclang-dev \
38+
libx11-dev libxrandr-dev libxcb1-dev libxcb-randr0-dev libxcb-shm0-dev \
39+
libxcb-xfixes0-dev libxfixes-dev libwayland-dev libxkbcommon-dev \
40+
libdbus-1-dev libudev-dev libva-dev mesa-vulkan-drivers libvulkan1 >/dev/null
41+
42+
- name: Rust toolchain
43+
if: inputs.rust-cache == 'true'
44+
uses: dtolnay/rust-toolchain@1.88.0
45+
with:
46+
targets: ${{ inputs.target }}
47+
48+
- name: cargo-nextest
49+
uses: taiki-e/install-action@v2
50+
with:
51+
tool: cargo-nextest@0.9.148
52+
53+
# One cache entry per OS per day (saved by main, nightly and the CI dev
54+
# branches), restored by PRs. Unlike swatinem/rust-cache it includes the
55+
# workspace crates and is refreshed daily even when Cargo.lock is
56+
# unchanged, so a PR only rebuilds what changed since the last save. The
57+
# chain restarts every ISO week so stale artifacts cannot pile up.
58+
- name: Build cache key
59+
id: build-key
60+
if: inputs.rust-cache == 'true'
61+
shell: bash
62+
run: |
63+
# runner.environment separates GitHub-hosted from self-hosted
64+
# (Blacksmith) builds: their paths and toolchains differ, so one's
65+
# target directory would only force a rebuild on the other.
66+
# A new week starts a new chain, seeded from last week's newest cache
67+
# of the same provider so the first build of the week stays warm.
68+
py="$(command -v python3 || command -v python)"
69+
week="$("$py" -c 'import datetime as d; y, w, _ = d.datetime.now(d.timezone.utc).isocalendar(); print(f"{y}-W{w:02d}")')"
70+
last_week="$("$py" -c 'import datetime as d; y, w, _ = (d.datetime.now(d.timezone.utc) - d.timedelta(days=7)).isocalendar(); print(f"{y}-W{w:02d}")')"
71+
provider="verify-build-v3-${{ inputs.target }}-${{ runner.environment }}"
72+
base="$provider-$week"
73+
echo "base=$base" >> "$GITHUB_OUTPUT"
74+
echo "previous=$provider-$last_week" >> "$GITHUB_OUTPUT"
75+
echo "key=$base-${{ hashFiles('Cargo.lock', 'Cargo.toml', 'rust-toolchain.toml') }}-$(date -u +%Y%m%d)" >> "$GITHUB_OUTPUT"
76+
77+
- name: Restore build cache
78+
id: build-cache
79+
if: inputs.rust-cache == 'true'
80+
uses: actions/cache/restore@v4
81+
with:
82+
path: |
83+
~/.cargo/registry/index/
84+
~/.cargo/registry/cache/
85+
~/.cargo/git/db/
86+
target/ci-verify/
87+
key: ${{ steps.build-key.outputs.key }}
88+
restore-keys: |
89+
${{ steps.build-key.outputs.base }}-${{ hashFiles('Cargo.lock', 'Cargo.toml', 'rust-toolchain.toml') }}-
90+
${{ steps.build-key.outputs.base }}-
91+
${{ steps.build-key.outputs.previous }}-
92+
93+
# Unchanged workspace crates are reused from the cache instead of being
94+
# rebuilt because the checkout touched every file (scripts/verify/mtime.py).
95+
- name: Reuse unchanged workspace crates
96+
if: inputs.rust-cache == 'true'
97+
shell: bash
98+
run: |
99+
"$(command -v python3 || command -v python)" scripts/verify/mtime.py apply target/ci-verify/verify-src-manifest.txt
100+
101+
- name: Native dependencies
102+
shell: bash
103+
env:
104+
RUST_TARGET_TRIPLE: ${{ inputs.target }}
105+
run: node scripts/setup.js
106+
107+
- name: FFmpeg libraries on the loader path (Linux)
108+
if: runner.os == 'Linux'
109+
shell: bash
110+
run: echo "LD_LIBRARY_PATH=${{ github.workspace }}/target/native-deps/lib" >> "$GITHUB_ENV"
111+
112+
# The app ships DXC next to its executable and cap-rendering looks for it
113+
# next to the test binary or in its profile dir; setup.js only fills
114+
# target/debug and target/release. Without it wgpu falls back to FXC.
115+
- name: DXC for the ci-verify profile (Windows)
116+
if: runner.os == 'Windows'
117+
shell: bash
118+
run: |
119+
mkdir -p target/ci-verify
120+
cp target/native-deps/dxc/dxcompiler.dll target/native-deps/dxc/dxil.dll target/ci-verify/
121+
122+
# Debug builds of wgpu (the ci-verify profile keeps debug assertions)
123+
# enable the D3D12 debug layer whenever d3d12SDKLayers.dll exists. When
124+
# that DLL comes from an unserviced Graphics Tools feature older than the
125+
# D3D12 runtime (blacksmith-8vcpu-windows-2025: layers 10.0.26100.1591,
126+
# runtime 10.0.26100.33296), D3D12CreateDevice fails with E_INVALIDARG on
127+
# every adapter, WARP included, and every case fails "No GPU adapter
128+
# found". Hiding the mismatched layer makes D3D12GetDebugInterface report
129+
# DXGI_ERROR_SDK_COMPONENT_MISSING, which wgpu treats as "no debug layer".
130+
- name: Hide a D3D12 debug layer that does not match the runtime (Windows)
131+
if: runner.os == 'Windows'
132+
shell: pwsh
133+
run: |
134+
$sys = Join-Path $env:WINDIR 'System32'
135+
$layers = Join-Path $sys 'd3d12SDKLayers.dll'
136+
$core = Join-Path $sys 'D3D12Core.dll'
137+
if (-not (Test-Path $layers)) { Write-Host 'no D3D12 debug layer installed'; exit 0 }
138+
$lv = (Get-Item $layers).VersionInfo
139+
$cv = (Get-Item $core).VersionInfo
140+
$lver = "$($lv.FileMajorPart).$($lv.FileMinorPart).$($lv.FileBuildPart).$($lv.FilePrivatePart)"
141+
$cver = "$($cv.FileMajorPart).$($cv.FileMinorPart).$($cv.FileBuildPart).$($cv.FilePrivatePart)"
142+
Write-Host "D3D12 debug layer $lver, runtime $cver"
143+
if ($lver -eq $cver) { exit 0 }
144+
takeown /f $layers | Out-Null
145+
icacls $layers /grant "*S-1-5-32-544:F" | Out-Null
146+
Rename-Item $layers 'd3d12SDKLayers.dll.mismatched'
147+
Write-Host "hid the mismatched debug layer"
148+
149+
- name: FFmpeg DLLs on PATH (Windows)
150+
if: runner.os == 'Windows'
151+
shell: pwsh
152+
run: Add-Content -Path $env:GITHUB_PATH -Value "${{ github.workspace }}\target\ffmpeg\bin"
153+
154+
- name: Fixture location
155+
shell: bash
156+
run: echo "CAP_VERIFY_FIXTURE_DIR=${{ runner.temp }}/verify-fixtures" >> "$GITHUB_ENV"
157+
158+
# Fixtures are generated by code (including the real-pipeline Studio and
159+
# Instant recordings, ~1 min of real-time capture); the key is every
160+
# crate's source and build script, the vendored crates (wgpu-hal, tao),
161+
# the workspace manifest, lockfile and toolchain, plus the native deps. Exact key only: a fixture made
162+
# by an older generator is never reused.
163+
- name: Restore fixtures
164+
id: fixtures
165+
uses: actions/cache/restore@v4
166+
with:
167+
path: ${{ runner.temp }}/verify-fixtures
168+
key: verify-fixtures-v1-${{ runner.os }}-${{ runner.environment }}-${{ inputs.tier }}-${{ hashFiles('crates/**/src/**', 'crates/**/build.rs', 'crates/**/Cargo.toml', 'Cargo.toml', 'Cargo.lock', 'rust-toolchain.toml', 'vendor/**', 'scripts/setup.js') }}

‎.github/workflows/baseline-diff.yml‎

Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
name: Baseline diff
2+
3+
# Report artifacts expire after 90 days and the baseline branch never
4+
# changes, so the baseline is re-run weekly.
5+
6+
on:
7+
workflow_run:
8+
workflows: ["Engine verify", "Verify recording (real capture)"]
9+
types: [completed]
10+
branches: [main]
11+
schedule:
12+
- cron: "17 2 * * 1"
13+
workflow_dispatch:
14+
inputs:
15+
main-run:
16+
description: "Run id on main to compare (Engine verify or real capture)"
17+
required: true
18+
baseline-run:
19+
description: "Baseline run id (default: the latest of the same workflow on scratch/prod-baseline)"
20+
default: ""
21+
22+
permissions:
23+
contents: read
24+
25+
concurrency:
26+
group: baseline-diff-${{ github.event.workflow_run.id || inputs.main-run || 'refresh' }}
27+
cancel-in-progress: false
28+
29+
env:
30+
BASELINE_BRANCH: scratch/prod-baseline
31+
32+
jobs:
33+
refresh:
34+
if: github.event_name == 'schedule'
35+
runs-on: blacksmith-2vcpu-ubuntu-2404
36+
permissions:
37+
actions: write
38+
steps:
39+
- name: Re-run the baseline
40+
env:
41+
GH_TOKEN: ${{ github.token }}
42+
run: |
43+
gh workflow run engine-verify.yml --repo "$GITHUB_REPOSITORY" --ref "$BASELINE_BRANCH" --raw-field tier=full
44+
gh workflow run verify-recording.yml --repo "$GITHUB_REPOSITORY" --ref "$BASELINE_BRANCH" \
45+
--raw-field tier=full --raw-field repeat=3 --raw-field runners=both
46+
47+
diff:
48+
permissions:
49+
contents: read
50+
actions: read
51+
if: >-
52+
github.event_name == 'workflow_dispatch' ||
53+
(github.event_name == 'workflow_run' && github.event.workflow_run.event == 'schedule')
54+
runs-on: blacksmith-2vcpu-ubuntu-2404
55+
steps:
56+
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
57+
with:
58+
persist-credentials: false
59+
60+
- name: Runs
61+
id: runs
62+
env:
63+
GH_TOKEN: ${{ github.token }}
64+
MAIN_RUN: ${{ github.event.workflow_run.id || inputs.main-run }}
65+
BASELINE_INPUT: ${{ inputs.baseline-run }}
66+
run: |
67+
set -euo pipefail
68+
workflow=$(gh run view "$MAIN_RUN" --repo "$GITHUB_REPOSITORY" --json workflowDatabaseId --jq .workflowDatabaseId)
69+
baseline="$BASELINE_INPUT"
70+
if [[ -z "$baseline" ]]; then
71+
# Baseline runs fail (0.6.0 has known bugs), but a run with a
72+
# cancelled or timed-out job stopped before reporting every case.
73+
for candidate in $(gh run list --repo "$GITHUB_REPOSITORY" --workflow "$workflow" --branch "$BASELINE_BRANCH" \
74+
--limit 20 --json databaseId,conclusion \
75+
--jq '.[] | select(.conclusion == "success" or .conclusion == "failure") | .databaseId'); do
76+
stopped=$(gh run view "$candidate" --repo "$GITHUB_REPOSITORY" --json conclusion,jobs \
77+
--jq '[.conclusion, .jobs[].conclusion] | map(select(. == "cancelled" or . == "timed_out")) | length')
78+
if [[ "$stopped" == 0 ]]; then
79+
baseline="$candidate"
80+
break
81+
fi
82+
done
83+
fi
84+
if [[ -z "$baseline" ]]; then
85+
echo "::error::no completed baseline run of workflow $workflow on $BASELINE_BRANCH"
86+
exit 1
87+
fi
88+
echo "main=$MAIN_RUN" >> "$GITHUB_OUTPUT"
89+
echo "baseline=$baseline" >> "$GITHUB_OUTPUT"
90+
echo "Comparing main run $MAIN_RUN with baseline run $baseline" | tee -a "$GITHUB_STEP_SUMMARY"
91+
92+
- name: Download reports
93+
env:
94+
GH_TOKEN: ${{ github.token }}
95+
MAIN_RUN: ${{ steps.runs.outputs.main }}
96+
BASELINE_RUN: ${{ steps.runs.outputs.baseline }}
97+
run: |
98+
set -euo pipefail
99+
for side in main baseline; do
100+
run=$([[ $side == main ]] && echo "$MAIN_RUN" || echo "$BASELINE_RUN")
101+
dest="$RUNNER_TEMP/reports/$side"
102+
mkdir -p "$dest"
103+
# Real capture also uploads its reports alone; runs from before
104+
# that only have the artifact with the recordings.
105+
names=$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$run/artifacts" --paginate --jq '.artifacts[].name')
106+
if grep -q '^verify-report-' <<< "$names"; then
107+
pattern='verify-report-*'
108+
elif grep -q '^verify-recording-report-' <<< "$names"; then
109+
pattern='verify-recording-report-*'
110+
elif grep -q '^verify-recording-' <<< "$names"; then
111+
pattern='verify-recording-*'
112+
else
113+
echo "::error::run $run has no verification reports"
114+
exit 1
115+
fi
116+
gh run download "$run" --repo "$GITHUB_REPOSITORY" --dir "$dest" --pattern "$pattern"
117+
find "$dest" -type d -name verify-scratch -prune -exec rm -r {} +
118+
echo "$side: $(find "$dest" -name '*.jsonl' | wc -l) report files"
119+
done
120+
121+
- name: Baseline covers main's runners
122+
shell: bash
123+
run: |
124+
set -euo pipefail
125+
# Lanes as diff.py names them: shard numbers and tiers dropped.
126+
lanes() {
127+
local dir
128+
for dir in "$1"/verify-*/; do
129+
if [[ -d "$dir" ]]; then basename "$dir"; fi
130+
done | sed -E 's/^verify-recording-report-/verify-recording-/; s/-(fast|full)(-[0-9]+)?$//' | sort -u
131+
}
132+
main_lanes=$(lanes "$RUNNER_TEMP/reports/main")
133+
baseline_lanes=$(lanes "$RUNNER_TEMP/reports/baseline")
134+
if [[ -z "$main_lanes" ]]; then
135+
echo "::error::main's run has no report directories"
136+
exit 1
137+
fi
138+
missing=$(comm -23 <(echo "$main_lanes") <(echo "$baseline_lanes"))
139+
if [[ -n "$missing" ]]; then
140+
echo "::error::the baseline run has no reports for: $(echo $missing)"
141+
exit 1
142+
fi
143+
144+
- name: Diff
145+
run: |
146+
python3 scripts/verify/diff.py \
147+
"$RUNNER_TEMP/reports/baseline" "$RUNNER_TEMP/reports/main" \
148+
--na scripts/verify/baseline-na.tsv \
149+
--title "cap-v0.6.0 baseline vs main" \
150+
--fail-on-worse > "$RUNNER_TEMP/baseline-diff.md"
151+
status=$?
152+
cat "$RUNNER_TEMP/baseline-diff.md" >> "$GITHUB_STEP_SUMMARY"
153+
exit "$status"
154+
shell: bash {0}
155+
156+
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
157+
if: always()
158+
with:
159+
name: baseline-diff
160+
path: ${{ runner.temp }}/baseline-diff.md
161+
if-no-files-found: ignore

0 commit comments

Comments
 (0)