Skip to content

Commit 00567db

Browse files
committed
Publish CI test and performance JSON to metrics site
1 parent 7a219c6 commit 00567db

25 files changed

Lines changed: 1919 additions & 32 deletions
Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
#!/usr/bin/env bash
2+
# REQ-WEB-002: collect data from authenticated, completed main workflow runs.
3+
set -euo pipefail
4+
destination="${1:?Expected a new output directory}"
5+
repository="${GITHUB_REPOSITORY:?Expected the source repository}"
6+
[[ "$repository" == managedcode/Synapse ]]
7+
[[ ! -e "$destination" ]]
8+
mkdir -p "$destination"
9+
printf '{"verification":null,"performance":null}\n' > "$destination/runs.json"
10+
11+
collect_artifact() {
12+
local stream="$1" name="$2" filename="$3" metadata="$4" started="$5"
13+
local selected artifact_id digest archive target
14+
selected="$(jq -c --arg name "$name" --arg started "$started" \
15+
'[.artifacts[] | select(.name == $name and .expired == false and .created_at >= $started)]
16+
| sort_by(.created_at, .id) | last // empty' "$metadata")"
17+
[[ -n "$selected" ]] || return 0
18+
artifact_id="$(jq -r '.id' <<< "$selected")"
19+
digest="$(jq -r '.digest // empty' <<< "$selected")"
20+
[[ "$artifact_id" =~ ^[0-9]+$ ]]
21+
# Refuse legacy unhashed, oversized, or otherwise unverifiable artifacts.
22+
[[ "$digest" =~ ^sha256:[0-9a-f]{64}$ ]] || return 0
23+
[[ "$(jq -r '.size_in_bytes <= 16777216' <<< "$selected")" == true ]] || return 0
24+
archive="$destination/$artifact_id.zip"
25+
if ! gh api "repos/$repository/actions/artifacts/${artifact_id}/zip" > "$archive"; then
26+
rm -f "$archive"
27+
return 0
28+
fi
29+
[[ "$(shasum -a 256 "$archive" | awk '{ print $1 }')" == "${digest#sha256:}" ]]
30+
# Read one expected root member; never extract archive paths or executable files.
31+
if [[ "$(unzip -Z -1 "$archive" | awk -v file="$filename" '$0 == file { count++ } END { print count+0 }')" != 1 ]]; then
32+
rm "$archive"
33+
return 0
34+
fi
35+
target="$destination/$stream/$name"
36+
mkdir -p "$target"
37+
unzip -p "$archive" "$filename" | head -c 16777217 > "$target/$filename"
38+
[[ "$(wc -c < "$target/$filename")" -le 16777216 ]]
39+
rm "$archive"
40+
jq --arg stream "$stream" --argjson artifact "$selected" \
41+
'.[$stream].artifact_manifest += [$artifact | {id,name,digest,created_at}]' \
42+
"$destination/runs.json" > "$destination/runs.next.json"
43+
mv "$destination/runs.next.json" "$destination/runs.json"
44+
}
45+
46+
for stream in verification performance; do
47+
workflow=verify
48+
[[ "$stream" != performance ]] || workflow=performance
49+
gh api "repos/$repository/actions/workflows/$workflow.yml/runs?branch=main&status=completed&per_page=100" \
50+
> "$destination/$stream-runs.json"
51+
run="$(jq -c --arg repository "$repository" \
52+
'[.workflow_runs[] | select(.head_branch == "main" and .head_repository.full_name == $repository
53+
and (.event == "push" or .event == "workflow_dispatch"))]
54+
| sort_by(.run_number, .run_attempt) | last // empty' "$destination/$stream-runs.json")"
55+
[[ -n "$run" ]] || continue
56+
run_id="$(jq -r '.id' <<< "$run")"
57+
[[ "$run_id" =~ ^[0-9]+$ ]]
58+
jq --arg stream "$stream" --argjson run "$run" \
59+
'.[$stream] = ($run | {id,run_number,run_attempt,head_sha,html_url,conclusion,updated_at,
60+
run_started_at,head_branch,event,repository:.head_repository.full_name,artifact_manifest:[]})' \
61+
"$destination/runs.json" > "$destination/runs.next.json"
62+
mv "$destination/runs.next.json" "$destination/runs.json"
63+
metadata="$destination/$stream-artifacts.json"
64+
gh api --paginate --slurp "repos/$repository/actions/runs/$run_id/artifacts?per_page=100" \
65+
| jq '{artifacts:[.[] | .artifacts[]]}' > "$metadata"
66+
started="$(jq -r '.run_started_at' <<< "$run")"
67+
if [[ "$stream" == verification ]]; then
68+
for runner in osx-arm64 linux-x64 win-x64; do
69+
collect_artifact "$stream" "test-results-$runner" test-results.json "$metadata" "$started"
70+
done
71+
else
72+
collect_artifact "$stream" performance-summary performance-results.json "$metadata" "$started"
73+
fi
74+
# A rerun may begin during collection; never attribute its artifacts to an old attempt.
75+
[[ "$(gh api "repos/$repository/actions/runs/$run_id" | jq -r \
76+
--argjson attempt "$(jq -r '.run_attempt' <<< "$run")" --arg started "$started" \
77+
--arg sha "$(jq -r '.head_sha' <<< "$run")" \
78+
'.status == "completed" and .run_attempt == $attempt and .run_started_at == $started and .head_sha == $sha')" == true ]]
79+
done

‎.github/workflows/pages.yml‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,18 @@ on:
77
- docs/images/**
88
- site/**
99
- .github/workflows/pages.yml
10+
- .github/scripts/collect-site-results.sh
11+
- experiments/Synapse.ReferenceBenchmarks/Features/Website/**
12+
- experiments/Synapse.ReferenceBenchmarks/HostedArtifactReportCommand.cs
13+
- experiments/Synapse.ReferenceBenchmarks/Program.cs
1014
workflow_dispatch:
15+
workflow_run:
16+
workflows: [verify, performance]
17+
types: [completed]
1118

1219
permissions:
1320
contents: read
21+
actions: read
1422

1523
concurrency:
1624
group: github-pages
@@ -19,10 +27,40 @@ concurrency:
1927
jobs:
2028
build:
2129
name: Build public site
30+
if: >-
31+
github.ref == 'refs/heads/main' &&
32+
(github.event_name != 'workflow_run' ||
33+
(github.event.workflow_run.head_branch == 'main' &&
34+
github.event.workflow_run.head_repository.full_name == github.repository &&
35+
(github.event.workflow_run.event == 'push' ||
36+
github.event.workflow_run.event == 'workflow_dispatch')))
2237
runs-on: ubuntu-24.04
38+
timeout-minutes: 15
2339
steps:
2440
- name: Checkout
2541
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
42+
with:
43+
ref: main
44+
persist-credentials: false
45+
46+
- name: Install pinned .NET SDK
47+
uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0
48+
with:
49+
global-json-file: global.json
50+
51+
- name: Install pinned Rust toolchain required by the reporter build
52+
run: rustup toolchain install 1.98.1 --profile minimal
53+
54+
- name: Collect latest completed main CI result artifacts
55+
shell: bash
56+
env:
57+
GH_TOKEN: ${{ github.token }}
58+
run: bash .github/scripts/collect-site-results.sh "${RUNNER_TEMP}/site-results"
59+
60+
- name: Build C# result validator
61+
run: |
62+
dotnet restore experiments/Synapse.ReferenceBenchmarks/Synapse.ReferenceBenchmarks.csproj --locked-mode
63+
dotnet build experiments/Synapse.ReferenceBenchmarks/Synapse.ReferenceBenchmarks.csproj --configuration Release --no-restore
2664
2765
- name: Configure GitHub Pages
2866
uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # v5.0.0
@@ -32,8 +70,12 @@ jobs:
3270
run: |
3371
mkdir -p _site/images
3472
cp site/index.html site/styles.css site/site.js site/favicon.svg _site/
73+
cp site/metrics.js _site/
3574
cp docs/images/*.svg _site/images/
3675
touch _site/.nojekyll
76+
dotnet experiments/Synapse.ReferenceBenchmarks/bin/Release/net10.0/Synapse.ReferenceBenchmarks.dll site-data \
77+
--artifacts "${RUNNER_TEMP}/site-results" --runs "${RUNNER_TEMP}/site-results/runs.json" \
78+
--output _site/data/latest.json
3779
3880
- name: Upload GitHub Pages artifact
3981
uses: actions/upload-pages-artifact@7b1f4a764d45c48632c6b24a0339c27f5614fb0b # v4.0.0

‎.github/workflows/performance.yml‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -376,13 +376,16 @@ jobs:
376376
--artifacts "${RUNNER_TEMP}/performance-artifacts" \
377377
--model-set benchmarks/model-sets/foundry-local-families.json \
378378
--output "${RUNNER_TEMP}/performance-summary.md" \
379+
--json "${RUNNER_TEMP}/performance-results.json" \
379380
--summary "${GITHUB_STEP_SUMMARY}"
380381
381382
- name: Preserve combined results table
382383
if: always()
383384
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
384385
with:
385386
name: performance-summary
386-
path: ${{ runner.temp }}/performance-summary.md
387+
path: |
388+
${{ runner.temp }}/performance-summary.md
389+
${{ runner.temp }}/performance-results.json
387390
if-no-files-found: warn
388-
retention-days: 30
391+
retention-days: 90

‎.github/workflows/verify.yml‎

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,36 @@ jobs:
126126
run: echo "SYNAPSE_LLAMACPP_EXECUTABLE=${GITHUB_WORKSPACE}/_external/llama.cpp/${{ matrix.llama_binary_directory }}/llama-completion${{ matrix.executable_suffix }}" >> "${GITHUB_ENV}"
127127

128128
- name: Test .NET
129-
run: dotnet test Synapse.slnx --configuration Release --no-build
129+
run: >-
130+
dotnet test Synapse.slnx --configuration Release --no-build --
131+
--report-trx --results-directory "${{ runner.temp }}/test-results"
132+
--minimum-expected-tests 1 --zero-tests-policy strict
133+
134+
- name: Export real .NET test results as JSON
135+
if: always()
136+
shell: bash
137+
run: |
138+
dotnet experiments/Synapse.ReferenceBenchmarks/bin/Release/net10.0/Synapse.ReferenceBenchmarks.dll test-report \
139+
--results "${RUNNER_TEMP}/test-results" --runner "${{ matrix.runtime_identifier }}" \
140+
--output "${RUNNER_TEMP}/test-results.json"
141+
142+
- name: Publish .NET test JSON
143+
if: always()
144+
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
145+
with:
146+
name: test-results-${{ matrix.runtime_identifier }}
147+
path: ${{ runner.temp }}/test-results.json
148+
if-no-files-found: warn
149+
retention-days: 90
150+
151+
- name: Preserve raw .NET TRX evidence
152+
if: always()
153+
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
154+
with:
155+
name: test-trx-${{ matrix.runtime_identifier }}
156+
path: ${{ runner.temp }}/test-results/*.trx
157+
if-no-files-found: warn
158+
retention-days: 90
130159

131160
- name: Check Rust formatting
132161
run: cargo fmt --manifest-path native/Cargo.toml --all --check

‎docs/ADR/ADR-010-github-pages-website-delivery.md‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,3 +31,34 @@ The repository does not maintain a generated `gh-pages` branch. A successful
3131
local source check or push is insufficient evidence: publication is complete
3232
only after the remote Pages workflow succeeds and the canonical HTTPS URL
3333
serves that revision.
34+
35+
## 2026-10-03: CI result publication (`TASK-WEB-002`)
36+
37+
Completed trusted `main` runs of `verify` and `performance` also trigger the
38+
Pages workflow. Pull-request and fork runs cannot publish. The build checks
39+
out trusted `main` source, reads Actions artifacts as data only, and selects
40+
the latest completed push/manual run of each workflow. Both streams retain
41+
their own run ID, attempt, source SHA, conclusion and time; they may describe
42+
different commits. Serialized deployments select current runs when building,
43+
so an older queued completion cannot deliberately republish its old snapshot.
44+
45+
Each verification runner exports a version-1 JSON projection of real .NET TRX
46+
results. Zero tests, absent reports, malformed reports and failing tests never
47+
become a passing result. Rust remains part of the workflow conclusion; .NET
48+
counts explicitly name their scope. Performance aggregation exports a
49+
version-1 JSON projection of the validated raw benchmark artifacts, including
50+
missing/invalid evidence, nullable metrics and measured sample counts. Native
51+
eval and reported decode rates, and request and fresh-process wall times,
52+
retain distinct scopes. These diagnostics confer no statistical winner verdict.
53+
54+
A C# command assembles `data/latest.json` with independent run provenance,
55+
validated test reports and performance rows. The public site displays these
56+
CI results separately from dated local Apple M2 Pro measurements. JSON and
57+
upstream run links are public; raw artifacts remain on GitHub Actions. Missing
58+
or expired artifacts are shown explicitly. No generated Git commits, runtime
59+
backend changes, Node.js or Python tooling are introduced.
60+
61+
The collector uses authenticated API artifact IDs from the selected run and
62+
attempt, expected names only, bounded downloads and a confined extraction
63+
directory. Artifact paths, schemas and runner identities are checked before
64+
publication. Only deployment receives Pages/OIDC write privileges.
Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
# Website metrics publication evidence — 2026-10-03
2+
3+
Scope: `REQ-WEB-002`, `AC-WEB-002-1..4`, `TASK-WEB-002`,
4+
`TEST-WEB-002-1..4`, ADR-010. Use the current checkout; preserve pre-existing
5+
model/runtime/docs changes. No worktree, stash, dependency or model change.
6+
7+
## Behavior-first verification
8+
9+
- Benchmark JSON export: initial 4/4 failed because `aggregate --json` was
10+
unsupported (exit 2); final 4/4 passed. Existing hosted-report 2/2 passed.
11+
- TRX export: initial five real-child tests failed because `test-report` was
12+
unsupported; the final seven test cases passed. They execute real TUnit
13+
fixtures and mutate real XML for invalid input cases; no service double.
14+
- Pipeline/site data: the initial three cases had two failures (no trigger
15+
and no site-data command). Four additional source/report regressions exposed
16+
invalid-data exception handling and scalar metadata acceptance before the
17+
repairs. The final positive case executes real TUnit -> TRX -> test JSON and
18+
recorded raw benchmark -> numerical JSON -> public site JSON.
19+
- Final Website suite: 19/19 passed, zero failed/skipped, with the real TRX
20+
retained under ignored `artifacts/website-publication/final-tests/`.
21+
22+
## Commands and outcomes
23+
24+
| Command | Outcome |
25+
|---|---|
26+
| `dotnet restore Synapse.slnx --locked-mode` | Exit 0, all six projects |
27+
| `dotnet build Synapse.slnx --configuration Release --no-restore --disable-build-servers -m:1` | Exit 0, zero warnings/errors |
28+
| `dotnet format Synapse.slnx --no-restore --verify-no-changes` | Exit 0 after the final constructor style fix |
29+
| `dotnet artifacts/website-publication/coverage-snapshot/ManagedCode.Synapse.IntegrationTests.dll --treenode-filter '/*/ManagedCode.Synapse.IntegrationTests.Features.Website/*/*' --minimum-expected-tests 19 --coverage --coverage-output-format cobertura --coverage-output /Users/ksemenenko/Developer/Synapse/artifacts/website-publication/coverage.cobertura.xml --results-directory /Users/ksemenenko/Developer/Synapse/artifacts/website-publication/final-tests --report-trx` | Exit 0, 19 passed, 17.105s |
30+
| `dotnet experiments/Synapse.ReferenceBenchmarks/bin/Release/net10.0/Synapse.ReferenceBenchmarks.dll test-report --results artifacts/website-publication/final-tests --runner osx-arm64 --output artifacts/website-publication/final-test-results.json` | Exit 0; actual 19 passed, zero failed/skipped |
31+
| `cargo test --manifest-path native/Cargo.toml --locked` | Exit 0; 32 actual tests, empty unit/doc targets do not count as required suites |
32+
| `cargo fmt --manifest-path native/Cargo.toml --all --check` | Exit 0 |
33+
| `cargo clippy --manifest-path native/Cargo.toml --workspace --all-targets -- -D warnings` | Exit 0 |
34+
| `dotnet list Synapse.slnx package --vulnerable --include-transitive` | Exit 0; no reported vulnerable packages in six projects |
35+
| `actionlint .github/workflows/pages.yml .github/workflows/verify.yml .github/workflows/performance.yml` | Exit 0 |
36+
| `bash -n .github/scripts/collect-site-results.sh` | Exit 0 |
37+
| `git diff --check`, `git diff --cached --check` | Exit 0 |
38+
39+
Rust commands use `PATH=/opt/homebrew/opt/rustup/bin:$PATH`. Sandbox feed/IPC
40+
access was granted for required checks. Coverage runs from a copied binary
41+
directory to preserve the normal build. C# XML analysis deduplicates filename
42+
and line number: Website source coverage is 340/367 (92.64%). Overall benchmark
43+
assembly coverage is 9.3% for this focused filter; this is not a full-solution
44+
or changed-line coverage claim.
45+
46+
## Artifact and source qualification
47+
48+
The real collector verified existing GitHub artifact SHA-256 values and
49+
retained independent metadata for verification run `36547564346` (failure)
50+
and performance run `36547564428` (success), both source
51+
`7a219c6cb6fdb851962a3f23a2f141abde20679d`. These older runs have no new JSON,
52+
so the produced `current-latest-validated.json` truthfully contains absent
53+
test artifacts and a null benchmark report. Historical Markdown-only
54+
artifacts are not relabeled as numerical evidence.
55+
56+
An independent source export consists of `git archive HEAD` plus exactly the
57+
staged publication patch; it contains none of the pre-existing model changes.
58+
Its first build caught an incorrectly positioned staged switch insertion;
59+
the staged-only patch coordinates were corrected before publication. The
60+
corrected export passed locked restore, Release solution build (5.80s,
61+
zero warnings/errors), 19/19 Website tests (12.268s) and 2/2 hosted-report
62+
compatibility tests (263ms), all exit 0. Remote publication follows separately.
63+
64+
Architecture review: no new runtime engines/dependencies, generated branches,
65+
or Node/Python tooling. Files/types stay within repository size limits. Only
66+
trusted completed `main` push/manual workflow runs may publish; collector
67+
checks attempt freshness, artifact digests, bounded content and expected root
68+
members without extracting untrusted paths. The site uses textContent for
69+
report data and validates GitHub links and numeric/schema fields.
70+
71+
## Current limitations
72+
73+
The C# preview serves source-identical HTML/JS with HTTP 200 and a truthful
74+
404 before metrics data exists. CUA exposes no enabled browser/app surface, so
75+
desktop/mobile rendering and browser JavaScript execution are unverified.
76+
The broader runtime/statistical benchmark plans remain open; these tests do
77+
not establish a performance winner. `TASK-WEB-002` stays `in_progress` until
78+
remote/live evidence and the outstanding browser review are recorded.
79+
80+
Automatic approval review initially rejected the scoped commit/push to `main`
81+
because the dependency-repair authorization does not cover this feature. No
82+
commit or push occurred during that attempt. On 2026-10-03 the user explicitly
83+
authorized pushing these prepared changes to `main`; publication proceeds
84+
under that authorization.

‎docs/Development/tasks.json‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,15 @@
127127
"dependsOn": [],
128128
"tests": ["TEST-WEB-001-1"]
129129
},
130+
{
131+
"id": "TASK-WEB-002",
132+
"requirement": "REQ-WEB-002",
133+
"slice": "Website",
134+
"stage": "D0",
135+
"status": "in_progress",
136+
"dependsOn": ["TASK-WEB-001"],
137+
"tests": ["TEST-WEB-002-1", "TEST-WEB-002-2", "TEST-WEB-002-3", "TEST-WEB-002-4"]
138+
},
130139
{
131140
"id": "TASK-CTX-001",
132141
"requirement": "REQ-CTX-001",

‎docs/Features/Website.md‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,3 +15,24 @@ that content changes.
1515
The files under `site/` own the rendered page; `docs/images/` owns its public
1616
diagrams. The workflow copies those sources without modifying them and keeps
1717
documentation links pointed at their canonical repository paths.
18+
19+
## Automatic CI metrics (`REQ-WEB-002`, ADR-010)
20+
21+
Functional test runners publish `test-results.json` from real .NET TRX files;
22+
benchmark aggregation publishes `performance-results.json` from validated raw
23+
JSON. Both use schema version 1. Missing values are null and missing/invalid
24+
evidence is retained. .NET counts and whole-workflow conclusions are separate.
25+
26+
Pages runs after trusted `main` push/manual workflows complete, downloads the
27+
latest completed verification and performance artifacts by authenticated run
28+
and artifact ID, and publishes `data/latest.json`. The two streams retain
29+
independent commit/run/attempt/time provenance. Failed and partial runs remain
30+
visible. Public CI tables never imply a cross-hardware performance verdict or
31+
replace the dated M2 Pro diagnostics.
32+
33+
| Acceptance criterion | Evidence |
34+
|---|---|
35+
| `AC-WEB-002-1` real nonzero TRX results export immutable JSON; failed, missing, zero and malformed results cannot pass | `TEST-WEB-002-1` `TestResultsPublicationTests` |
36+
| `AC-WEB-002-2` numerical benchmark JSON retains sample counts, nullable metrics, phase scopes and invalid/missing evidence | `TEST-WEB-002-2` `PerformancePublicationTests` |
37+
| `AC-WEB-002-3` trusted completed runs trigger a confined artifact-to-Pages build with independently attributable streams | `TEST-WEB-002-3` `MetricsPipelineTests`, remote Actions evidence |
38+
| `AC-WEB-002-4` the responsive site displays current test counts and filterable scoped benchmark rows, with provenance and honest unavailable states | `TEST-WEB-002-4` rendered desktop/mobile review and public JSON response |

0 commit comments

Comments
 (0)