diff --git a/PUBLICATION_MANIFEST.json b/PUBLICATION_MANIFEST.json index 639f51b..4653741 100644 --- a/PUBLICATION_MANIFEST.json +++ b/PUBLICATION_MANIFEST.json @@ -6,7 +6,7 @@ "Technical-preview source; technical acceptance does not authorize publication.", "Verified-base evidence is scoped to the accepted clean source tree. Changed candidates require an exact-source controlled acceptance record and successful hosted CI.", "Project name, licence split and author/copyright attribution are owner-confirmed; know-how/rights review and public visibility authorization remain separate human decisions.", - "Scope revision 0.3.0: the original prepublication blanket results ban is preserved in Git history. The owner directed the bounded retrospective descriptive table after being informed that independent human validation and the old claims gate remained pending. This explicit exception does not authorize official release or establish independent approval; the owner decision is separately recorded in controlled records outside public Git." + "Scope revision 0.3.0: the original prepublication blanket results ban is preserved in Git history. The owner directed the bounded retrospective descriptive table after being informed that independent human validation and the old claims gate remained pending. This explicit exception does not authorize official release or establish independent approval; the owner decision is separately recorded in controlled records outside public Git. Scope extension: the owner-authorized 21-model, two-series descriptive study and approved aggregates, charts and safe semantic summaries are limited to docs/studies/2026-10-aggregates.json, docs/studies/2026-10-case-churn.svg, docs/studies/2026-10-compliance-diagnostic.csv, docs/studies/2026-10-goal-strict.svg, docs/studies/2026-10-model-flows.csv, docs/studies/2026-10-models.csv, docs/studies/2026-10-observations.md. This does not authorize future results, raw traces or billing disclosure; independent human validation and official-release boundaries remain unchanged." ], "release": { "project": "OperateBench", @@ -144,7 +144,7 @@ { "class": "unvalidated_results", "description": "Unsupported official leaderboards, composite capability scores, universal rankings, production capability claims and general named-model comparisons; unvalidated results outside the explicit descriptive-table exception.", - "reason": "Independent human semantic validation and official-release claims review remain pending. Only docs/MODEL_RESULTS.md is excepted under the bounded owner-directed descriptive scope; this is not a validation PASS." + "reason": "Independent human semantic validation and official-release claims review remain pending. docs/MODEL_RESULTS.md is the primary historical path excepted under the bounded owner-directed descriptive scope; this is not a validation PASS. Scope extension: the owner-authorized 21-model, two-series descriptive study and approved aggregates, charts and safe semantic summaries are limited to docs/studies/2026-10-aggregates.json, docs/studies/2026-10-case-churn.svg, docs/studies/2026-10-compliance-diagnostic.csv, docs/studies/2026-10-goal-strict.svg, docs/studies/2026-10-model-flows.csv, docs/studies/2026-10-models.csv, docs/studies/2026-10-observations.md. This does not authorize future results, raw traces or billing disclosure; independent human validation and official-release boundaries remain unchanged." }, { "class": "source_history", @@ -198,7 +198,7 @@ "value": true, "path": "docs/MODEL_RESULTS.md", "scope": "owner_directed_retrospective_descriptive_only", - "statement": "One English table reports all 19 exact IDs: a 16-model, 704-assignment historical campaign and three separately executed successors, with distinct cohorts and no ranking across cohorts. One trial per model on each of 44 related synthetic cases; assigned and scored denominators are disclosed, with technical exclusions excluded from scored denominators rather than counted as failures. Costs are estimates, not invoices. Raw traces remain private. Independent human semantic validation is pending; the owner decision is separately recorded, not independent approval. No official Lifecycle Track leaderboard, universal ranking or production capability claim is authorized.", + "statement": "One English table reports all 19 exact IDs: a 16-model, 704-assignment historical campaign and three separately executed successors, with distinct cohorts and no ranking across cohorts. One trial per model on each of 44 related synthetic cases; assigned and scored denominators are disclosed, with technical exclusions excluded from scored denominators rather than counted as failures. Costs are estimates, not invoices. Raw traces remain private. Independent human semantic validation is pending; the owner decision is separately recorded, not independent approval. No official Lifecycle Track leaderboard, universal ranking or production capability claim is authorized. Scope extension: the owner-authorized 21-model, two-series descriptive study and approved aggregates, charts and safe semantic summaries are limited to docs/studies/2026-10-aggregates.json, docs/studies/2026-10-case-churn.svg, docs/studies/2026-10-compliance-diagnostic.csv, docs/studies/2026-10-goal-strict.svg, docs/studies/2026-10-model-flows.csv, docs/studies/2026-10-models.csv, docs/studies/2026-10-observations.md. This does not authorize future results, raw traces or billing disclosure; independent human validation and official-release boundaries remain unchanged.", "human_semantic_validation": "pending", "official_release_authorized": false } diff --git a/PUBLIC_RELEASE_CHECKLIST.md b/PUBLIC_RELEASE_CHECKLIST.md index 115a176..12bd789 100644 --- a/PUBLIC_RELEASE_CHECKLIST.md +++ b/PUBLIC_RELEASE_CHECKLIST.md @@ -151,7 +151,7 @@ and confirm none of the following appears in any form: BoundaryBench run is Lifecycle evidence. **Current-scope exception (manifest revision 0.3.0).** The original blanket ban -remains historical in Git. Only the independently labelled, owner-directed +remains historical in Git. The independently labelled, owner-directed retrospective [descriptive table](docs/MODEL_RESULTS.md) is excepted: one English table of 19 exact IDs, comprising the 16-model, 704-assignment historical campaign and three separately executed successors. Cohorts remain distinct, with no @@ -164,6 +164,21 @@ and is not independent approval. This exception does not close G8, G11 or G12, permit official/general claims, or bypass exact-commit review, privacy, secrets, licensing or other official-release blockers. +The bounded exception also covers the owner-authorized 21-model, two-series +[descriptive study](docs/studies/2026-10-observations.md), approved aggregates, +charts and safe semantic summaries, limited to these six companions: + +- `docs/studies/2026-10-aggregates.json` +- `docs/studies/2026-10-case-churn.svg` +- `docs/studies/2026-10-compliance-diagnostic.csv` +- `docs/studies/2026-10-goal-strict.svg` +- `docs/studies/2026-10-model-flows.csv` +- `docs/studies/2026-10-models.csv` + +This extension authorizes no future results, raw traces or billing disclosure. +Independent human semantic validation remains pending; the same G8, G11, G12 and +official-release boundaries above apply. + Positively present: - [ ] Prior art conceded in `docs/RELATED_WORK.md`, with sources. diff --git a/README.md b/README.md index cabae01..6edd5d7 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,8 @@ external events change independently of the agent. > has one Semantic Scenario with three frozen scenario variants of synthetic > Maintenance. Commerce and Property Compliance are separate SDK incubators, > not additional official benchmark evidence. Boundary is a separate track. +> Read the [21-model, two-series October observation study](docs/studies/2026-10-observations.md) for goal versus strict checks, case churn and a separate Compliance correction diagnostic. The historical 19-model table below is preserved. +> > The [model results page](docs/MODEL_RESULTS.md) > reports a descriptive full44 comparison of 19 exact IDs. > No official Lifecycle model results or leaderboard are published. No proven incremental lifecycle diff --git a/docs/MODEL_RESULTS.md b/docs/MODEL_RESULTS.md index acead29..0824bc4 100644 --- a/docs/MODEL_RESULTS.md +++ b/docs/MODEL_RESULTS.md @@ -1,5 +1,7 @@ # Model results — descriptive observations: 44 related synthetic cases per model, one trial per case; separate sources/settings; no independent human review yet; not a general-capability ranking +See the [21-model, two-series October observation study](studies/2026-10-observations.md) for outcome versus strict checks, changed cases and a separate Compliance correction diagnostic. This historical 19-model source table and its scores are preserved. + | Model | Series (distinct runs/settings) | Scored / assigned | Unscored (excluded / aborted) | Goal achieved | All quality checks | Estimated cost (USD, not invoice) | | --- | --- | --- | --- | --- | --- | --- | | `gpt-6-astra` | [Baseline original campaign](https://github.com/DwellyOrg/OperateBench/commit/58b36826894c4306aa3ab) | 44/44 | — | 44/44 (100.0%) | 42/44 (95.5%) | $239 | diff --git a/docs/studies/2026-10-aggregates.json b/docs/studies/2026-10-aggregates.json new file mode 100644 index 0000000..5f0504f --- /dev/null +++ b/docs/studies/2026-10-aggregates.json @@ -0,0 +1,3041 @@ +{ + "format": "operatebench.descriptive-study-2026-10.v1", + "scope": "Owner-run exploratory synthetic multi-flow observations. Not an official Lifecycle model ranking, controlled same-settings replicate or production validation.", + "model_count": 21, + "cases_per_model": 44, + "case_mix": { + "maintenance": 3, + "commerce": 16, + "compliance": 25 + }, + "series_labels": { + "old": "Historical owner-run observations across source cohorts", + "new": "2026-10-24 repeat observations" + }, + "counts": { + "scored_records": 1833, + "pairs": 924, + "common_scored": 910, + "strict_flips": 66, + "goal_flips": 44 + }, + "all_scored_totals": { + "old": { + "goals": 451, + "strict": 250, + "scored": 916 + }, + "new": { + "goals": 443, + "strict": 261, + "scored": 917 + } + }, + "strict_flip_goal_cross_tab": [ + { + "flow": "maintenance", + "old_goal": true, + "new_goal": true, + "count": 3 + }, + { + "flow": "compliance", + "old_goal": true, + "new_goal": true, + "count": 30 + }, + { + "flow": "commerce", + "old_goal": true, + "new_goal": true, + "count": 28 + }, + { + "flow": "compliance", + "old_goal": true, + "new_goal": false, + "count": 1 + }, + { + "flow": "commerce", + "old_goal": true, + "new_goal": false, + "count": 2 + }, + { + "flow": "maintenance", + "old_goal": false, + "new_goal": true, + "count": 1 + }, + { + "flow": "commerce", + "old_goal": false, + "new_goal": true, + "count": 1 + } + ], + "models": [ + { + "model": "Qwen/Qwen3.5-397B-A17B-FP8", + "common_goal_new": 3, + "common_goal_old": 3, + "common_scored": 44, + "common_strict_new": 0, + "common_strict_old": 0, + "goal_gained": 0, + "goal_lost": 0, + "goal_retained": 3, + "goal_stable_fail": 41, + "new_excluded": 0, + "new_goals": 3, + "new_scored": 44, + "new_strict": 0, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 3, + "old_scored": 44, + "old_strict": 0, + "pairs": 44, + "strict_gained": 0, + "strict_lost": 0, + "strict_retained": 0, + "strict_stable_fail": 44, + "latest_scored_decisions": 1352, + "latest_cost_usd": null + }, + { + "model": "Qwen/Qwen3.5-9B", + "common_goal_new": 0, + "common_goal_old": 0, + "common_scored": 44, + "common_strict_new": 0, + "common_strict_old": 0, + "goal_gained": 0, + "goal_lost": 0, + "goal_retained": 0, + "goal_stable_fail": 44, + "new_excluded": 0, + "new_goals": 0, + "new_scored": 44, + "new_strict": 0, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 0, + "old_scored": 44, + "old_strict": 0, + "pairs": 44, + "strict_gained": 0, + "strict_lost": 0, + "strict_retained": 0, + "strict_stable_fail": 44, + "latest_scored_decisions": 1226, + "latest_cost_usd": null + }, + { + "model": "Qwen/Qwen3.6-35B-A3B-FP8", + "common_goal_new": 7, + "common_goal_old": 8, + "common_scored": 44, + "common_strict_new": 0, + "common_strict_old": 0, + "goal_gained": 2, + "goal_lost": 3, + "goal_retained": 5, + "goal_stable_fail": 34, + "new_excluded": 0, + "new_goals": 7, + "new_scored": 44, + "new_strict": 0, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 8, + "old_scored": 44, + "old_strict": 0, + "pairs": 44, + "strict_gained": 0, + "strict_lost": 0, + "strict_retained": 0, + "strict_stable_fail": 44, + "latest_scored_decisions": 1845, + "latest_cost_usd": null + }, + { + "model": "Qwen/Qwen3.8-27B-FP8", + "common_goal_new": 2, + "common_goal_old": 1, + "common_scored": 44, + "common_strict_new": 2, + "common_strict_old": 1, + "goal_gained": 1, + "goal_lost": 0, + "goal_retained": 1, + "goal_stable_fail": 42, + "new_excluded": 0, + "new_goals": 2, + "new_scored": 44, + "new_strict": 2, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 1, + "old_scored": 44, + "old_strict": 1, + "pairs": 44, + "strict_gained": 1, + "strict_lost": 0, + "strict_retained": 1, + "strict_stable_fail": 42, + "latest_scored_decisions": 2608, + "latest_cost_usd": null + }, + { + "model": "claude-fable-5-1", + "common_goal_new": 43, + "common_goal_old": 43, + "common_scored": 43, + "common_strict_new": 34, + "common_strict_old": 35, + "goal_gained": 0, + "goal_lost": 0, + "goal_retained": 43, + "goal_stable_fail": 0, + "new_excluded": 0, + "new_goals": 44, + "new_scored": 44, + "new_strict": 35, + "old_aborted": 0, + "old_excluded": 1, + "old_goals": 43, + "old_scored": 43, + "old_strict": 35, + "pairs": 44, + "strict_gained": 1, + "strict_lost": 2, + "strict_retained": 33, + "strict_stable_fail": 7, + "latest_scored_decisions": 853, + "latest_cost_usd": null + }, + { + "model": "claude-opus-5-5", + "common_goal_new": 44, + "common_goal_old": 43, + "common_scored": 44, + "common_strict_new": 26, + "common_strict_old": 22, + "goal_gained": 1, + "goal_lost": 0, + "goal_retained": 43, + "goal_stable_fail": 0, + "new_excluded": 0, + "new_goals": 44, + "new_scored": 44, + "new_strict": 26, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 43, + "old_scored": 44, + "old_strict": 22, + "pairs": 44, + "strict_gained": 7, + "strict_lost": 3, + "strict_retained": 19, + "strict_stable_fail": 15, + "latest_scored_decisions": 845, + "latest_cost_usd": null + }, + { + "model": "deepseek-ai/DeepSeek-V4-Flash-0731", + "common_goal_new": 3, + "common_goal_old": 2, + "common_scored": 44, + "common_strict_new": 1, + "common_strict_old": 1, + "goal_gained": 1, + "goal_lost": 0, + "goal_retained": 2, + "goal_stable_fail": 41, + "new_excluded": 0, + "new_goals": 3, + "new_scored": 44, + "new_strict": 1, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 2, + "old_scored": 44, + "old_strict": 1, + "pairs": 44, + "strict_gained": 0, + "strict_lost": 0, + "strict_retained": 1, + "strict_stable_fail": 43, + "latest_scored_decisions": 2523, + "latest_cost_usd": null + }, + { + "model": "deepseek-ai/DeepSeek-V4.1-Flash", + "common_goal_new": 7, + "common_goal_old": 9, + "common_scored": 43, + "common_strict_new": 2, + "common_strict_old": 0, + "goal_gained": 0, + "goal_lost": 2, + "goal_retained": 7, + "goal_stable_fail": 34, + "new_excluded": 1, + "new_goals": 7, + "new_scored": 43, + "new_strict": 2, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 9, + "old_scored": 44, + "old_strict": 0, + "pairs": 44, + "strict_gained": 2, + "strict_lost": 0, + "strict_retained": 0, + "strict_stable_fail": 41, + "latest_scored_decisions": 2930, + "latest_cost_usd": null + }, + { + "model": "gemini-3.8-flash", + "common_goal_new": 42, + "common_goal_old": 42, + "common_scored": 42, + "common_strict_new": 19, + "common_strict_old": 20, + "goal_gained": 0, + "goal_lost": 0, + "goal_retained": 42, + "goal_stable_fail": 0, + "new_excluded": 1, + "new_goals": 43, + "new_scored": 43, + "new_strict": 19, + "old_aborted": 0, + "old_excluded": 1, + "old_goals": 43, + "old_scored": 43, + "old_strict": 20, + "pairs": 44, + "strict_gained": 4, + "strict_lost": 5, + "strict_retained": 15, + "strict_stable_fail": 18, + "latest_scored_decisions": 697, + "latest_cost_usd": null + }, + { + "model": "gemini-4-argon", + "common_goal_new": 44, + "common_goal_old": 44, + "common_scored": 44, + "common_strict_new": 42, + "common_strict_old": 42, + "goal_gained": 0, + "goal_lost": 0, + "goal_retained": 44, + "goal_stable_fail": 0, + "new_excluded": 0, + "new_goals": 44, + "new_scored": 44, + "new_strict": 42, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 44, + "old_scored": 44, + "old_strict": 42, + "pairs": 44, + "strict_gained": 1, + "strict_lost": 1, + "strict_retained": 41, + "strict_stable_fail": 1, + "latest_scored_decisions": 757, + "latest_cost_usd": null + }, + { + "model": "gpt-6-astra", + "common_goal_new": 43, + "common_goal_old": 43, + "common_scored": 43, + "common_strict_new": 42, + "common_strict_old": 41, + "goal_gained": 0, + "goal_lost": 0, + "goal_retained": 43, + "goal_stable_fail": 0, + "new_excluded": 1, + "new_goals": 43, + "new_scored": 43, + "new_strict": 42, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 44, + "old_scored": 44, + "old_strict": 42, + "pairs": 44, + "strict_gained": 1, + "strict_lost": 0, + "strict_retained": 41, + "strict_stable_fail": 1, + "latest_scored_decisions": 804, + "latest_cost_usd": null + }, + { + "model": "gpt-6-luna", + "common_goal_new": 9, + "common_goal_old": 9, + "common_scored": 40, + "common_strict_new": 0, + "common_strict_old": 0, + "goal_gained": 1, + "goal_lost": 1, + "goal_retained": 8, + "goal_stable_fail": 30, + "new_excluded": 0, + "new_goals": 9, + "new_scored": 44, + "new_strict": 0, + "old_aborted": 4, + "old_excluded": 0, + "old_goals": 9, + "old_scored": 40, + "old_strict": 0, + "pairs": 44, + "strict_gained": 0, + "strict_lost": 0, + "strict_retained": 0, + "strict_stable_fail": 40, + "latest_scored_decisions": 2432, + "latest_cost_usd": null + }, + { + "model": "gpt-6-sol", + "common_goal_new": 40, + "common_goal_old": 39, + "common_scored": 44, + "common_strict_new": 20, + "common_strict_old": 19, + "goal_gained": 4, + "goal_lost": 3, + "goal_retained": 36, + "goal_stable_fail": 1, + "new_excluded": 0, + "new_goals": 40, + "new_scored": 44, + "new_strict": 20, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 39, + "old_scored": 44, + "old_strict": 19, + "pairs": 44, + "strict_gained": 7, + "strict_lost": 6, + "strict_retained": 13, + "strict_stable_fail": 18, + "latest_scored_decisions": 1187, + "latest_cost_usd": null + }, + { + "model": "gpt-6.1-sol", + "common_goal_new": 44, + "common_goal_old": 44, + "common_scored": 44, + "common_strict_new": 42, + "common_strict_old": 42, + "goal_gained": 0, + "goal_lost": 0, + "goal_retained": 44, + "goal_stable_fail": 0, + "new_excluded": 0, + "new_goals": 44, + "new_scored": 44, + "new_strict": 42, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 44, + "old_scored": 44, + "old_strict": 42, + "pairs": 44, + "strict_gained": 2, + "strict_lost": 2, + "strict_retained": 40, + "strict_stable_fail": 0, + "latest_scored_decisions": 829, + "latest_cost_usd": null + }, + { + "model": "mistral-medium-3-5", + "common_goal_new": 0, + "common_goal_old": 0, + "common_scored": 44, + "common_strict_new": 0, + "common_strict_old": 0, + "goal_gained": 0, + "goal_lost": 0, + "goal_retained": 0, + "goal_stable_fail": 44, + "new_excluded": 0, + "new_goals": 0, + "new_scored": 44, + "new_strict": 0, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 0, + "old_scored": 44, + "old_strict": 0, + "pairs": 44, + "strict_gained": 0, + "strict_lost": 0, + "strict_retained": 0, + "strict_stable_fail": 44, + "latest_scored_decisions": 2403, + "latest_cost_usd": null + }, + { + "model": "muse-spark-1.3", + "common_goal_new": 19, + "common_goal_old": 18, + "common_scored": 44, + "common_strict_new": 5, + "common_strict_old": 4, + "goal_gained": 2, + "goal_lost": 1, + "goal_retained": 17, + "goal_stable_fail": 24, + "new_excluded": 0, + "new_goals": 19, + "new_scored": 44, + "new_strict": 5, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 18, + "old_scored": 44, + "old_strict": 4, + "pairs": 44, + "strict_gained": 4, + "strict_lost": 3, + "strict_retained": 1, + "strict_stable_fail": 36, + "latest_scored_decisions": 1119, + "latest_cost_usd": null + }, + { + "model": "muse-wonder-1.0", + "common_goal_new": 20, + "common_goal_old": 25, + "common_scored": 40, + "common_strict_new": 10, + "common_strict_old": 11, + "goal_gained": 1, + "goal_lost": 6, + "goal_retained": 19, + "goal_stable_fail": 14, + "new_excluded": 4, + "new_goals": 20, + "new_scored": 40, + "new_strict": 10, + "old_aborted": 0, + "old_excluded": 1, + "old_goals": 27, + "old_scored": 43, + "old_strict": 12, + "pairs": 44, + "strict_gained": 2, + "strict_lost": 3, + "strict_retained": 8, + "strict_stable_fail": 27, + "latest_scored_decisions": 901, + "latest_cost_usd": null + }, + { + "model": "nvidia/GLM-5.2-NVFP4-262K", + "common_goal_new": 29, + "common_goal_old": 34, + "common_scored": 44, + "common_strict_new": 6, + "common_strict_old": 6, + "goal_gained": 1, + "goal_lost": 6, + "goal_retained": 28, + "goal_stable_fail": 9, + "new_excluded": 0, + "new_goals": 29, + "new_scored": 44, + "new_strict": 6, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 34, + "old_scored": 44, + "old_strict": 6, + "pairs": 44, + "strict_gained": 2, + "strict_lost": 2, + "strict_retained": 4, + "strict_stable_fail": 36, + "latest_scored_decisions": 1796, + "latest_cost_usd": null + }, + { + "model": "nvidia/Kimi-K2.6-NVFP4", + "common_goal_new": 7, + "common_goal_old": 7, + "common_scored": 43, + "common_strict_new": 0, + "common_strict_old": 0, + "goal_gained": 0, + "goal_lost": 0, + "goal_retained": 7, + "goal_stable_fail": 36, + "new_excluded": 0, + "new_goals": 7, + "new_scored": 44, + "new_strict": 0, + "old_aborted": 0, + "old_excluded": 1, + "old_goals": 7, + "old_scored": 43, + "old_strict": 0, + "pairs": 44, + "strict_gained": 0, + "strict_lost": 0, + "strict_retained": 0, + "strict_stable_fail": 43, + "latest_scored_decisions": 1190, + "latest_cost_usd": null + }, + { + "model": "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4", + "common_goal_new": 0, + "common_goal_old": 0, + "common_scored": 44, + "common_strict_new": 0, + "common_strict_old": 0, + "goal_gained": 0, + "goal_lost": 0, + "goal_retained": 0, + "goal_stable_fail": 44, + "new_excluded": 0, + "new_goals": 0, + "new_scored": 44, + "new_strict": 0, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 0, + "old_scored": 44, + "old_strict": 0, + "pairs": 44, + "strict_gained": 0, + "strict_lost": 0, + "strict_retained": 0, + "strict_stable_fail": 44, + "latest_scored_decisions": 2246, + "latest_cost_usd": null + }, + { + "model": "zai-org/GLM-5.3-Flash", + "common_goal_new": 35, + "common_goal_old": 33, + "common_scored": 44, + "common_strict_new": 9, + "common_strict_old": 4, + "goal_gained": 5, + "goal_lost": 3, + "goal_retained": 30, + "goal_stable_fail": 6, + "new_excluded": 0, + "new_goals": 35, + "new_scored": 44, + "new_strict": 9, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 33, + "old_scored": 44, + "old_strict": 4, + "pairs": 44, + "strict_gained": 5, + "strict_lost": 0, + "strict_retained": 4, + "strict_stable_fail": 35, + "latest_scored_decisions": 1316, + "latest_cost_usd": null + } + ], + "model_flows": [ + { + "model": "Qwen/Qwen3.5-397B-A17B-FP8", + "flow": "commerce", + "common_goal_new": 3, + "common_goal_old": 3, + "common_scored": 16, + "common_strict_new": 0, + "common_strict_old": 0, + "goal_gained": 0, + "goal_lost": 0, + "goal_retained": 3, + "goal_stable_fail": 13, + "new_excluded": 0, + "new_goals": 3, + "new_scored": 16, + "new_strict": 0, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 3, + "old_scored": 16, + "old_strict": 0, + "pairs": 16, + "strict_gained": 0, + "strict_lost": 0, + "strict_retained": 0, + "strict_stable_fail": 16 + }, + { + "model": "Qwen/Qwen3.5-397B-A17B-FP8", + "flow": "compliance", + "common_goal_new": 0, + "common_goal_old": 0, + "common_scored": 25, + "common_strict_new": 0, + "common_strict_old": 0, + "goal_gained": 0, + "goal_lost": 0, + "goal_retained": 0, + "goal_stable_fail": 25, + "new_excluded": 0, + "new_goals": 0, + "new_scored": 25, + "new_strict": 0, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 0, + "old_scored": 25, + "old_strict": 0, + "pairs": 25, + "strict_gained": 0, + "strict_lost": 0, + "strict_retained": 0, + "strict_stable_fail": 25 + }, + { + "model": "Qwen/Qwen3.5-397B-A17B-FP8", + "flow": "maintenance", + "common_goal_new": 0, + "common_goal_old": 0, + "common_scored": 3, + "common_strict_new": 0, + "common_strict_old": 0, + "goal_gained": 0, + "goal_lost": 0, + "goal_retained": 0, + "goal_stable_fail": 3, + "new_excluded": 0, + "new_goals": 0, + "new_scored": 3, + "new_strict": 0, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 0, + "old_scored": 3, + "old_strict": 0, + "pairs": 3, + "strict_gained": 0, + "strict_lost": 0, + "strict_retained": 0, + "strict_stable_fail": 3 + }, + { + "model": "Qwen/Qwen3.5-9B", + "flow": "commerce", + "common_goal_new": 0, + "common_goal_old": 0, + "common_scored": 16, + "common_strict_new": 0, + "common_strict_old": 0, + "goal_gained": 0, + "goal_lost": 0, + "goal_retained": 0, + "goal_stable_fail": 16, + "new_excluded": 0, + "new_goals": 0, + "new_scored": 16, + "new_strict": 0, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 0, + "old_scored": 16, + "old_strict": 0, + "pairs": 16, + "strict_gained": 0, + "strict_lost": 0, + "strict_retained": 0, + "strict_stable_fail": 16 + }, + { + "model": "Qwen/Qwen3.5-9B", + "flow": "compliance", + "common_goal_new": 0, + "common_goal_old": 0, + "common_scored": 25, + "common_strict_new": 0, + "common_strict_old": 0, + "goal_gained": 0, + "goal_lost": 0, + "goal_retained": 0, + "goal_stable_fail": 25, + "new_excluded": 0, + "new_goals": 0, + "new_scored": 25, + "new_strict": 0, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 0, + "old_scored": 25, + "old_strict": 0, + "pairs": 25, + "strict_gained": 0, + "strict_lost": 0, + "strict_retained": 0, + "strict_stable_fail": 25 + }, + { + "model": "Qwen/Qwen3.5-9B", + "flow": "maintenance", + "common_goal_new": 0, + "common_goal_old": 0, + "common_scored": 3, + "common_strict_new": 0, + "common_strict_old": 0, + "goal_gained": 0, + "goal_lost": 0, + "goal_retained": 0, + "goal_stable_fail": 3, + "new_excluded": 0, + "new_goals": 0, + "new_scored": 3, + "new_strict": 0, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 0, + "old_scored": 3, + "old_strict": 0, + "pairs": 3, + "strict_gained": 0, + "strict_lost": 0, + "strict_retained": 0, + "strict_stable_fail": 3 + }, + { + "model": "Qwen/Qwen3.6-35B-A3B-FP8", + "flow": "commerce", + "common_goal_new": 7, + "common_goal_old": 8, + "common_scored": 16, + "common_strict_new": 0, + "common_strict_old": 0, + "goal_gained": 2, + "goal_lost": 3, + "goal_retained": 5, + "goal_stable_fail": 6, + "new_excluded": 0, + "new_goals": 7, + "new_scored": 16, + "new_strict": 0, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 8, + "old_scored": 16, + "old_strict": 0, + "pairs": 16, + "strict_gained": 0, + "strict_lost": 0, + "strict_retained": 0, + "strict_stable_fail": 16 + }, + { + "model": "Qwen/Qwen3.6-35B-A3B-FP8", + "flow": "compliance", + "common_goal_new": 0, + "common_goal_old": 0, + "common_scored": 25, + "common_strict_new": 0, + "common_strict_old": 0, + "goal_gained": 0, + "goal_lost": 0, + "goal_retained": 0, + "goal_stable_fail": 25, + "new_excluded": 0, + "new_goals": 0, + "new_scored": 25, + "new_strict": 0, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 0, + "old_scored": 25, + "old_strict": 0, + "pairs": 25, + "strict_gained": 0, + "strict_lost": 0, + "strict_retained": 0, + "strict_stable_fail": 25 + }, + { + "model": "Qwen/Qwen3.6-35B-A3B-FP8", + "flow": "maintenance", + "common_goal_new": 0, + "common_goal_old": 0, + "common_scored": 3, + "common_strict_new": 0, + "common_strict_old": 0, + "goal_gained": 0, + "goal_lost": 0, + "goal_retained": 0, + "goal_stable_fail": 3, + "new_excluded": 0, + "new_goals": 0, + "new_scored": 3, + "new_strict": 0, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 0, + "old_scored": 3, + "old_strict": 0, + "pairs": 3, + "strict_gained": 0, + "strict_lost": 0, + "strict_retained": 0, + "strict_stable_fail": 3 + }, + { + "model": "Qwen/Qwen3.8-27B-FP8", + "flow": "commerce", + "common_goal_new": 2, + "common_goal_old": 1, + "common_scored": 16, + "common_strict_new": 2, + "common_strict_old": 1, + "goal_gained": 1, + "goal_lost": 0, + "goal_retained": 1, + "goal_stable_fail": 14, + "new_excluded": 0, + "new_goals": 2, + "new_scored": 16, + "new_strict": 2, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 1, + "old_scored": 16, + "old_strict": 1, + "pairs": 16, + "strict_gained": 1, + "strict_lost": 0, + "strict_retained": 1, + "strict_stable_fail": 14 + }, + { + "model": "Qwen/Qwen3.8-27B-FP8", + "flow": "compliance", + "common_goal_new": 0, + "common_goal_old": 0, + "common_scored": 25, + "common_strict_new": 0, + "common_strict_old": 0, + "goal_gained": 0, + "goal_lost": 0, + "goal_retained": 0, + "goal_stable_fail": 25, + "new_excluded": 0, + "new_goals": 0, + "new_scored": 25, + "new_strict": 0, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 0, + "old_scored": 25, + "old_strict": 0, + "pairs": 25, + "strict_gained": 0, + "strict_lost": 0, + "strict_retained": 0, + "strict_stable_fail": 25 + }, + { + "model": "Qwen/Qwen3.8-27B-FP8", + "flow": "maintenance", + "common_goal_new": 0, + "common_goal_old": 0, + "common_scored": 3, + "common_strict_new": 0, + "common_strict_old": 0, + "goal_gained": 0, + "goal_lost": 0, + "goal_retained": 0, + "goal_stable_fail": 3, + "new_excluded": 0, + "new_goals": 0, + "new_scored": 3, + "new_strict": 0, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 0, + "old_scored": 3, + "old_strict": 0, + "pairs": 3, + "strict_gained": 0, + "strict_lost": 0, + "strict_retained": 0, + "strict_stable_fail": 3 + }, + { + "model": "claude-fable-5-1", + "flow": "commerce", + "common_goal_new": 15, + "common_goal_old": 15, + "common_scored": 15, + "common_strict_new": 15, + "common_strict_old": 15, + "goal_gained": 0, + "goal_lost": 0, + "goal_retained": 15, + "goal_stable_fail": 0, + "new_excluded": 0, + "new_goals": 16, + "new_scored": 16, + "new_strict": 16, + "old_aborted": 0, + "old_excluded": 1, + "old_goals": 15, + "old_scored": 15, + "old_strict": 15, + "pairs": 16, + "strict_gained": 0, + "strict_lost": 0, + "strict_retained": 15, + "strict_stable_fail": 0 + }, + { + "model": "claude-fable-5-1", + "flow": "compliance", + "common_goal_new": 25, + "common_goal_old": 25, + "common_scored": 25, + "common_strict_new": 19, + "common_strict_old": 19, + "goal_gained": 0, + "goal_lost": 0, + "goal_retained": 25, + "goal_stable_fail": 0, + "new_excluded": 0, + "new_goals": 25, + "new_scored": 25, + "new_strict": 19, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 25, + "old_scored": 25, + "old_strict": 19, + "pairs": 25, + "strict_gained": 1, + "strict_lost": 1, + "strict_retained": 18, + "strict_stable_fail": 5 + }, + { + "model": "claude-fable-5-1", + "flow": "maintenance", + "common_goal_new": 3, + "common_goal_old": 3, + "common_scored": 3, + "common_strict_new": 0, + "common_strict_old": 1, + "goal_gained": 0, + "goal_lost": 0, + "goal_retained": 3, + "goal_stable_fail": 0, + "new_excluded": 0, + "new_goals": 3, + "new_scored": 3, + "new_strict": 0, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 3, + "old_scored": 3, + "old_strict": 1, + "pairs": 3, + "strict_gained": 0, + "strict_lost": 1, + "strict_retained": 0, + "strict_stable_fail": 2 + }, + { + "model": "claude-opus-5-5", + "flow": "commerce", + "common_goal_new": 16, + "common_goal_old": 16, + "common_scored": 16, + "common_strict_new": 11, + "common_strict_old": 13, + "goal_gained": 0, + "goal_lost": 0, + "goal_retained": 16, + "goal_stable_fail": 0, + "new_excluded": 0, + "new_goals": 16, + "new_scored": 16, + "new_strict": 11, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 16, + "old_scored": 16, + "old_strict": 13, + "pairs": 16, + "strict_gained": 1, + "strict_lost": 3, + "strict_retained": 10, + "strict_stable_fail": 2 + }, + { + "model": "claude-opus-5-5", + "flow": "compliance", + "common_goal_new": 25, + "common_goal_old": 25, + "common_scored": 25, + "common_strict_new": 14, + "common_strict_old": 8, + "goal_gained": 0, + "goal_lost": 0, + "goal_retained": 25, + "goal_stable_fail": 0, + "new_excluded": 0, + "new_goals": 25, + "new_scored": 25, + "new_strict": 14, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 25, + "old_scored": 25, + "old_strict": 8, + "pairs": 25, + "strict_gained": 6, + "strict_lost": 0, + "strict_retained": 8, + "strict_stable_fail": 11 + }, + { + "model": "claude-opus-5-5", + "flow": "maintenance", + "common_goal_new": 3, + "common_goal_old": 2, + "common_scored": 3, + "common_strict_new": 1, + "common_strict_old": 1, + "goal_gained": 1, + "goal_lost": 0, + "goal_retained": 2, + "goal_stable_fail": 0, + "new_excluded": 0, + "new_goals": 3, + "new_scored": 3, + "new_strict": 1, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 2, + "old_scored": 3, + "old_strict": 1, + "pairs": 3, + "strict_gained": 0, + "strict_lost": 0, + "strict_retained": 1, + "strict_stable_fail": 2 + }, + { + "model": "deepseek-ai/DeepSeek-V4-Flash-0731", + "flow": "commerce", + "common_goal_new": 2, + "common_goal_old": 2, + "common_scored": 16, + "common_strict_new": 1, + "common_strict_old": 1, + "goal_gained": 0, + "goal_lost": 0, + "goal_retained": 2, + "goal_stable_fail": 14, + "new_excluded": 0, + "new_goals": 2, + "new_scored": 16, + "new_strict": 1, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 2, + "old_scored": 16, + "old_strict": 1, + "pairs": 16, + "strict_gained": 0, + "strict_lost": 0, + "strict_retained": 1, + "strict_stable_fail": 15 + }, + { + "model": "deepseek-ai/DeepSeek-V4-Flash-0731", + "flow": "compliance", + "common_goal_new": 1, + "common_goal_old": 0, + "common_scored": 25, + "common_strict_new": 0, + "common_strict_old": 0, + "goal_gained": 1, + "goal_lost": 0, + "goal_retained": 0, + "goal_stable_fail": 24, + "new_excluded": 0, + "new_goals": 1, + "new_scored": 25, + "new_strict": 0, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 0, + "old_scored": 25, + "old_strict": 0, + "pairs": 25, + "strict_gained": 0, + "strict_lost": 0, + "strict_retained": 0, + "strict_stable_fail": 25 + }, + { + "model": "deepseek-ai/DeepSeek-V4-Flash-0731", + "flow": "maintenance", + "common_goal_new": 0, + "common_goal_old": 0, + "common_scored": 3, + "common_strict_new": 0, + "common_strict_old": 0, + "goal_gained": 0, + "goal_lost": 0, + "goal_retained": 0, + "goal_stable_fail": 3, + "new_excluded": 0, + "new_goals": 0, + "new_scored": 3, + "new_strict": 0, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 0, + "old_scored": 3, + "old_strict": 0, + "pairs": 3, + "strict_gained": 0, + "strict_lost": 0, + "strict_retained": 0, + "strict_stable_fail": 3 + }, + { + "model": "deepseek-ai/DeepSeek-V4.1-Flash", + "flow": "commerce", + "common_goal_new": 7, + "common_goal_old": 9, + "common_scored": 16, + "common_strict_new": 2, + "common_strict_old": 0, + "goal_gained": 0, + "goal_lost": 2, + "goal_retained": 7, + "goal_stable_fail": 7, + "new_excluded": 0, + "new_goals": 7, + "new_scored": 16, + "new_strict": 2, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 9, + "old_scored": 16, + "old_strict": 0, + "pairs": 16, + "strict_gained": 2, + "strict_lost": 0, + "strict_retained": 0, + "strict_stable_fail": 14 + }, + { + "model": "deepseek-ai/DeepSeek-V4.1-Flash", + "flow": "compliance", + "common_goal_new": 0, + "common_goal_old": 0, + "common_scored": 24, + "common_strict_new": 0, + "common_strict_old": 0, + "goal_gained": 0, + "goal_lost": 0, + "goal_retained": 0, + "goal_stable_fail": 24, + "new_excluded": 1, + "new_goals": 0, + "new_scored": 24, + "new_strict": 0, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 0, + "old_scored": 25, + "old_strict": 0, + "pairs": 25, + "strict_gained": 0, + "strict_lost": 0, + "strict_retained": 0, + "strict_stable_fail": 24 + }, + { + "model": "deepseek-ai/DeepSeek-V4.1-Flash", + "flow": "maintenance", + "common_goal_new": 0, + "common_goal_old": 0, + "common_scored": 3, + "common_strict_new": 0, + "common_strict_old": 0, + "goal_gained": 0, + "goal_lost": 0, + "goal_retained": 0, + "goal_stable_fail": 3, + "new_excluded": 0, + "new_goals": 0, + "new_scored": 3, + "new_strict": 0, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 0, + "old_scored": 3, + "old_strict": 0, + "pairs": 3, + "strict_gained": 0, + "strict_lost": 0, + "strict_retained": 0, + "strict_stable_fail": 3 + }, + { + "model": "gemini-3.8-flash", + "flow": "commerce", + "common_goal_new": 16, + "common_goal_old": 16, + "common_scored": 16, + "common_strict_new": 12, + "common_strict_old": 10, + "goal_gained": 0, + "goal_lost": 0, + "goal_retained": 16, + "goal_stable_fail": 0, + "new_excluded": 0, + "new_goals": 16, + "new_scored": 16, + "new_strict": 12, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 16, + "old_scored": 16, + "old_strict": 10, + "pairs": 16, + "strict_gained": 3, + "strict_lost": 1, + "strict_retained": 9, + "strict_stable_fail": 3 + }, + { + "model": "gemini-3.8-flash", + "flow": "compliance", + "common_goal_new": 23, + "common_goal_old": 23, + "common_scored": 23, + "common_strict_new": 7, + "common_strict_old": 10, + "goal_gained": 0, + "goal_lost": 0, + "goal_retained": 23, + "goal_stable_fail": 0, + "new_excluded": 1, + "new_goals": 24, + "new_scored": 24, + "new_strict": 7, + "old_aborted": 0, + "old_excluded": 1, + "old_goals": 24, + "old_scored": 24, + "old_strict": 10, + "pairs": 25, + "strict_gained": 1, + "strict_lost": 4, + "strict_retained": 6, + "strict_stable_fail": 12 + }, + { + "model": "gemini-3.8-flash", + "flow": "maintenance", + "common_goal_new": 3, + "common_goal_old": 3, + "common_scored": 3, + "common_strict_new": 0, + "common_strict_old": 0, + "goal_gained": 0, + "goal_lost": 0, + "goal_retained": 3, + "goal_stable_fail": 0, + "new_excluded": 0, + "new_goals": 3, + "new_scored": 3, + "new_strict": 0, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 3, + "old_scored": 3, + "old_strict": 0, + "pairs": 3, + "strict_gained": 0, + "strict_lost": 0, + "strict_retained": 0, + "strict_stable_fail": 3 + }, + { + "model": "gemini-4-argon", + "flow": "commerce", + "common_goal_new": 16, + "common_goal_old": 16, + "common_scored": 16, + "common_strict_new": 15, + "common_strict_old": 15, + "goal_gained": 0, + "goal_lost": 0, + "goal_retained": 16, + "goal_stable_fail": 0, + "new_excluded": 0, + "new_goals": 16, + "new_scored": 16, + "new_strict": 15, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 16, + "old_scored": 16, + "old_strict": 15, + "pairs": 16, + "strict_gained": 0, + "strict_lost": 0, + "strict_retained": 15, + "strict_stable_fail": 1 + }, + { + "model": "gemini-4-argon", + "flow": "compliance", + "common_goal_new": 25, + "common_goal_old": 25, + "common_scored": 25, + "common_strict_new": 24, + "common_strict_old": 25, + "goal_gained": 0, + "goal_lost": 0, + "goal_retained": 25, + "goal_stable_fail": 0, + "new_excluded": 0, + "new_goals": 25, + "new_scored": 25, + "new_strict": 24, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 25, + "old_scored": 25, + "old_strict": 25, + "pairs": 25, + "strict_gained": 0, + "strict_lost": 1, + "strict_retained": 24, + "strict_stable_fail": 0 + }, + { + "model": "gemini-4-argon", + "flow": "maintenance", + "common_goal_new": 3, + "common_goal_old": 3, + "common_scored": 3, + "common_strict_new": 3, + "common_strict_old": 2, + "goal_gained": 0, + "goal_lost": 0, + "goal_retained": 3, + "goal_stable_fail": 0, + "new_excluded": 0, + "new_goals": 3, + "new_scored": 3, + "new_strict": 3, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 3, + "old_scored": 3, + "old_strict": 2, + "pairs": 3, + "strict_gained": 1, + "strict_lost": 0, + "strict_retained": 2, + "strict_stable_fail": 0 + }, + { + "model": "gpt-6-astra", + "flow": "commerce", + "common_goal_new": 16, + "common_goal_old": 16, + "common_scored": 16, + "common_strict_new": 15, + "common_strict_old": 14, + "goal_gained": 0, + "goal_lost": 0, + "goal_retained": 16, + "goal_stable_fail": 0, + "new_excluded": 0, + "new_goals": 16, + "new_scored": 16, + "new_strict": 15, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 16, + "old_scored": 16, + "old_strict": 14, + "pairs": 16, + "strict_gained": 1, + "strict_lost": 0, + "strict_retained": 14, + "strict_stable_fail": 1 + }, + { + "model": "gpt-6-astra", + "flow": "compliance", + "common_goal_new": 25, + "common_goal_old": 25, + "common_scored": 25, + "common_strict_new": 25, + "common_strict_old": 25, + "goal_gained": 0, + "goal_lost": 0, + "goal_retained": 25, + "goal_stable_fail": 0, + "new_excluded": 0, + "new_goals": 25, + "new_scored": 25, + "new_strict": 25, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 25, + "old_scored": 25, + "old_strict": 25, + "pairs": 25, + "strict_gained": 0, + "strict_lost": 0, + "strict_retained": 25, + "strict_stable_fail": 0 + }, + { + "model": "gpt-6-astra", + "flow": "maintenance", + "common_goal_new": 2, + "common_goal_old": 2, + "common_scored": 2, + "common_strict_new": 2, + "common_strict_old": 2, + "goal_gained": 0, + "goal_lost": 0, + "goal_retained": 2, + "goal_stable_fail": 0, + "new_excluded": 1, + "new_goals": 2, + "new_scored": 2, + "new_strict": 2, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 3, + "old_scored": 3, + "old_strict": 3, + "pairs": 3, + "strict_gained": 0, + "strict_lost": 0, + "strict_retained": 2, + "strict_stable_fail": 0 + }, + { + "model": "gpt-6-luna", + "flow": "commerce", + "common_goal_new": 9, + "common_goal_old": 9, + "common_scored": 15, + "common_strict_new": 0, + "common_strict_old": 0, + "goal_gained": 1, + "goal_lost": 1, + "goal_retained": 8, + "goal_stable_fail": 5, + "new_excluded": 0, + "new_goals": 9, + "new_scored": 16, + "new_strict": 0, + "old_aborted": 1, + "old_excluded": 0, + "old_goals": 9, + "old_scored": 15, + "old_strict": 0, + "pairs": 16, + "strict_gained": 0, + "strict_lost": 0, + "strict_retained": 0, + "strict_stable_fail": 15 + }, + { + "model": "gpt-6-luna", + "flow": "compliance", + "common_goal_new": 0, + "common_goal_old": 0, + "common_scored": 25, + "common_strict_new": 0, + "common_strict_old": 0, + "goal_gained": 0, + "goal_lost": 0, + "goal_retained": 0, + "goal_stable_fail": 25, + "new_excluded": 0, + "new_goals": 0, + "new_scored": 25, + "new_strict": 0, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 0, + "old_scored": 25, + "old_strict": 0, + "pairs": 25, + "strict_gained": 0, + "strict_lost": 0, + "strict_retained": 0, + "strict_stable_fail": 25 + }, + { + "model": "gpt-6-luna", + "flow": "maintenance", + "common_goal_new": 0, + "common_goal_old": 0, + "common_scored": 0, + "common_strict_new": 0, + "common_strict_old": 0, + "goal_gained": 0, + "goal_lost": 0, + "goal_retained": 0, + "goal_stable_fail": 0, + "new_excluded": 0, + "new_goals": 0, + "new_scored": 3, + "new_strict": 0, + "old_aborted": 3, + "old_excluded": 0, + "old_goals": 0, + "old_scored": 0, + "old_strict": 0, + "pairs": 3, + "strict_gained": 0, + "strict_lost": 0, + "strict_retained": 0, + "strict_stable_fail": 0 + }, + { + "model": "gpt-6-sol", + "flow": "commerce", + "common_goal_new": 16, + "common_goal_old": 14, + "common_scored": 16, + "common_strict_new": 11, + "common_strict_old": 10, + "goal_gained": 2, + "goal_lost": 0, + "goal_retained": 14, + "goal_stable_fail": 0, + "new_excluded": 0, + "new_goals": 16, + "new_scored": 16, + "new_strict": 11, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 14, + "old_scored": 16, + "old_strict": 10, + "pairs": 16, + "strict_gained": 2, + "strict_lost": 1, + "strict_retained": 9, + "strict_stable_fail": 4 + }, + { + "model": "gpt-6-sol", + "flow": "compliance", + "common_goal_new": 22, + "common_goal_old": 23, + "common_scored": 25, + "common_strict_new": 8, + "common_strict_old": 8, + "goal_gained": 2, + "goal_lost": 3, + "goal_retained": 20, + "goal_stable_fail": 0, + "new_excluded": 0, + "new_goals": 22, + "new_scored": 25, + "new_strict": 8, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 23, + "old_scored": 25, + "old_strict": 8, + "pairs": 25, + "strict_gained": 5, + "strict_lost": 5, + "strict_retained": 3, + "strict_stable_fail": 12 + }, + { + "model": "gpt-6-sol", + "flow": "maintenance", + "common_goal_new": 2, + "common_goal_old": 2, + "common_scored": 3, + "common_strict_new": 1, + "common_strict_old": 1, + "goal_gained": 0, + "goal_lost": 0, + "goal_retained": 2, + "goal_stable_fail": 1, + "new_excluded": 0, + "new_goals": 2, + "new_scored": 3, + "new_strict": 1, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 2, + "old_scored": 3, + "old_strict": 1, + "pairs": 3, + "strict_gained": 0, + "strict_lost": 0, + "strict_retained": 1, + "strict_stable_fail": 2 + }, + { + "model": "gpt-6.1-sol", + "flow": "commerce", + "common_goal_new": 16, + "common_goal_old": 16, + "common_scored": 16, + "common_strict_new": 16, + "common_strict_old": 16, + "goal_gained": 0, + "goal_lost": 0, + "goal_retained": 16, + "goal_stable_fail": 0, + "new_excluded": 0, + "new_goals": 16, + "new_scored": 16, + "new_strict": 16, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 16, + "old_scored": 16, + "old_strict": 16, + "pairs": 16, + "strict_gained": 0, + "strict_lost": 0, + "strict_retained": 16, + "strict_stable_fail": 0 + }, + { + "model": "gpt-6.1-sol", + "flow": "compliance", + "common_goal_new": 25, + "common_goal_old": 25, + "common_scored": 25, + "common_strict_new": 23, + "common_strict_old": 24, + "goal_gained": 0, + "goal_lost": 0, + "goal_retained": 25, + "goal_stable_fail": 0, + "new_excluded": 0, + "new_goals": 25, + "new_scored": 25, + "new_strict": 23, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 25, + "old_scored": 25, + "old_strict": 24, + "pairs": 25, + "strict_gained": 1, + "strict_lost": 2, + "strict_retained": 22, + "strict_stable_fail": 0 + }, + { + "model": "gpt-6.1-sol", + "flow": "maintenance", + "common_goal_new": 3, + "common_goal_old": 3, + "common_scored": 3, + "common_strict_new": 3, + "common_strict_old": 2, + "goal_gained": 0, + "goal_lost": 0, + "goal_retained": 3, + "goal_stable_fail": 0, + "new_excluded": 0, + "new_goals": 3, + "new_scored": 3, + "new_strict": 3, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 3, + "old_scored": 3, + "old_strict": 2, + "pairs": 3, + "strict_gained": 1, + "strict_lost": 0, + "strict_retained": 2, + "strict_stable_fail": 0 + }, + { + "model": "mistral-medium-3-5", + "flow": "commerce", + "common_goal_new": 0, + "common_goal_old": 0, + "common_scored": 16, + "common_strict_new": 0, + "common_strict_old": 0, + "goal_gained": 0, + "goal_lost": 0, + "goal_retained": 0, + "goal_stable_fail": 16, + "new_excluded": 0, + "new_goals": 0, + "new_scored": 16, + "new_strict": 0, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 0, + "old_scored": 16, + "old_strict": 0, + "pairs": 16, + "strict_gained": 0, + "strict_lost": 0, + "strict_retained": 0, + "strict_stable_fail": 16 + }, + { + "model": "mistral-medium-3-5", + "flow": "compliance", + "common_goal_new": 0, + "common_goal_old": 0, + "common_scored": 25, + "common_strict_new": 0, + "common_strict_old": 0, + "goal_gained": 0, + "goal_lost": 0, + "goal_retained": 0, + "goal_stable_fail": 25, + "new_excluded": 0, + "new_goals": 0, + "new_scored": 25, + "new_strict": 0, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 0, + "old_scored": 25, + "old_strict": 0, + "pairs": 25, + "strict_gained": 0, + "strict_lost": 0, + "strict_retained": 0, + "strict_stable_fail": 25 + }, + { + "model": "mistral-medium-3-5", + "flow": "maintenance", + "common_goal_new": 0, + "common_goal_old": 0, + "common_scored": 3, + "common_strict_new": 0, + "common_strict_old": 0, + "goal_gained": 0, + "goal_lost": 0, + "goal_retained": 0, + "goal_stable_fail": 3, + "new_excluded": 0, + "new_goals": 0, + "new_scored": 3, + "new_strict": 0, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 0, + "old_scored": 3, + "old_strict": 0, + "pairs": 3, + "strict_gained": 0, + "strict_lost": 0, + "strict_retained": 0, + "strict_stable_fail": 3 + }, + { + "model": "muse-spark-1.3", + "flow": "commerce", + "common_goal_new": 16, + "common_goal_old": 16, + "common_scored": 16, + "common_strict_new": 4, + "common_strict_old": 4, + "goal_gained": 0, + "goal_lost": 0, + "goal_retained": 16, + "goal_stable_fail": 0, + "new_excluded": 0, + "new_goals": 16, + "new_scored": 16, + "new_strict": 4, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 16, + "old_scored": 16, + "old_strict": 4, + "pairs": 16, + "strict_gained": 3, + "strict_lost": 3, + "strict_retained": 1, + "strict_stable_fail": 9 + }, + { + "model": "muse-spark-1.3", + "flow": "compliance", + "common_goal_new": 1, + "common_goal_old": 1, + "common_scored": 25, + "common_strict_new": 0, + "common_strict_old": 0, + "goal_gained": 1, + "goal_lost": 1, + "goal_retained": 0, + "goal_stable_fail": 23, + "new_excluded": 0, + "new_goals": 1, + "new_scored": 25, + "new_strict": 0, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 1, + "old_scored": 25, + "old_strict": 0, + "pairs": 25, + "strict_gained": 0, + "strict_lost": 0, + "strict_retained": 0, + "strict_stable_fail": 25 + }, + { + "model": "muse-spark-1.3", + "flow": "maintenance", + "common_goal_new": 2, + "common_goal_old": 1, + "common_scored": 3, + "common_strict_new": 1, + "common_strict_old": 0, + "goal_gained": 1, + "goal_lost": 0, + "goal_retained": 1, + "goal_stable_fail": 1, + "new_excluded": 0, + "new_goals": 2, + "new_scored": 3, + "new_strict": 1, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 1, + "old_scored": 3, + "old_strict": 0, + "pairs": 3, + "strict_gained": 1, + "strict_lost": 0, + "strict_retained": 0, + "strict_stable_fail": 2 + }, + { + "model": "muse-wonder-1.0", + "flow": "commerce", + "common_goal_new": 14, + "common_goal_old": 15, + "common_scored": 16, + "common_strict_new": 8, + "common_strict_old": 10, + "goal_gained": 1, + "goal_lost": 2, + "goal_retained": 13, + "goal_stable_fail": 0, + "new_excluded": 0, + "new_goals": 14, + "new_scored": 16, + "new_strict": 8, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 15, + "old_scored": 16, + "old_strict": 10, + "pairs": 16, + "strict_gained": 1, + "strict_lost": 3, + "strict_retained": 7, + "strict_stable_fail": 5 + }, + { + "model": "muse-wonder-1.0", + "flow": "compliance", + "common_goal_new": 5, + "common_goal_old": 9, + "common_scored": 23, + "common_strict_new": 2, + "common_strict_old": 1, + "goal_gained": 0, + "goal_lost": 4, + "goal_retained": 5, + "goal_stable_fail": 14, + "new_excluded": 2, + "new_goals": 5, + "new_scored": 23, + "new_strict": 2, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 10, + "old_scored": 25, + "old_strict": 1, + "pairs": 25, + "strict_gained": 1, + "strict_lost": 0, + "strict_retained": 1, + "strict_stable_fail": 21 + }, + { + "model": "muse-wonder-1.0", + "flow": "maintenance", + "common_goal_new": 1, + "common_goal_old": 1, + "common_scored": 1, + "common_strict_new": 0, + "common_strict_old": 0, + "goal_gained": 0, + "goal_lost": 0, + "goal_retained": 1, + "goal_stable_fail": 0, + "new_excluded": 2, + "new_goals": 1, + "new_scored": 1, + "new_strict": 0, + "old_aborted": 0, + "old_excluded": 1, + "old_goals": 2, + "old_scored": 2, + "old_strict": 1, + "pairs": 3, + "strict_gained": 0, + "strict_lost": 0, + "strict_retained": 0, + "strict_stable_fail": 1 + }, + { + "model": "nvidia/GLM-5.2-NVFP4-262K", + "flow": "commerce", + "common_goal_new": 16, + "common_goal_old": 16, + "common_scored": 16, + "common_strict_new": 6, + "common_strict_old": 6, + "goal_gained": 0, + "goal_lost": 0, + "goal_retained": 16, + "goal_stable_fail": 0, + "new_excluded": 0, + "new_goals": 16, + "new_scored": 16, + "new_strict": 6, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 16, + "old_scored": 16, + "old_strict": 6, + "pairs": 16, + "strict_gained": 2, + "strict_lost": 2, + "strict_retained": 4, + "strict_stable_fail": 8 + }, + { + "model": "nvidia/GLM-5.2-NVFP4-262K", + "flow": "compliance", + "common_goal_new": 13, + "common_goal_old": 17, + "common_scored": 25, + "common_strict_new": 0, + "common_strict_old": 0, + "goal_gained": 1, + "goal_lost": 5, + "goal_retained": 12, + "goal_stable_fail": 7, + "new_excluded": 0, + "new_goals": 13, + "new_scored": 25, + "new_strict": 0, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 17, + "old_scored": 25, + "old_strict": 0, + "pairs": 25, + "strict_gained": 0, + "strict_lost": 0, + "strict_retained": 0, + "strict_stable_fail": 25 + }, + { + "model": "nvidia/GLM-5.2-NVFP4-262K", + "flow": "maintenance", + "common_goal_new": 0, + "common_goal_old": 1, + "common_scored": 3, + "common_strict_new": 0, + "common_strict_old": 0, + "goal_gained": 0, + "goal_lost": 1, + "goal_retained": 0, + "goal_stable_fail": 2, + "new_excluded": 0, + "new_goals": 0, + "new_scored": 3, + "new_strict": 0, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 1, + "old_scored": 3, + "old_strict": 0, + "pairs": 3, + "strict_gained": 0, + "strict_lost": 0, + "strict_retained": 0, + "strict_stable_fail": 3 + }, + { + "model": "nvidia/Kimi-K2.6-NVFP4", + "flow": "commerce", + "common_goal_new": 7, + "common_goal_old": 7, + "common_scored": 16, + "common_strict_new": 0, + "common_strict_old": 0, + "goal_gained": 0, + "goal_lost": 0, + "goal_retained": 7, + "goal_stable_fail": 9, + "new_excluded": 0, + "new_goals": 7, + "new_scored": 16, + "new_strict": 0, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 7, + "old_scored": 16, + "old_strict": 0, + "pairs": 16, + "strict_gained": 0, + "strict_lost": 0, + "strict_retained": 0, + "strict_stable_fail": 16 + }, + { + "model": "nvidia/Kimi-K2.6-NVFP4", + "flow": "compliance", + "common_goal_new": 0, + "common_goal_old": 0, + "common_scored": 24, + "common_strict_new": 0, + "common_strict_old": 0, + "goal_gained": 0, + "goal_lost": 0, + "goal_retained": 0, + "goal_stable_fail": 24, + "new_excluded": 0, + "new_goals": 0, + "new_scored": 25, + "new_strict": 0, + "old_aborted": 0, + "old_excluded": 1, + "old_goals": 0, + "old_scored": 24, + "old_strict": 0, + "pairs": 25, + "strict_gained": 0, + "strict_lost": 0, + "strict_retained": 0, + "strict_stable_fail": 24 + }, + { + "model": "nvidia/Kimi-K2.6-NVFP4", + "flow": "maintenance", + "common_goal_new": 0, + "common_goal_old": 0, + "common_scored": 3, + "common_strict_new": 0, + "common_strict_old": 0, + "goal_gained": 0, + "goal_lost": 0, + "goal_retained": 0, + "goal_stable_fail": 3, + "new_excluded": 0, + "new_goals": 0, + "new_scored": 3, + "new_strict": 0, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 0, + "old_scored": 3, + "old_strict": 0, + "pairs": 3, + "strict_gained": 0, + "strict_lost": 0, + "strict_retained": 0, + "strict_stable_fail": 3 + }, + { + "model": "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4", + "flow": "commerce", + "common_goal_new": 0, + "common_goal_old": 0, + "common_scored": 16, + "common_strict_new": 0, + "common_strict_old": 0, + "goal_gained": 0, + "goal_lost": 0, + "goal_retained": 0, + "goal_stable_fail": 16, + "new_excluded": 0, + "new_goals": 0, + "new_scored": 16, + "new_strict": 0, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 0, + "old_scored": 16, + "old_strict": 0, + "pairs": 16, + "strict_gained": 0, + "strict_lost": 0, + "strict_retained": 0, + "strict_stable_fail": 16 + }, + { + "model": "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4", + "flow": "compliance", + "common_goal_new": 0, + "common_goal_old": 0, + "common_scored": 25, + "common_strict_new": 0, + "common_strict_old": 0, + "goal_gained": 0, + "goal_lost": 0, + "goal_retained": 0, + "goal_stable_fail": 25, + "new_excluded": 0, + "new_goals": 0, + "new_scored": 25, + "new_strict": 0, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 0, + "old_scored": 25, + "old_strict": 0, + "pairs": 25, + "strict_gained": 0, + "strict_lost": 0, + "strict_retained": 0, + "strict_stable_fail": 25 + }, + { + "model": "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4", + "flow": "maintenance", + "common_goal_new": 0, + "common_goal_old": 0, + "common_scored": 3, + "common_strict_new": 0, + "common_strict_old": 0, + "goal_gained": 0, + "goal_lost": 0, + "goal_retained": 0, + "goal_stable_fail": 3, + "new_excluded": 0, + "new_goals": 0, + "new_scored": 3, + "new_strict": 0, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 0, + "old_scored": 3, + "old_strict": 0, + "pairs": 3, + "strict_gained": 0, + "strict_lost": 0, + "strict_retained": 0, + "strict_stable_fail": 3 + }, + { + "model": "zai-org/GLM-5.3-Flash", + "flow": "commerce", + "common_goal_new": 14, + "common_goal_old": 11, + "common_scored": 16, + "common_strict_new": 5, + "common_strict_old": 3, + "goal_gained": 3, + "goal_lost": 0, + "goal_retained": 11, + "goal_stable_fail": 2, + "new_excluded": 0, + "new_goals": 14, + "new_scored": 16, + "new_strict": 5, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 11, + "old_scored": 16, + "old_strict": 3, + "pairs": 16, + "strict_gained": 2, + "strict_lost": 0, + "strict_retained": 3, + "strict_stable_fail": 11 + }, + { + "model": "zai-org/GLM-5.3-Flash", + "flow": "compliance", + "common_goal_new": 21, + "common_goal_old": 21, + "common_scored": 25, + "common_strict_new": 4, + "common_strict_old": 1, + "goal_gained": 2, + "goal_lost": 2, + "goal_retained": 19, + "goal_stable_fail": 2, + "new_excluded": 0, + "new_goals": 21, + "new_scored": 25, + "new_strict": 4, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 21, + "old_scored": 25, + "old_strict": 1, + "pairs": 25, + "strict_gained": 3, + "strict_lost": 0, + "strict_retained": 1, + "strict_stable_fail": 21 + }, + { + "model": "zai-org/GLM-5.3-Flash", + "flow": "maintenance", + "common_goal_new": 0, + "common_goal_old": 1, + "common_scored": 3, + "common_strict_new": 0, + "common_strict_old": 0, + "goal_gained": 0, + "goal_lost": 1, + "goal_retained": 0, + "goal_stable_fail": 2, + "new_excluded": 0, + "new_goals": 0, + "new_scored": 3, + "new_strict": 0, + "old_aborted": 0, + "old_excluded": 0, + "old_goals": 1, + "old_scored": 3, + "old_strict": 0, + "pairs": 3, + "strict_gained": 0, + "strict_lost": 0, + "strict_retained": 0, + "strict_stable_fail": 3 + } + ], + "flow_totals": { + "maintenance": { + "common_goal_new": 22, + "common_goal_old": 22, + "common_scored": 57, + "common_strict_new": 11, + "common_strict_old": 9, + "goal_gained": 2, + "goal_lost": 2, + "goal_retained": 20, + "goal_stable_fail": 33, + "new_excluded": 3, + "new_goals": 22, + "new_scored": 60, + "new_strict": 11, + "old_aborted": 3, + "old_excluded": 1, + "old_goals": 24, + "old_scored": 59, + "old_strict": 11, + "pairs": 63, + "strict_gained": 3, + "strict_lost": 1, + "strict_retained": 8, + "strict_stable_fail": 45 + }, + "commerce": { + "common_goal_new": 208, + "common_goal_old": 206, + "common_scored": 334, + "common_strict_new": 123, + "common_strict_old": 118, + "goal_gained": 10, + "goal_lost": 8, + "goal_retained": 198, + "goal_stable_fail": 118, + "new_excluded": 0, + "new_goals": 209, + "new_scored": 336, + "new_strict": 124, + "old_aborted": 1, + "old_excluded": 1, + "old_goals": 206, + "old_scored": 334, + "old_strict": 118, + "pairs": 336, + "strict_gained": 18, + "strict_lost": 13, + "strict_retained": 105, + "strict_stable_fail": 198 + }, + "compliance": { + "common_goal_new": 211, + "common_goal_old": 219, + "common_scored": 519, + "common_strict_new": 126, + "common_strict_old": 121, + "goal_gained": 7, + "goal_lost": 15, + "goal_retained": 204, + "goal_stable_fail": 293, + "new_excluded": 4, + "new_goals": 212, + "new_scored": 521, + "new_strict": 126, + "old_aborted": 0, + "old_excluded": 2, + "old_goals": 221, + "old_scored": 523, + "old_strict": 121, + "pairs": 525, + "strict_gained": 18, + "strict_lost": 13, + "strict_retained": 108, + "strict_stable_fail": 380 + } + }, + "compliance_diagnostic": { + "scope": "Compliance-only offline diagnostic, not authenticated replay or official regrade; no new model calls. Other flow grades not re-evaluated.", + "baseline": "Original unversioned evaluator shipped with pack0.2.0", + "candidate_evaluator_version": "0.3.0", + "counts": { + "old_records": 523, + "old_findings_changed": 12, + "old_retrieval_changed": 12, + "old_reliable_changed": 5, + "old_baseline_reliable": 121, + "old_candidate_reliable": 116, + "old_goal": 221, + "old_baseline_retrieval": 424, + "old_candidate_retrieval": 412, + "new_records": 521, + "new_findings_changed": 11, + "new_retrieval_changed": 11, + "new_reliable_changed": 6, + "new_baseline_reliable": 126, + "new_candidate_reliable": 120, + "new_goal": 212, + "new_baseline_retrieval": 426, + "new_candidate_retrieval": 415 + }, + "models": [ + { + "epoch": "new", + "model": "Qwen/Qwen3.5-397B-A17B-FP8", + "records": 25, + "findings_changed": 0, + "retrieval_changed": 0, + "reliable_changed": 0, + "baseline_reliable": 0, + "candidate_reliable": 0, + "goal": 0, + "baseline_retrieval": 5, + "candidate_retrieval": 5 + }, + { + "epoch": "new", + "model": "Qwen/Qwen3.5-9B", + "records": 25, + "findings_changed": 0, + "retrieval_changed": 0, + "reliable_changed": 0, + "baseline_reliable": 0, + "candidate_reliable": 0, + "goal": 0, + "baseline_retrieval": 25, + "candidate_retrieval": 25 + }, + { + "epoch": "new", + "model": "Qwen/Qwen3.6-35B-A3B-FP8", + "records": 25, + "findings_changed": 0, + "retrieval_changed": 0, + "reliable_changed": 0, + "baseline_reliable": 0, + "candidate_reliable": 0, + "goal": 0, + "baseline_retrieval": 8, + "candidate_retrieval": 8 + }, + { + "epoch": "new", + "model": "Qwen/Qwen3.8-27B-FP8", + "records": 25, + "findings_changed": 0, + "retrieval_changed": 0, + "reliable_changed": 0, + "baseline_reliable": 0, + "candidate_reliable": 0, + "goal": 0, + "baseline_retrieval": 19, + "candidate_retrieval": 19 + }, + { + "epoch": "new", + "model": "claude-fable-5-1", + "records": 25, + "findings_changed": 0, + "retrieval_changed": 0, + "reliable_changed": 0, + "baseline_reliable": 19, + "candidate_reliable": 19, + "goal": 25, + "baseline_retrieval": 25, + "candidate_retrieval": 25 + }, + { + "epoch": "new", + "model": "claude-opus-5-5", + "records": 25, + "findings_changed": 0, + "retrieval_changed": 0, + "reliable_changed": 0, + "baseline_reliable": 14, + "candidate_reliable": 14, + "goal": 25, + "baseline_retrieval": 25, + "candidate_retrieval": 25 + }, + { + "epoch": "new", + "model": "deepseek-ai/DeepSeek-V4-Flash-0731", + "records": 25, + "findings_changed": 0, + "retrieval_changed": 0, + "reliable_changed": 0, + "baseline_reliable": 0, + "candidate_reliable": 0, + "goal": 1, + "baseline_retrieval": 25, + "candidate_retrieval": 25 + }, + { + "epoch": "new", + "model": "deepseek-ai/DeepSeek-V4.1-Flash", + "records": 24, + "findings_changed": 0, + "retrieval_changed": 0, + "reliable_changed": 0, + "baseline_reliable": 0, + "candidate_reliable": 0, + "goal": 0, + "baseline_retrieval": 23, + "candidate_retrieval": 23 + }, + { + "epoch": "new", + "model": "gemini-3.8-flash", + "records": 24, + "findings_changed": 5, + "retrieval_changed": 5, + "reliable_changed": 4, + "baseline_reliable": 7, + "candidate_reliable": 3, + "goal": 24, + "baseline_retrieval": 8, + "candidate_retrieval": 3 + }, + { + "epoch": "new", + "model": "gemini-4-argon", + "records": 25, + "findings_changed": 0, + "retrieval_changed": 0, + "reliable_changed": 0, + "baseline_reliable": 24, + "candidate_reliable": 24, + "goal": 25, + "baseline_retrieval": 25, + "candidate_retrieval": 25 + }, + { + "epoch": "new", + "model": "gpt-6-astra", + "records": 25, + "findings_changed": 0, + "retrieval_changed": 0, + "reliable_changed": 0, + "baseline_reliable": 25, + "candidate_reliable": 25, + "goal": 25, + "baseline_retrieval": 25, + "candidate_retrieval": 25 + }, + { + "epoch": "new", + "model": "gpt-6-luna", + "records": 25, + "findings_changed": 0, + "retrieval_changed": 0, + "reliable_changed": 0, + "baseline_reliable": 0, + "candidate_reliable": 0, + "goal": 0, + "baseline_retrieval": 25, + "candidate_retrieval": 25 + }, + { + "epoch": "new", + "model": "gpt-6-sol", + "records": 25, + "findings_changed": 0, + "retrieval_changed": 0, + "reliable_changed": 0, + "baseline_reliable": 8, + "candidate_reliable": 8, + "goal": 22, + "baseline_retrieval": 25, + "candidate_retrieval": 25 + }, + { + "epoch": "new", + "model": "gpt-6.1-sol", + "records": 25, + "findings_changed": 0, + "retrieval_changed": 0, + "reliable_changed": 0, + "baseline_reliable": 23, + "candidate_reliable": 23, + "goal": 25, + "baseline_retrieval": 25, + "candidate_retrieval": 25 + }, + { + "epoch": "new", + "model": "mistral-medium-3-5", + "records": 25, + "findings_changed": 0, + "retrieval_changed": 0, + "reliable_changed": 0, + "baseline_reliable": 0, + "candidate_reliable": 0, + "goal": 0, + "baseline_retrieval": 25, + "candidate_retrieval": 25 + }, + { + "epoch": "new", + "model": "muse-spark-1.3", + "records": 25, + "findings_changed": 0, + "retrieval_changed": 0, + "reliable_changed": 0, + "baseline_reliable": 0, + "candidate_reliable": 0, + "goal": 1, + "baseline_retrieval": 24, + "candidate_retrieval": 24 + }, + { + "epoch": "new", + "model": "muse-wonder-1.0", + "records": 23, + "findings_changed": 0, + "retrieval_changed": 0, + "reliable_changed": 0, + "baseline_reliable": 2, + "candidate_reliable": 2, + "goal": 5, + "baseline_retrieval": 23, + "candidate_retrieval": 23 + }, + { + "epoch": "new", + "model": "nvidia/GLM-5.2-NVFP4-262K", + "records": 25, + "findings_changed": 0, + "retrieval_changed": 0, + "reliable_changed": 0, + "baseline_reliable": 0, + "candidate_reliable": 0, + "goal": 13, + "baseline_retrieval": 24, + "candidate_retrieval": 24 + }, + { + "epoch": "new", + "model": "nvidia/Kimi-K2.6-NVFP4", + "records": 25, + "findings_changed": 0, + "retrieval_changed": 0, + "reliable_changed": 0, + "baseline_reliable": 0, + "candidate_reliable": 0, + "goal": 0, + "baseline_retrieval": 1, + "candidate_retrieval": 1 + }, + { + "epoch": "new", + "model": "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4", + "records": 25, + "findings_changed": 0, + "retrieval_changed": 0, + "reliable_changed": 0, + "baseline_reliable": 0, + "candidate_reliable": 0, + "goal": 0, + "baseline_retrieval": 25, + "candidate_retrieval": 25 + }, + { + "epoch": "new", + "model": "zai-org/GLM-5.3-Flash", + "records": 25, + "findings_changed": 6, + "retrieval_changed": 6, + "reliable_changed": 2, + "baseline_reliable": 4, + "candidate_reliable": 2, + "goal": 21, + "baseline_retrieval": 16, + "candidate_retrieval": 10 + }, + { + "epoch": "old", + "model": "Qwen/Qwen3.5-397B-A17B-FP8", + "records": 25, + "findings_changed": 0, + "retrieval_changed": 0, + "reliable_changed": 0, + "baseline_reliable": 0, + "candidate_reliable": 0, + "goal": 0, + "baseline_retrieval": 5, + "candidate_retrieval": 5 + }, + { + "epoch": "old", + "model": "Qwen/Qwen3.5-9B", + "records": 25, + "findings_changed": 0, + "retrieval_changed": 0, + "reliable_changed": 0, + "baseline_reliable": 0, + "candidate_reliable": 0, + "goal": 0, + "baseline_retrieval": 24, + "candidate_retrieval": 24 + }, + { + "epoch": "old", + "model": "Qwen/Qwen3.6-35B-A3B-FP8", + "records": 25, + "findings_changed": 0, + "retrieval_changed": 0, + "reliable_changed": 0, + "baseline_reliable": 0, + "candidate_reliable": 0, + "goal": 0, + "baseline_retrieval": 8, + "candidate_retrieval": 8 + }, + { + "epoch": "old", + "model": "Qwen/Qwen3.8-27B-FP8", + "records": 25, + "findings_changed": 0, + "retrieval_changed": 0, + "reliable_changed": 0, + "baseline_reliable": 0, + "candidate_reliable": 0, + "goal": 0, + "baseline_retrieval": 18, + "candidate_retrieval": 18 + }, + { + "epoch": "old", + "model": "claude-fable-5-1", + "records": 25, + "findings_changed": 0, + "retrieval_changed": 0, + "reliable_changed": 0, + "baseline_reliable": 19, + "candidate_reliable": 19, + "goal": 25, + "baseline_retrieval": 25, + "candidate_retrieval": 25 + }, + { + "epoch": "old", + "model": "claude-opus-5-5", + "records": 25, + "findings_changed": 0, + "retrieval_changed": 0, + "reliable_changed": 0, + "baseline_reliable": 8, + "candidate_reliable": 8, + "goal": 25, + "baseline_retrieval": 25, + "candidate_retrieval": 25 + }, + { + "epoch": "old", + "model": "deepseek-ai/DeepSeek-V4-Flash-0731", + "records": 25, + "findings_changed": 0, + "retrieval_changed": 0, + "reliable_changed": 0, + "baseline_reliable": 0, + "candidate_reliable": 0, + "goal": 0, + "baseline_retrieval": 25, + "candidate_retrieval": 25 + }, + { + "epoch": "old", + "model": "deepseek-ai/DeepSeek-V4.1-Flash", + "records": 25, + "findings_changed": 0, + "retrieval_changed": 0, + "reliable_changed": 0, + "baseline_reliable": 0, + "candidate_reliable": 0, + "goal": 0, + "baseline_retrieval": 25, + "candidate_retrieval": 25 + }, + { + "epoch": "old", + "model": "gemini-3.8-flash", + "records": 24, + "findings_changed": 5, + "retrieval_changed": 5, + "reliable_changed": 4, + "baseline_reliable": 10, + "candidate_reliable": 6, + "goal": 24, + "baseline_retrieval": 11, + "candidate_retrieval": 6 + }, + { + "epoch": "old", + "model": "gemini-4-argon", + "records": 25, + "findings_changed": 0, + "retrieval_changed": 0, + "reliable_changed": 0, + "baseline_reliable": 25, + "candidate_reliable": 25, + "goal": 25, + "baseline_retrieval": 25, + "candidate_retrieval": 25 + }, + { + "epoch": "old", + "model": "gpt-6-astra", + "records": 25, + "findings_changed": 0, + "retrieval_changed": 0, + "reliable_changed": 0, + "baseline_reliable": 25, + "candidate_reliable": 25, + "goal": 25, + "baseline_retrieval": 25, + "candidate_retrieval": 25 + }, + { + "epoch": "old", + "model": "gpt-6-luna", + "records": 25, + "findings_changed": 0, + "retrieval_changed": 0, + "reliable_changed": 0, + "baseline_reliable": 0, + "candidate_reliable": 0, + "goal": 0, + "baseline_retrieval": 25, + "candidate_retrieval": 25 + }, + { + "epoch": "old", + "model": "gpt-6-sol", + "records": 25, + "findings_changed": 0, + "retrieval_changed": 0, + "reliable_changed": 0, + "baseline_reliable": 8, + "candidate_reliable": 8, + "goal": 23, + "baseline_retrieval": 25, + "candidate_retrieval": 25 + }, + { + "epoch": "old", + "model": "gpt-6.1-sol", + "records": 25, + "findings_changed": 0, + "retrieval_changed": 0, + "reliable_changed": 0, + "baseline_reliable": 24, + "candidate_reliable": 24, + "goal": 25, + "baseline_retrieval": 25, + "candidate_retrieval": 25 + }, + { + "epoch": "old", + "model": "mistral-medium-3-5", + "records": 25, + "findings_changed": 0, + "retrieval_changed": 0, + "reliable_changed": 0, + "baseline_reliable": 0, + "candidate_reliable": 0, + "goal": 0, + "baseline_retrieval": 25, + "candidate_retrieval": 25 + }, + { + "epoch": "old", + "model": "muse-spark-1.3", + "records": 25, + "findings_changed": 0, + "retrieval_changed": 0, + "reliable_changed": 0, + "baseline_reliable": 0, + "candidate_reliable": 0, + "goal": 1, + "baseline_retrieval": 24, + "candidate_retrieval": 24 + }, + { + "epoch": "old", + "model": "muse-wonder-1.0", + "records": 25, + "findings_changed": 0, + "retrieval_changed": 0, + "reliable_changed": 0, + "baseline_reliable": 1, + "candidate_reliable": 1, + "goal": 10, + "baseline_retrieval": 25, + "candidate_retrieval": 25 + }, + { + "epoch": "old", + "model": "nvidia/GLM-5.2-NVFP4-262K", + "records": 25, + "findings_changed": 0, + "retrieval_changed": 0, + "reliable_changed": 0, + "baseline_reliable": 0, + "candidate_reliable": 0, + "goal": 17, + "baseline_retrieval": 24, + "candidate_retrieval": 24 + }, + { + "epoch": "old", + "model": "nvidia/Kimi-K2.6-NVFP4", + "records": 24, + "findings_changed": 0, + "retrieval_changed": 0, + "reliable_changed": 0, + "baseline_reliable": 0, + "candidate_reliable": 0, + "goal": 0, + "baseline_retrieval": 0, + "candidate_retrieval": 0 + }, + { + "epoch": "old", + "model": "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4", + "records": 25, + "findings_changed": 0, + "retrieval_changed": 0, + "reliable_changed": 0, + "baseline_reliable": 0, + "candidate_reliable": 0, + "goal": 0, + "baseline_retrieval": 25, + "candidate_retrieval": 25 + }, + { + "epoch": "old", + "model": "zai-org/GLM-5.3-Flash", + "records": 25, + "findings_changed": 7, + "retrieval_changed": 7, + "reliable_changed": 1, + "baseline_reliable": 1, + "candidate_reliable": 0, + "goal": 21, + "baseline_retrieval": 10, + "candidate_retrieval": 3 + } + ], + "baseline_parity_records": 1044, + "terminal_attempts": { + "old": 18, + "new": 17 + }, + "unmapped_terminal_claim_code_policy": "Unchanged; no broad terminal-refusal expansion." + }, + "provenance": { + "public_results": "https://dwellyorg.github.io/operatebench-results-preview/", + "public_repo": "https://github.com/DwellyOrg/OperateBench", + "fix_pr": "https://github.com/DwellyOrg/OperateBench/pull/37", + "evaluator_source_sha256": "4899ba74ff12e798306b9b85492444272b36f2d03254a1ff3ca6be6c9654bee9", + "corpus_access": "Owner-retained scored records and raw trajectories are not publicly released; aggregate source data is supplied here. Independent external raw-record reproduction is not provided." + }, + "limitations": [ + "Related variants are not independent semantic tasks. Official Lifecycle scope is one Maintenance Semantic Scenario with3variants; Commerce and Compliance are SDK incubators.", + "One trial per model/case per series. Historical and repeat series differ in evaluator/runtime/adapters/settings; no pooling or stochasticity/learning/significance claim.", + "No independent human semantic validation, no provider attestation, no production/legal validity claim.", + "Costs are not published, not zero; no cost-efficiency claim. Latest decision totals are scored Core decisions only, not generation RPCs/all-attempt calls.", + "Original scores remain original. The separately labelled diagnostic changes only Compliance evaluation on saved records." + ] +} diff --git a/docs/studies/2026-10-case-churn.svg b/docs/studies/2026-10-case-churn.svg new file mode 100644 index 0000000..d1b416a --- /dev/null +++ b/docs/studies/2026-10-case-churn.svg @@ -0,0 +1,16 @@ + +Common-scored strict gains and lossesAmong 910 common-scored cases: Maintenance 3 gained and 1 lost; Commerce 18 gained and 13 lost; Compliance 18 gained and 13 lost. Total 39 gained and 27 lost. + +Common-scored strict gains and losses +Gained (upper bar) +Lost (lower bar) +Maintenance +3 +1 +Commerce +18 +13 +Compliance +18 +13 + diff --git a/docs/studies/2026-10-compliance-diagnostic.csv b/docs/studies/2026-10-compliance-diagnostic.csv new file mode 100644 index 0000000..495163f --- /dev/null +++ b/docs/studies/2026-10-compliance-diagnostic.csv @@ -0,0 +1,43 @@ +epoch,model,records,findings_changed,retrieval_changed,reliable_changed,baseline_reliable,candidate_reliable,goal,baseline_retrieval,candidate_retrieval +new,Qwen/Qwen3.5-397B-A17B-FP8,25,0,0,0,0,0,0,5,5 +new,Qwen/Qwen3.5-9B,25,0,0,0,0,0,0,25,25 +new,Qwen/Qwen3.6-35B-A3B-FP8,25,0,0,0,0,0,0,8,8 +new,Qwen/Qwen3.8-27B-FP8,25,0,0,0,0,0,0,19,19 +new,claude-fable-5-1,25,0,0,0,19,19,25,25,25 +new,claude-opus-5-5,25,0,0,0,14,14,25,25,25 +new,deepseek-ai/DeepSeek-V4-Flash-0731,25,0,0,0,0,0,1,25,25 +new,deepseek-ai/DeepSeek-V4.1-Flash,24,0,0,0,0,0,0,23,23 +new,gemini-3.8-flash,24,5,5,4,7,3,24,8,3 +new,gemini-4-argon,25,0,0,0,24,24,25,25,25 +new,gpt-6-astra,25,0,0,0,25,25,25,25,25 +new,gpt-6-luna,25,0,0,0,0,0,0,25,25 +new,gpt-6-sol,25,0,0,0,8,8,22,25,25 +new,gpt-6.1-sol,25,0,0,0,23,23,25,25,25 +new,mistral-medium-3-5,25,0,0,0,0,0,0,25,25 +new,muse-spark-1.3,25,0,0,0,0,0,1,24,24 +new,muse-wonder-1.0,23,0,0,0,2,2,5,23,23 +new,nvidia/GLM-5.2-NVFP4-262K,25,0,0,0,0,0,13,24,24 +new,nvidia/Kimi-K2.6-NVFP4,25,0,0,0,0,0,0,1,1 +new,nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4,25,0,0,0,0,0,0,25,25 +new,zai-org/GLM-5.3-Flash,25,6,6,2,4,2,21,16,10 +old,Qwen/Qwen3.5-397B-A17B-FP8,25,0,0,0,0,0,0,5,5 +old,Qwen/Qwen3.5-9B,25,0,0,0,0,0,0,24,24 +old,Qwen/Qwen3.6-35B-A3B-FP8,25,0,0,0,0,0,0,8,8 +old,Qwen/Qwen3.8-27B-FP8,25,0,0,0,0,0,0,18,18 +old,claude-fable-5-1,25,0,0,0,19,19,25,25,25 +old,claude-opus-5-5,25,0,0,0,8,8,25,25,25 +old,deepseek-ai/DeepSeek-V4-Flash-0731,25,0,0,0,0,0,0,25,25 +old,deepseek-ai/DeepSeek-V4.1-Flash,25,0,0,0,0,0,0,25,25 +old,gemini-3.8-flash,24,5,5,4,10,6,24,11,6 +old,gemini-4-argon,25,0,0,0,25,25,25,25,25 +old,gpt-6-astra,25,0,0,0,25,25,25,25,25 +old,gpt-6-luna,25,0,0,0,0,0,0,25,25 +old,gpt-6-sol,25,0,0,0,8,8,23,25,25 +old,gpt-6.1-sol,25,0,0,0,24,24,25,25,25 +old,mistral-medium-3-5,25,0,0,0,0,0,0,25,25 +old,muse-spark-1.3,25,0,0,0,0,0,1,24,24 +old,muse-wonder-1.0,25,0,0,0,1,1,10,25,25 +old,nvidia/GLM-5.2-NVFP4-262K,25,0,0,0,0,0,17,24,24 +old,nvidia/Kimi-K2.6-NVFP4,24,0,0,0,0,0,0,0,0 +old,nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4,25,0,0,0,0,0,0,25,25 +old,zai-org/GLM-5.3-Flash,25,7,7,1,1,0,21,10,3 diff --git a/docs/studies/2026-10-goal-strict.svg b/docs/studies/2026-10-goal-strict.svg new file mode 100644 index 0000000..e7113b4 --- /dev/null +++ b/docs/studies/2026-10-goal-strict.svg @@ -0,0 +1,70 @@ + +Latest goal and original strict ratesTwo bars per model; goal above original strict. Exact fractions are in the adjacent report table. Scale is zero to 100 percent. + +Latest goal and original strict rates +Goal (upper bar) +Original strict (lower bar) +gpt-6-astra +43/43 +42/43 +gemini-4-argon +44/44 +42/44 +gpt-6.1-sol +44/44 +42/44 +claude-fable-5-1 +44/44 +35/44 +claude-opus-5-5 +44/44 +26/44 +gpt-6-sol +40/44 +20/44 +gemini-3.8-flash +43/43 +19/43 +muse-wonder-1.0 +20/40 +10/40 +zai-org/GLM-5.3-Flash +35/44 +9/44 +nvidia/GLM-5.2-NVFP4-262K +29/44 +6/44 +muse-spark-1.3 +19/44 +5/44 +deepseek-ai/DeepSeek-V4.1-Flash +7/43 +2/43 +Qwen/Qwen3.8-27B-FP8 +2/44 +2/44 +deepseek-ai/DeepSeek-V4-Flash-0731 +3/44 +1/44 +gpt-6-luna +9/44 +0/44 +nvidia/Kimi-K2.6-NVFP4 +7/44 +0/44 +Qwen/Qwen3.6-35B-A3B-FP8 +7/44 +0/44 +Qwen/Qwen3.5-397B-A17B-FP8 +3/44 +0/44 +mistral-medium-3-5 +0/44 +0/44 +nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 +0/44 +0/44 +Qwen/Qwen3.5-9B +0/44 +0/44 + diff --git a/docs/studies/2026-10-model-flows.csv b/docs/studies/2026-10-model-flows.csv new file mode 100644 index 0000000..7419ceb --- /dev/null +++ b/docs/studies/2026-10-model-flows.csv @@ -0,0 +1,64 @@ +model,flow,common_goal_new,common_goal_old,common_scored,common_strict_new,common_strict_old,goal_gained,goal_lost,goal_retained,goal_stable_fail,new_excluded,new_goals,new_scored,new_strict,old_aborted,old_excluded,old_goals,old_scored,old_strict,pairs,strict_gained,strict_lost,strict_retained,strict_stable_fail +Qwen/Qwen3.5-397B-A17B-FP8,commerce,3,3,16,0,0,0,0,3,13,0,3,16,0,0,0,3,16,0,16,0,0,0,16 +Qwen/Qwen3.5-397B-A17B-FP8,compliance,0,0,25,0,0,0,0,0,25,0,0,25,0,0,0,0,25,0,25,0,0,0,25 +Qwen/Qwen3.5-397B-A17B-FP8,maintenance,0,0,3,0,0,0,0,0,3,0,0,3,0,0,0,0,3,0,3,0,0,0,3 +Qwen/Qwen3.5-9B,commerce,0,0,16,0,0,0,0,0,16,0,0,16,0,0,0,0,16,0,16,0,0,0,16 +Qwen/Qwen3.5-9B,compliance,0,0,25,0,0,0,0,0,25,0,0,25,0,0,0,0,25,0,25,0,0,0,25 +Qwen/Qwen3.5-9B,maintenance,0,0,3,0,0,0,0,0,3,0,0,3,0,0,0,0,3,0,3,0,0,0,3 +Qwen/Qwen3.6-35B-A3B-FP8,commerce,7,8,16,0,0,2,3,5,6,0,7,16,0,0,0,8,16,0,16,0,0,0,16 +Qwen/Qwen3.6-35B-A3B-FP8,compliance,0,0,25,0,0,0,0,0,25,0,0,25,0,0,0,0,25,0,25,0,0,0,25 +Qwen/Qwen3.6-35B-A3B-FP8,maintenance,0,0,3,0,0,0,0,0,3,0,0,3,0,0,0,0,3,0,3,0,0,0,3 +Qwen/Qwen3.8-27B-FP8,commerce,2,1,16,2,1,1,0,1,14,0,2,16,2,0,0,1,16,1,16,1,0,1,14 +Qwen/Qwen3.8-27B-FP8,compliance,0,0,25,0,0,0,0,0,25,0,0,25,0,0,0,0,25,0,25,0,0,0,25 +Qwen/Qwen3.8-27B-FP8,maintenance,0,0,3,0,0,0,0,0,3,0,0,3,0,0,0,0,3,0,3,0,0,0,3 +claude-fable-5-1,commerce,15,15,15,15,15,0,0,15,0,0,16,16,16,0,1,15,15,15,16,0,0,15,0 +claude-fable-5-1,compliance,25,25,25,19,19,0,0,25,0,0,25,25,19,0,0,25,25,19,25,1,1,18,5 +claude-fable-5-1,maintenance,3,3,3,0,1,0,0,3,0,0,3,3,0,0,0,3,3,1,3,0,1,0,2 +claude-opus-5-5,commerce,16,16,16,11,13,0,0,16,0,0,16,16,11,0,0,16,16,13,16,1,3,10,2 +claude-opus-5-5,compliance,25,25,25,14,8,0,0,25,0,0,25,25,14,0,0,25,25,8,25,6,0,8,11 +claude-opus-5-5,maintenance,3,2,3,1,1,1,0,2,0,0,3,3,1,0,0,2,3,1,3,0,0,1,2 +deepseek-ai/DeepSeek-V4-Flash-0731,commerce,2,2,16,1,1,0,0,2,14,0,2,16,1,0,0,2,16,1,16,0,0,1,15 +deepseek-ai/DeepSeek-V4-Flash-0731,compliance,1,0,25,0,0,1,0,0,24,0,1,25,0,0,0,0,25,0,25,0,0,0,25 +deepseek-ai/DeepSeek-V4-Flash-0731,maintenance,0,0,3,0,0,0,0,0,3,0,0,3,0,0,0,0,3,0,3,0,0,0,3 +deepseek-ai/DeepSeek-V4.1-Flash,commerce,7,9,16,2,0,0,2,7,7,0,7,16,2,0,0,9,16,0,16,2,0,0,14 +deepseek-ai/DeepSeek-V4.1-Flash,compliance,0,0,24,0,0,0,0,0,24,1,0,24,0,0,0,0,25,0,25,0,0,0,24 +deepseek-ai/DeepSeek-V4.1-Flash,maintenance,0,0,3,0,0,0,0,0,3,0,0,3,0,0,0,0,3,0,3,0,0,0,3 +gemini-3.8-flash,commerce,16,16,16,12,10,0,0,16,0,0,16,16,12,0,0,16,16,10,16,3,1,9,3 +gemini-3.8-flash,compliance,23,23,23,7,10,0,0,23,0,1,24,24,7,0,1,24,24,10,25,1,4,6,12 +gemini-3.8-flash,maintenance,3,3,3,0,0,0,0,3,0,0,3,3,0,0,0,3,3,0,3,0,0,0,3 +gemini-4-argon,commerce,16,16,16,15,15,0,0,16,0,0,16,16,15,0,0,16,16,15,16,0,0,15,1 +gemini-4-argon,compliance,25,25,25,24,25,0,0,25,0,0,25,25,24,0,0,25,25,25,25,0,1,24,0 +gemini-4-argon,maintenance,3,3,3,3,2,0,0,3,0,0,3,3,3,0,0,3,3,2,3,1,0,2,0 +gpt-6-astra,commerce,16,16,16,15,14,0,0,16,0,0,16,16,15,0,0,16,16,14,16,1,0,14,1 +gpt-6-astra,compliance,25,25,25,25,25,0,0,25,0,0,25,25,25,0,0,25,25,25,25,0,0,25,0 +gpt-6-astra,maintenance,2,2,2,2,2,0,0,2,0,1,2,2,2,0,0,3,3,3,3,0,0,2,0 +gpt-6-luna,commerce,9,9,15,0,0,1,1,8,5,0,9,16,0,1,0,9,15,0,16,0,0,0,15 +gpt-6-luna,compliance,0,0,25,0,0,0,0,0,25,0,0,25,0,0,0,0,25,0,25,0,0,0,25 +gpt-6-luna,maintenance,0,0,0,0,0,0,0,0,0,0,0,3,0,3,0,0,0,0,3,0,0,0,0 +gpt-6-sol,commerce,16,14,16,11,10,2,0,14,0,0,16,16,11,0,0,14,16,10,16,2,1,9,4 +gpt-6-sol,compliance,22,23,25,8,8,2,3,20,0,0,22,25,8,0,0,23,25,8,25,5,5,3,12 +gpt-6-sol,maintenance,2,2,3,1,1,0,0,2,1,0,2,3,1,0,0,2,3,1,3,0,0,1,2 +gpt-6.1-sol,commerce,16,16,16,16,16,0,0,16,0,0,16,16,16,0,0,16,16,16,16,0,0,16,0 +gpt-6.1-sol,compliance,25,25,25,23,24,0,0,25,0,0,25,25,23,0,0,25,25,24,25,1,2,22,0 +gpt-6.1-sol,maintenance,3,3,3,3,2,0,0,3,0,0,3,3,3,0,0,3,3,2,3,1,0,2,0 +mistral-medium-3-5,commerce,0,0,16,0,0,0,0,0,16,0,0,16,0,0,0,0,16,0,16,0,0,0,16 +mistral-medium-3-5,compliance,0,0,25,0,0,0,0,0,25,0,0,25,0,0,0,0,25,0,25,0,0,0,25 +mistral-medium-3-5,maintenance,0,0,3,0,0,0,0,0,3,0,0,3,0,0,0,0,3,0,3,0,0,0,3 +muse-spark-1.3,commerce,16,16,16,4,4,0,0,16,0,0,16,16,4,0,0,16,16,4,16,3,3,1,9 +muse-spark-1.3,compliance,1,1,25,0,0,1,1,0,23,0,1,25,0,0,0,1,25,0,25,0,0,0,25 +muse-spark-1.3,maintenance,2,1,3,1,0,1,0,1,1,0,2,3,1,0,0,1,3,0,3,1,0,0,2 +muse-wonder-1.0,commerce,14,15,16,8,10,1,2,13,0,0,14,16,8,0,0,15,16,10,16,1,3,7,5 +muse-wonder-1.0,compliance,5,9,23,2,1,0,4,5,14,2,5,23,2,0,0,10,25,1,25,1,0,1,21 +muse-wonder-1.0,maintenance,1,1,1,0,0,0,0,1,0,2,1,1,0,0,1,2,2,1,3,0,0,0,1 +nvidia/GLM-5.2-NVFP4-262K,commerce,16,16,16,6,6,0,0,16,0,0,16,16,6,0,0,16,16,6,16,2,2,4,8 +nvidia/GLM-5.2-NVFP4-262K,compliance,13,17,25,0,0,1,5,12,7,0,13,25,0,0,0,17,25,0,25,0,0,0,25 +nvidia/GLM-5.2-NVFP4-262K,maintenance,0,1,3,0,0,0,1,0,2,0,0,3,0,0,0,1,3,0,3,0,0,0,3 +nvidia/Kimi-K2.6-NVFP4,commerce,7,7,16,0,0,0,0,7,9,0,7,16,0,0,0,7,16,0,16,0,0,0,16 +nvidia/Kimi-K2.6-NVFP4,compliance,0,0,24,0,0,0,0,0,24,0,0,25,0,0,1,0,24,0,25,0,0,0,24 +nvidia/Kimi-K2.6-NVFP4,maintenance,0,0,3,0,0,0,0,0,3,0,0,3,0,0,0,0,3,0,3,0,0,0,3 +nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4,commerce,0,0,16,0,0,0,0,0,16,0,0,16,0,0,0,0,16,0,16,0,0,0,16 +nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4,compliance,0,0,25,0,0,0,0,0,25,0,0,25,0,0,0,0,25,0,25,0,0,0,25 +nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4,maintenance,0,0,3,0,0,0,0,0,3,0,0,3,0,0,0,0,3,0,3,0,0,0,3 +zai-org/GLM-5.3-Flash,commerce,14,11,16,5,3,3,0,11,2,0,14,16,5,0,0,11,16,3,16,2,0,3,11 +zai-org/GLM-5.3-Flash,compliance,21,21,25,4,1,2,2,19,2,0,21,25,4,0,0,21,25,1,25,3,0,1,21 +zai-org/GLM-5.3-Flash,maintenance,0,1,3,0,0,0,1,0,2,0,0,3,0,0,0,1,3,0,3,0,0,0,3 diff --git a/docs/studies/2026-10-models.csv b/docs/studies/2026-10-models.csv new file mode 100644 index 0000000..61413ff --- /dev/null +++ b/docs/studies/2026-10-models.csv @@ -0,0 +1,22 @@ +model,common_goal_new,common_goal_old,common_scored,common_strict_new,common_strict_old,goal_gained,goal_lost,goal_retained,goal_stable_fail,new_excluded,new_goals,new_scored,new_strict,old_aborted,old_excluded,old_goals,old_scored,old_strict,pairs,strict_gained,strict_lost,strict_retained,strict_stable_fail,latest_scored_decisions,latest_cost_usd +Qwen/Qwen3.5-397B-A17B-FP8,3,3,44,0,0,0,0,3,41,0,3,44,0,0,0,3,44,0,44,0,0,0,44,1352, +Qwen/Qwen3.5-9B,0,0,44,0,0,0,0,0,44,0,0,44,0,0,0,0,44,0,44,0,0,0,44,1226, +Qwen/Qwen3.6-35B-A3B-FP8,7,8,44,0,0,2,3,5,34,0,7,44,0,0,0,8,44,0,44,0,0,0,44,1845, +Qwen/Qwen3.8-27B-FP8,2,1,44,2,1,1,0,1,42,0,2,44,2,0,0,1,44,1,44,1,0,1,42,2608, +claude-fable-5-1,43,43,43,34,35,0,0,43,0,0,44,44,35,0,1,43,43,35,44,1,2,33,7,853, +claude-opus-5-5,44,43,44,26,22,1,0,43,0,0,44,44,26,0,0,43,44,22,44,7,3,19,15,845, +deepseek-ai/DeepSeek-V4-Flash-0731,3,2,44,1,1,1,0,2,41,0,3,44,1,0,0,2,44,1,44,0,0,1,43,2523, +deepseek-ai/DeepSeek-V4.1-Flash,7,9,43,2,0,0,2,7,34,1,7,43,2,0,0,9,44,0,44,2,0,0,41,2930, +gemini-3.8-flash,42,42,42,19,20,0,0,42,0,1,43,43,19,0,1,43,43,20,44,4,5,15,18,697, +gemini-4-argon,44,44,44,42,42,0,0,44,0,0,44,44,42,0,0,44,44,42,44,1,1,41,1,757, +gpt-6-astra,43,43,43,42,41,0,0,43,0,1,43,43,42,0,0,44,44,42,44,1,0,41,1,804, +gpt-6-luna,9,9,40,0,0,1,1,8,30,0,9,44,0,4,0,9,40,0,44,0,0,0,40,2432, +gpt-6-sol,40,39,44,20,19,4,3,36,1,0,40,44,20,0,0,39,44,19,44,7,6,13,18,1187, +gpt-6.1-sol,44,44,44,42,42,0,0,44,0,0,44,44,42,0,0,44,44,42,44,2,2,40,0,829, +mistral-medium-3-5,0,0,44,0,0,0,0,0,44,0,0,44,0,0,0,0,44,0,44,0,0,0,44,2403, +muse-spark-1.3,19,18,44,5,4,2,1,17,24,0,19,44,5,0,0,18,44,4,44,4,3,1,36,1119, +muse-wonder-1.0,20,25,40,10,11,1,6,19,14,4,20,40,10,0,1,27,43,12,44,2,3,8,27,901, +nvidia/GLM-5.2-NVFP4-262K,29,34,44,6,6,1,6,28,9,0,29,44,6,0,0,34,44,6,44,2,2,4,36,1796, +nvidia/Kimi-K2.6-NVFP4,7,7,43,0,0,0,0,7,36,0,7,44,0,0,1,7,43,0,44,0,0,0,43,1190, +nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4,0,0,44,0,0,0,0,0,44,0,0,44,0,0,0,0,44,0,44,0,0,0,44,2246, +zai-org/GLM-5.3-Flash,35,33,44,9,4,5,3,30,6,0,35,44,9,0,0,33,44,4,44,5,0,4,35,1316, diff --git a/docs/studies/2026-10-observations.md b/docs/studies/2026-10-observations.md new file mode 100644 index 0000000..5b27be5 --- /dev/null +++ b/docs/studies/2026-10-observations.md @@ -0,0 +1,104 @@ +# Achieving the outcome is not the same as running the operation well + +An agent can finish the intended workflow while making a decision that should have been avoided. In these exploratory synthetic observations, the latest series records 443 goal achievements among 917 scored cases, but only 261 original strict passes. “Strict” means passing all reported quality checks, not simply reaching the goal. The difference makes the trajectory worth examining alongside the outcome. It does not establish that every failed check caused harm, or that every stored pass was free of refused actions. + +This study describes 21 exact model IDs across two owner-run series. Each model was assigned 44 related synthetic cases: three Maintenance, 16 Commerce and 25 Compliance. There was one trial per case per series. Historical and repeat observations used different evaluator/runtime versions, adapters and settings. They are not a same-condition statistical replication, evidence of model learning, or a validated capability ranking. The repeat is dated 2026-10-24; historical observations span source cohorts. + +## Outcomes and original reported checks + +The historical series has 916 scored cases, four excluded and four aborted; the repeat has 917 scored and seven excluded, out of 924 assignments each. Historical goals and strict passes were 451 and 250. These totals describe each series separately. Exclusions and aborts receive no inferred grade. All rates use scored denominators; costs are not published (null), never zero. + +Original grades are displayed. The known Compliance terminal read-guard omission and the separate diagnostic measured below affect some stored strict passes. + +![Latest goal and original strict fractions for all 21 models; exact values follow in the table.](2026-10-goal-strict.svg) + +*Figure 1. Upper blue bars show goal achievement, lower brown bars original strict passes, on a 0–100% scale. The table is the full text alternative. These are original grades, before the separate Compliance diagnostic.* + +The table sorts by exact latest strict/scored fraction descending, then goal/scored descending, then casefolded and exact ID. This is a reading order, not an official ranking. Historical cells give goal count and strict count with their shared scored denominator. Latest decision counts cover scored Core decisions only, not generation RPCs or all attempted calls; they do not measure cost or efficiency. Full historical dispositions are in the aggregate CSV. + +| Exact model ID | Historical goal; strict / scored | Latest goal / scored | Latest original strict / scored | Latest excluded | Latest scored Core decisions | +|---|---:|---:|---:|---:|---:| +| `gpt-6-astra` | 44; 42 / 44 | 43/43 | 42/43 | 1 | 804 | +| `gemini-4-argon` | 44; 42 / 44 | 44/44 | 42/44 | 0 | 757 | +| `gpt-6.1-sol` | 44; 42 / 44 | 44/44 | 42/44 | 0 | 829 | +| `claude-fable-5-1` | 43; 35 / 43 | 44/44 | 35/44 | 0 | 853 | +| `claude-opus-5-5` | 43; 22 / 44 | 44/44 | 26/44 | 0 | 845 | +| `gpt-6-sol` | 39; 19 / 44 | 40/44 | 20/44 | 0 | 1187 | +| `gemini-3.8-flash` | 43; 20 / 43 | 43/43 | 19/43 | 1 | 697 | +| `muse-wonder-1.0` | 27; 12 / 43 | 20/40 | 10/40 | 4 | 901 | +| `zai-org/GLM-5.3-Flash` | 33; 4 / 44 | 35/44 | 9/44 | 0 | 1316 | +| `nvidia/GLM-5.2-NVFP4-262K` | 34; 6 / 44 | 29/44 | 6/44 | 0 | 1796 | +| `muse-spark-1.3` | 18; 4 / 44 | 19/44 | 5/44 | 0 | 1119 | +| `deepseek-ai/DeepSeek-V4.1-Flash` | 9; 0 / 44 | 7/43 | 2/43 | 1 | 2930 | +| `Qwen/Qwen3.8-27B-FP8` | 1; 1 / 44 | 2/44 | 2/44 | 0 | 2608 | +| `deepseek-ai/DeepSeek-V4-Flash-0731` | 2; 1 / 44 | 3/44 | 1/44 | 0 | 2523 | +| `gpt-6-luna` | 9; 0 / 40 | 9/44 | 0/44 | 0 | 2432 | +| `nvidia/Kimi-K2.6-NVFP4` | 7; 0 / 43 | 7/44 | 0/44 | 0 | 1190 | +| `Qwen/Qwen3.6-35B-A3B-FP8` | 8; 0 / 44 | 7/44 | 0/44 | 0 | 1845 | +| `Qwen/Qwen3.5-397B-A17B-FP8` | 3; 0 / 44 | 3/44 | 0/44 | 0 | 1352 | +| `mistral-medium-3-5` | 0; 0 / 44 | 0/44 | 0/44 | 0 | 2403 | +| `nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4` | 0; 0 / 44 | 0/44 | 0/44 | 0 | 2246 | +| `Qwen/Qwen3.5-9B` | 0; 0 / 44 | 0/44 | 0/44 | 0 | 1226 | + +A goal count alone also hides variation within a row. For example, `gpt-6-luna` reaches nine goals out of 44 scored cases in the repeat, yet has no original strict passes. That does not tell us which check prevented each pass, whether an action was blocked, or whether recovery occurred. Those questions require examining trajectories and evaluator semantics. Conversely, a high strict fraction describes these assigned cases under their reported checks; it supplies no estimate for a different workload, deployment environment or number of repeated attempts. + +## Equal scores can conceal different passed cases + +Among the 910 model–case pairs scored in both series, 66 changed strict status: 39 gained a pass and 27 lost one. In 61 of those 66, the goal was true in both series. The other five comprise three goal losses and two goal gains. Thus 61 strict-status changes had no goal-status change. These counts alone do not distinguish agent behavior from evaluator, runtime, adapter or settings differences, identify blocked proposals, or prove harm. The inspected semantic examples below provide separate evidence. + +`gemini-4-argon` stayed at 42 total strict passes with one gained case and one lost case (41 retained cases); `gpt-6.1-sol` stayed at 42 with two gains and two losses (40 retained cases). `gpt-6-sol` moved from 19 to 20 with seven gains and six losses. Those are common-scored comparisons, not pooled episodes or repeated estimates of a stable underlying success probability. A small net change can hide substantial case churn. + +![Strict gains and losses by flow: Maintenance 3 and 1; Commerce 18 and 13; Compliance 18 and 13.](2026-10-case-churn.svg) + +*Figure 2. Counts on common-scored pairs only: 39 gains and 27 losses. The following flow table supplies denominators. These are descriptive transitions, without confidence intervals or significance claims.* + +| Flow (cases/model) | Historical goal; strict / scored | Repeat goal; strict / scored | Historical excluded; aborted | Repeat excluded | Common scored | Strict gained; lost | +|---|---:|---:|---:|---:|---:|---:| +| Maintenance (3) | 24; 11 / 59 | 22; 11 / 60 | 1; 3 | 3 | 57 | 3; 1 | +| Commerce (16) | 206; 118 / 334 | 209; 124 / 336 | 1; 1 | 0 | 334 | 18; 13 | +| Compliance (25) | 221; 121 / 523 | 212; 126 / 521 | 2; 0 | 4 | 519 | 18; 13 | + +The case mix matters: Compliance contributes 25 of 44 assignments per model, Commerce 16 and Maintenance three. Aggregate rates therefore reflect these weights, rather than an equal balance of flows. The public 63-row model-by-flow CSV makes individual flow differences inspectable without treating related variants as independent tasks. + +## What the trajectories illustrate + +The following semantic summaries come from AI-assisted maintainer review of owner-retained traces, not independent human semantic validation. Raw trajectories remain private; no provider wording or trace-level proof is supplied here. + +For Opus in Commerce UK_NORMAL, both series complete a single synthetic GBP105 refund. The historical case passed strict checks; the repeat did not. The repeat resubmits an already accepted `decide_entitlement`, receives `DUPLICATE_ENTITLEMENT`, corrects the mistake and completes. That rejected duplicate decision is not a duplicate payment. The example separates a recovered operational error from the successful refund. It cannot by itself establish real-world financial harm. + +For Argon in Maintenance V1, a historical premature COMPLETE attempt is refused while the simulated customer still reports persistence. Both versions ultimately complete the intended workflow, and the repeat avoids that attempt. In the repeat AU_remediation case, however, a send payload missing a recipient is rejected before recovery. This helps explain how 42→42 can contain changed faults. It is a synthetic workflow, not an actual customer interaction or certification. + +For GLM5.3 in Compliance SC_qualification and NZ_qualification, saved repeat strict passes include a stale COMPLETE refusal, a fresh reread and legitimate referral completion. Runtime protection worked, but the original evaluator omitted the terminal read-guard fault. The diagnostic below records a retrieval failure. A referral does not certify property compliance. This limits interpretation of the earlier full-cohort original reported strict-count increase from 4→9; it does not show that all study results are invalid. + +## Separate Compliance correction diagnostic + +The offline diagnostic covers all 1,044 scored Compliance records, with original-baseline parity on all 1,044. There were 35 stale terminal attempts (18 historical, 17 repeat). Findings and retrieval change on 23 records (12 historical, 11 repeat); 11 stored strict passes become diagnostic strict failures (five historical, six repeat). Goals and the other ten dimensions remain unchanged. Attempts, changed records and changed passes are different units. + +| Series | Exact model ID | Original strict → diagnostic strict / Compliance scored | +|---|---|---:| +| Repeat | `gemini-3.8-flash` | 7 → 3 / 24 | +| Repeat | `zai-org/GLM-5.3-Flash` | 4 → 2 / 25 | +| Historical | `gemini-3.8-flash` | 10 → 6 / 24 | +| Historical | `zai-org/GLM-5.3-Flash` | 1 → 0 / 25 | + +These four rows use **Compliance denominators**, not full 44-case corrected model totals. Across Compliance, original strict counts of 121/523 and 126/521 become diagnostic 116/523 and 120/521. Headline grades above remain original. Other flows were not reevaluated. This is an offline saved-record diagnostic, not authenticated replay, an official regrade or a new model run. + +The correction is a separate [candidate in PR #37](https://github.com/DwellyOrg/OperateBench/pull/37), not a claim of merged main behavior. Candidate evaluator version: 0.3.0; evaluator source SHA-256: `4899ba74ff12e798306b9b85492444272b36f2d03254a1ff3ca6be6c9654bee9`. Its terminal handling change is bounded; the unmapped terminal-claim code policy is unchanged. + +## Methodology, reproducibility and limits + +[Aggregate JSON](2026-10-aggregates.json), [21-model CSV](2026-10-models.csv), [63 model–flow rows](2026-10-model-flows.csv) and [42 Compliance model–series diagnostic rows](2026-10-compliance-diagnostic.csv) support recomputing the fractions, order, flow totals and charts. CSV empty cost fields represent JSON null. JSON `old` means historical; `new` means repeat. `common_*` fields restrict comparisons to cases scored in both series; `strict_gained` and `strict_lost` count paired transitions. Diagnostic `reliable` fields mean strict passes, not goal completions. The JSON cross-tab supplies goal status within strict flips. + +Public code and the correction PR support inspection of evaluator logic and deterministic reference/negative controls. Existing offline Maintenance commands, run from the repository with its dependencies installed, are: + +```sh +operatebench run --pack maintenance --spec examples/operatebench/maintenance_v0_1.yaml --scenario V1 --agent reference --output v1_reference.json +operatebench replay --pack maintenance --spec examples/operatebench/maintenance_v0_1.yaml --run v1_reference.json +operatebench check --pack maintenance --spec examples/operatebench/maintenance_v0_1.yaml +``` + +These exercise reference behavior and targeted negative controls; they do not reproduce the paid model study. Owner-retained raw records and trajectories are not released. An outsider cannot fully reproduce the historical study from aggregates alone. No independent human semantic validation or provider attestation is claimed. + +Official Lifecycle scope remains **one Maintenance Semantic Scenario with three variants**. Commerce and Compliance are SDK incubators, not additional admitted official tracks. Related synthetic cases do not establish legal validity, production safety or deployment performance. This report makes no official Lifecycle model ranking, novelty, competitor or superiority claim. The [preserved historical 19-model table](../MODEL_RESULTS.md) and [results preview](https://dwellyorg.github.io/operatebench-results-preview/) provide complementary source context. + +Content and aggregate data: © 2026 Prospire Technologies Ltd, CC BY 4.0; see [LICENSE-DATA](../../LICENSE-DATA). Original ownership and attribution are retained. diff --git a/tests/test_repository_governance.py b/tests/test_repository_governance.py index c760b05..3e64f15 100644 --- a/tests/test_repository_governance.py +++ b/tests/test_repository_governance.py @@ -270,10 +270,38 @@ def test_descriptive_exception_does_not_authorize_official_or_human_pass() -> No ) assert "Raw provider traces" in excluded["provider_run_artifacts"]["reason"] checklist = _flowed("PUBLIC_RELEASE_CHECKLIST.md") - assert "Only the independently labelled, owner-directed retrospective" in checklist + assert "The independently labelled, owner-directed retrospective" in checklist assert "This exception does not close G8, G11 or G12" in checklist assert "Apart from the specific descriptive-table exception in section 6" in checklist assert ( "Any leaderboard, ranking, composite capability score, or comparison" not in checklist ) + + +def test_observation_study_exception_has_exact_bounded_artifacts() -> None: + manifest = json.loads((ROOT / "PUBLICATION_MANIFEST.json").read_text()) + report = manifest["declarations"]["descriptive_model_results"] + statement = report["statement"] + expected = { + "docs/studies/2026-10-observations.md", + "docs/studies/2026-10-aggregates.json", + "docs/studies/2026-10-case-churn.svg", + "docs/studies/2026-10-goal-strict.svg", + "docs/studies/2026-10-models.csv", + "docs/studies/2026-10-model-flows.csv", + "docs/studies/2026-10-compliance-diagnostic.csv", + } + import re + + assert ( + set(re.findall(r"docs/studies/[\w-]+\.(?:md|csv|json|svg)", statement)) + == expected + ) + assert all((ROOT / path).is_file() for path in expected) + assert report["path"] == "docs/MODEL_RESULTS.md" + assert report["human_semantic_validation"] == "pending" + assert report["official_release_authorized"] is False + assert ( + "does not authorize future results, raw traces or billing disclosure" in statement + )