diff --git a/architecture/overview.md b/architecture/overview.md index 4230845..2d8d1a2 100644 --- a/architecture/overview.md +++ b/architecture/overview.md @@ -184,7 +184,7 @@ Ordered newest first. See [roadmap.md](../roadmap.md) for the canonical list of - Repeat-issue detection (prior resolution surfacing) - Proactive health alerts (webhooks on state transitions) - SLA tracking (response time, resolution time, breach flags) -- Proof layer: 3 eval suites (56 assertions), 2 benchmarks (8/8 vs 2/8 stateless) +- Proof layer: 3 eval suites (56 assertions), support-workflow benchmark (8 criteria, open harness) ### v0.5 — Reliability & Trust - True multi-tenant isolation (tenant_id on all tables, query-scoped) diff --git a/roadmap.md b/roadmap.md index 7dd21ca..1af6d4e 100644 --- a/roadmap.md +++ b/roadmap.md @@ -62,7 +62,7 @@ Statewave is purpose-built for **support-agent workflows** — the first use cas - [x] SLA tracking (response time, resolution time, breach flags) - [x] SLA integration into health + handoff - [x] Product website (statewave.ai) -- [x] Proof layer: 3 eval suites (56 assertions), 2 benchmarks (8/8 vs 2/8) +- [x] Proof layer: 3 eval suites (56 assertions), support-workflow benchmark (8 criteria, open harness) --- diff --git a/tools/_proof_figures.py b/tools/_proof_figures.py index fbc75ba..6cd03ac 100644 --- a/tools/_proof_figures.py +++ b/tools/_proof_figures.py @@ -118,15 +118,15 @@ "source": "grep -cE 'assert ' statewave-examples/eval-support-agent/test_support_advanced.py", "note": "Advanced eval assertions.", }, - "benchmark_statewave": { - "value": "8/8", - "source": "cat statewave-examples/benchmark-support-agent/results/*.json # Statewave score", - "note": "Statewave score on the support-workflow benchmark.", - }, - "benchmark_naive": { - "value": "2/8", - "source": "cat statewave-examples/benchmark-support-agent/results/*.json # naive score", - "note": "Naive concatenation baseline on the same benchmark.", + # The self-scored 8/8-vs-2/8 benchmark COMPARISON was retired from every + # living surface (statewave-web #293 + the docs PR that changed this + # file): scoring your own harness against a strawman is not a + # trustworthy proof figure. The criteria COUNT remains a fact worth + # mirroring — the harness is open, readers score it themselves. + "support_criteria": { + "value": "8", + "source": "ls statewave-examples/benchmark-support-agent/criteria # or the harness README criteria list", + "note": "Number of support-workflow benchmark criteria (open harness).", }, } @@ -187,17 +187,10 @@ "mechanical", ), ( - "overview_bench_sw", + "overview_bench_criteria", "statewave-docs/architecture/overview.md", - r"2 benchmarks \((\d+/\d+) vs \d+/\d+", - "benchmark_statewave", - "mechanical", - ), - ( - "overview_bench_naive", - "statewave-docs/architecture/overview.md", - r"2 benchmarks \(\d+/\d+ vs (\d+/\d+)", - "benchmark_naive", + r"support-workflow benchmark \((\d+) criteria", + "support_criteria", "mechanical", ), # --- roadmap.md --- @@ -209,17 +202,10 @@ "mechanical", ), ( - "roadmap_bench_sw", - "statewave-docs/roadmap.md", - r"2 benchmarks \((\d+/\d+) vs \d+/\d+\)", - "benchmark_statewave", - "mechanical", - ), - ( - "roadmap_bench_naive", + "roadmap_bench_criteria", "statewave-docs/roadmap.md", - r"2 benchmarks \(\d+/\d+ vs (\d+/\d+)\)", - "benchmark_naive", + r"support-workflow benchmark \((\d+) criteria", + "support_criteria", "mechanical", ), # --- why-statewave.md: per-suite breakdown + benchmark table row --- @@ -266,17 +252,10 @@ "mechanical", ), ( - "why_bench_sw", + "why_bench_criteria", "statewave-docs/why-statewave.md", - r"Statewave (\d+/\d+) vs Naive \d+/\d+", - "benchmark_statewave", - "mechanical", - ), - ( - "why_bench_naive", - "statewave-docs/why-statewave.md", - r"Statewave \d+/\d+ vs Naive (\d+/\d+)", - "benchmark_naive", + r"Support-workflow benchmark: (\d+) criteria \(active-issue", + "support_criteria", "mechanical", ), # --- statewave/README.md (core) --- @@ -295,62 +274,32 @@ "mechanical", ), # --- statewave-web: proof-stats.ts SSoT for the marketing site --- + # statewave-web #271/#293 centralized the site's literals into + # PROOF_FIGURES and retired the self-scored benchmark tiles, so the + # mirrors grep the named constants instead of tile literals. ( "proofstats_unit", "statewave-web/src/lib/proof-stats.ts", - r"value: '(\d+)', label: 'Unit tests'", + r"unitTests: '(\d+)'", "server_tests_unit", "mechanical", ), ( "proofstats_assertions", "statewave-web/src/lib/proof-stats.ts", - r"value: '(\d+)', label: 'Eval assertions'", + r"evalAssertions: '(\d+)'", "eval_assertions", "mechanical", ), ( - "proofstats_bench_sw", + "proofstats_criteria", "statewave-web/src/lib/proof-stats.ts", - r"value: '(\d+/\d+)', label: 'Support workflow score'", - "benchmark_statewave", + r"supportCriteria: '(\d+)'", + "support_criteria", "mechanical", ), - ( - "proofstats_bench_naive", - "statewave-web/src/lib/proof-stats.ts", - r"value: '(\d+/\d+)', label: 'Naive approach score'", - "benchmark_naive", - "mechanical", - ), - # --- statewave-web: HomePage prose --- - ( - "homepage_bench_sw", - "statewave-web/src/pages/HomePage.tsx", - r"Statewave scores (\d+/\d+) on support workflow", - "benchmark_statewave", - "mechanical", - ), - ( - "homepage_bench_naive", - "statewave-web/src/pages/HomePage.tsx", - r"naive approaches score (\d+/\d+)", - "benchmark_naive", - "mechanical", - ), - # --- editorial: narrative prose a human owns; reported, never failed --- - ( - "blog_bench_sw", - "statewave-web/src/content/blog/persistent-memory-for-ai-support-agents.mdx", - r"agent scores (\d+/\d+) on the same dataset", - "benchmark_statewave", - "editorial", - ), - ( - "blog_bench_naive", - "statewave-web/src/content/blog/persistent-memory-for-ai-support-agents.mdx", - r"baseline \(concatenate the last N turns\) scores (\d+/\d+)", - "benchmark_naive", - "editorial", - ), + # HomePage score prose and the blog's self-scored comparison were removed + # with the retirement (web #271/#293); the homepage now quotes + # PROOF_FIGURES constants directly in JSX, so proof-stats.ts is the one + # greppable web surface and no separate HomePage/blog mirrors remain. ] diff --git a/why-statewave.md b/why-statewave.md index 1e2d0f3..bc37b04 100644 --- a/why-statewave.md +++ b/why-statewave.md @@ -101,7 +101,7 @@ These claims are backed by the [support-agent context quality eval](https://gith | Handoff carries provenance | Advanced eval: handoff provenance includes episode_ids and resolution_ids | | Proactive health alerts on degradation | Unit tests: webhook fired on healthy→watch, watch→at_risk, healthy→at_risk; no spam on unchanged | | Health recovery confirmation | Unit tests: `subject.health_improved` fired on at_risk→watch, watch→healthy, at_risk→healthy | -| Support workflow superiority vs naive | Workflow benchmark: Statewave 8/8 vs Naive 2/8 on active-issue, repeat-detection, health, provenance, resolution-ranking | +| Support workflow coverage | Support-workflow benchmark: 8 criteria (active-issue, repeat-detection, health, provenance, resolution-ranking) — the harness is open in statewave-examples, so read the criteria and run it rather than trusting a self-scored comparison | | SLA tracking with breach detection | Unit tests: first-response time, resolution time, breach flags, custom thresholds; integrated into health scoring and handoff | | SLA breaches degrade health score | Unit tests: sla_resolution_breaches and slow_first_response signals penalize health deterministically | | SLA context in handoff packs | Unit tests: breach flags and open-issue age appear in handoff when relevant, absent when clean |