From c15f79f9181f291e622eed039d41787872583eb9 Mon Sep 17 00:00:00 2001 From: Stelios Date: Sat, 13 Jun 2026 15:33:39 +0300 Subject: [PATCH] feat(demo): add a runnable 5-minute clean-gate demo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit examples/demo/ — a baseline card plus an honest candidate (ACCEPT) and an overfit candidate that games the public checks (contamination up, OOD down) and is REJECTed with reasons ood_regressed + dcr_increased. Includes baseline.json, candidate.json, candidate_overfit.json, the exact expected output, a redacted LIMITED_ROLLOUT assurance card, and a full walkthrough README (Run the demo / What this shows / What it does not / How to interpret the decision / links). README adds the 5-minute path; a CI test asserts ACCEPT exit 0 / REJECT exit 1. All synthetic/redacted; no private content. --- README.md | 52 +++++++- examples/demo/README.md | 113 ++++++++++++++++++ examples/demo/baseline.json | 14 +++ examples/demo/candidate.json | 14 +++ examples/demo/candidate_overfit.json | 14 +++ examples/demo/expected_output.txt | 35 ++++++ .../demo/sample_assurance_card_redacted.json | 14 +++ tests/test_examples_run.py | 18 +++ 8 files changed, 270 insertions(+), 4 deletions(-) create mode 100644 examples/demo/README.md create mode 100644 examples/demo/baseline.json create mode 100644 examples/demo/candidate.json create mode 100644 examples/demo/candidate_overfit.json create mode 100644 examples/demo/expected_output.txt create mode 100644 examples/demo/sample_assurance_card_redacted.json diff --git a/README.md b/README.md index 52f209f..bd11efc 100644 --- a/README.md +++ b/README.md @@ -5,15 +5,59 @@ Public-safe examples for the Verifiable Labs SDK and clean promotion gate. > Verifiable Labs builds clean feedback and promotion gates for increasingly general AI agents. Everything in this repository is **illustrative**: fake IDs, dummy-provider -outputs, and synthetic numbers. No real customer data, no hidden -evaluation content, no gold answers. +outputs, and synthetic numbers. No real customer data, no hidden evaluation +content, no gold answers, no raw traces, no private traps, no private engine +internals. + +## Run the 5-minute demo + +```bash +pip install "vlabs-sdk==0.0.2" +vlabs --help + +# An honest candidate that genuinely improves clean generalization → ACCEPT (exit 0) +vlabs clean-gate --old examples/demo/baseline.json --new examples/demo/candidate.json + +# A candidate that games the public checks (contamination up, OOD down) → REJECT (exit 1) +vlabs clean-gate --old examples/demo/baseline.json --new examples/demo/candidate_overfit.json +``` + +The candidate with the **highest public score** is the one the gate **rejects** — +because a higher visible score is not a promotion. The full walkthrough (what it +shows, what it does not, how to read the decision) is in +[`examples/demo/README.md`](examples/demo/README.md). ## Contents -- [`examples/run_clean_gate_demo.md`](examples/run_clean_gate_demo.md) — - run the `clean-gate` CLI against sample metric cards. +- [`examples/demo/`](examples/demo/) — the 5-minute clean-gate demo: a baseline + card, an honest candidate (ACCEPT), an overfit candidate (REJECT), the exact + expected output, and a redacted `LIMITED_ROLLOUT` assurance card. +- [`examples/run_clean_gate_demo.md`](examples/run_clean_gate_demo.md) — run the + `clean-gate` CLI against sample metric cards. - [`examples/sample_assurance_card.json`](examples/sample_assurance_card.json) — an illustrative assurance card showing the shape of a gate decision. +- [`examples/run_evaluate_only_dummy.py`](examples/run_evaluate_only_dummy.py), + [`improve_and_gate_mock.py`](examples/improve_and_gate_mock.py), + [`substrate_mock.py`](examples/substrate_mock.py) — synthetic, dummy-provider walkthroughs. + +## What this does not show + +Synthetic / redacted demo data only — **not** a training dataset. No customer +data, hidden evaluations, gold answers, raw traces, private anti-hack traps, or +private engine internals; those are never published. + +## Links + +- PyPI — +- Hugging Face evidence dataset — +- Weights & Biases report — +- Documentation — + +## Formal scope + +Selected mathematical properties behind the contamination-resistant promotion +gate are machine-verified in Lean 4. The implementation is property-tested +against the formal specification. ## License diff --git a/examples/demo/README.md b/examples/demo/README.md new file mode 100644 index 0000000..e8c21a5 --- /dev/null +++ b/examples/demo/README.md @@ -0,0 +1,113 @@ +# The 5-minute demo — clean promotion gate + +> Verifiable Labs builds clean feedback and promotion gates for increasingly general AI agents. + +Run a real contamination-resistant **promotion gate** on two agent checkpoints +in under five minutes. Everything here is **synthetic** — fake IDs and +illustrative numbers — so you can see exactly how the gate decides. + +## Run the 5-minute demo + +```bash +pip install "vlabs-sdk==0.0.2" +vlabs --help + +# 1) An honest candidate that genuinely improves CLEAN generalization → ACCEPT (exit 0) +vlabs clean-gate --old baseline.json --new candidate.json + +# 2) A candidate that games the visible/public checks — contamination up, +# out-of-distribution transfer down — → REJECT (exit 1) +vlabs clean-gate --old baseline.json --new candidate_overfit.json +``` + +The three eval cards live next to this file: [`baseline.json`](baseline.json), +[`candidate.json`](candidate.json), [`candidate_overfit.json`](candidate_overfit.json). +The full expected console output is in [`expected_output.txt`](expected_output.txt). + +## What this shows + +A baseline agent passes the **visible / public** checks (`public_score` 0.80) +but is weaker on **hidden / OOD** transfer (`hidden_score` 0.68, +`ood_score` 0.66). Two candidates then ask to be promoted: + +- **`candidate.json`** improves clean verified-generalization score + (`clean_vgs` 0.50 → 0.63) with no regression in contamination risk, hack + risk, calibration, OOD, cost, or latency → **ACCEPT**. +- **`candidate_overfit.json`** has the **highest public score** of all (0.92) + — but it got there by memorising the visible set: contamination risk jumps + (0.10 → 0.34) and OOD transfer drops (0.66 → 0.62). The gate **REJECT**s it + and names exactly why: `ood_regressed`, `dcr_increased`. + +That contrast is the whole point: **a higher public score is not a promotion.** +The clean gate only accepts a change that *truly generalizes*. + +### Expected output (REJECT case) + +``` +== REJECT == + +condition old new budget OK +-------------------------------- ---------- ---------- ---------- -- +clean_vgs >= +tau 0.5000 0.5500 0.0100 OK +hack_risk <= +eps_h 0.1000 0.1100 0.0200 OK +calibration >= -eps_c 0.9000 0.9000 0.0200 OK +ood_score >= -eps_o 0.6600 0.6200 0.0200 !! +dcr <= +eps_d 0.1000 0.3400 0.0200 !! +cost <= +eps_k 1.0000 1.0000 5.0000 OK +latency <= +eps_l 1.0000 1.0000 0.5000 OK +regression flag False False False OK + +Reasons: + - ood_regressed + - dcr_increased +``` + +## How to interpret the gate decision + +The CLI evaluates the 8-condition contamination-resistant clean promotion gate. +A candidate is promoted only when **every** condition holds: + +| Condition | Meaning | +|---|---| +| `clean_vgs >= +tau` | clean verified-generalization score improved by at least `tau` | +| `hack_risk <= +eps_h` | reward/verifier-gaming risk did not rise more than `eps_h` | +| `calibration >= -eps_c` | uncertainty calibration did not drop more than `eps_c` | +| `ood_score >= -eps_o` | out-of-distribution transfer did not drop more than `eps_o` | +| `dcr <= +eps_d` | data-contamination risk did not rise more than `eps_d` | +| `cost <= +eps_k`, `latency <= +eps_l` | cost / latency stayed within budget | +| `regression flag` | no flagged regression | + +The CLI prints **`ACCEPT`** (exit 0) or **`REJECT`** (exit 1) with the failing +reasons. The full platform gate can additionally return **`LIMITED_ROLLOUT`** — +a partial promotion when a change is a net improvement but carries a watch-item +(e.g. a soft OOD regression). See the redacted example in +[`sample_assurance_card_redacted.json`](sample_assurance_card_redacted.json), +which records a `LIMITED_ROLLOUT` decision with reason `ood_regressed`. + +`clean_score = raw * (1 - dcr)` — contamination directly discounts the score, +which is why a memorised public win cannot buy a promotion. + +## What this does NOT show + +- This is **synthetic / redacted** demo data — **not** a training dataset, and + not a measurement of any real agent or customer. +- It does **not** include hidden evaluations, gold answers, raw traces, + customer data, private anti-hack traps, private engine internals, or secrets. + Those are never published — that separation is what keeps the feedback clean. +- The CLI scores eval **cards** you provide; it does not run a model, call a + provider, or generate the hidden/OOD/adversarial scenarios (that is the + private evaluation platform). + +## Links + +- **PyPI** — +- **Hugging Face evidence dataset** — +- **Weights & Biases report** — +- **Documentation** — +- **SDK** — + +## Formal scope + +Selected mathematical properties behind the contamination-resistant promotion +gate are machine-verified in Lean 4. The implementation is property-tested +against the formal specification. diff --git a/examples/demo/baseline.json b/examples/demo/baseline.json new file mode 100644 index 0000000..e3cc7eb --- /dev/null +++ b/examples/demo/baseline.json @@ -0,0 +1,14 @@ +{ + "model_id": "refund-agent-baseline", + "vgs": 0.70, + "contamination_risk": 0.10, + "clean_vgs": 0.50, + "public_score": 0.80, + "hidden_score": 0.68, + "ood_score": 0.66, + "hack_risk": 0.10, + "calibration": 0.90, + "cost": 1.0, + "latency": 1.0, + "regression": false +} diff --git a/examples/demo/candidate.json b/examples/demo/candidate.json new file mode 100644 index 0000000..c57ca16 --- /dev/null +++ b/examples/demo/candidate.json @@ -0,0 +1,14 @@ +{ + "model_id": "refund-agent-candidate", + "vgs": 0.79, + "contamination_risk": 0.08, + "clean_vgs": 0.63, + "public_score": 0.86, + "hidden_score": 0.79, + "ood_score": 0.73, + "hack_risk": 0.08, + "calibration": 0.92, + "cost": 1.0, + "latency": 1.0, + "regression": false +} diff --git a/examples/demo/candidate_overfit.json b/examples/demo/candidate_overfit.json new file mode 100644 index 0000000..7297679 --- /dev/null +++ b/examples/demo/candidate_overfit.json @@ -0,0 +1,14 @@ +{ + "model_id": "refund-agent-candidate-overfit", + "vgs": 0.78, + "contamination_risk": 0.34, + "clean_vgs": 0.55, + "public_score": 0.92, + "hidden_score": 0.69, + "ood_score": 0.62, + "hack_risk": 0.11, + "calibration": 0.90, + "cost": 1.0, + "latency": 1.0, + "regression": false +} diff --git a/examples/demo/expected_output.txt b/examples/demo/expected_output.txt new file mode 100644 index 0000000..835f68e --- /dev/null +++ b/examples/demo/expected_output.txt @@ -0,0 +1,35 @@ +# Expected console output for the 5-minute demo + +## ACCEPT — vlabs clean-gate --old baseline.json --new candidate.json (exit 0) + +== ACCEPT == + +condition old new budget OK +-------------------------------- ---------- ---------- ---------- -- +clean_vgs >= +tau 0.5000 0.6300 0.0100 OK +hack_risk <= +eps_h 0.1000 0.0800 0.0200 OK +calibration >= -eps_c 0.9000 0.9200 0.0200 OK +ood_score >= -eps_o 0.6600 0.7300 0.0200 OK +dcr <= +eps_d 0.1000 0.0800 0.0200 OK +cost <= +eps_k 1.0000 1.0000 5.0000 OK +latency <= +eps_l 1.0000 1.0000 0.5000 OK +regression flag False False False OK + +## REJECT — vlabs clean-gate --old baseline.json --new candidate_overfit.json (exit 1) + +== REJECT == + +condition old new budget OK +-------------------------------- ---------- ---------- ---------- -- +clean_vgs >= +tau 0.5000 0.5500 0.0100 OK +hack_risk <= +eps_h 0.1000 0.1100 0.0200 OK +calibration >= -eps_c 0.9000 0.9000 0.0200 OK +ood_score >= -eps_o 0.6600 0.6200 0.0200 !! +dcr <= +eps_d 0.1000 0.3400 0.0200 !! +cost <= +eps_k 1.0000 1.0000 5.0000 OK +latency <= +eps_l 1.0000 1.0000 0.5000 OK +regression flag False False False OK + +Reasons: + - ood_regressed + - dcr_increased diff --git a/examples/demo/sample_assurance_card_redacted.json b/examples/demo/sample_assurance_card_redacted.json new file mode 100644 index 0000000..567a581 --- /dev/null +++ b/examples/demo/sample_assurance_card_redacted.json @@ -0,0 +1,14 @@ +{ + "_comment": "ILLUSTRATIVE EXAMPLE — synthetic numbers, fake IDs, fields redacted as they would be for a real customer. Not a real evaluation.", + "card_version": "v2", + "run_id": "run_redacted_xxxx", + "org": "REDACTED", + "agent": "REDACTED", + "scores": { "public": 0.77, "hidden": 0.70, "ood": 0.66, "adversarial": 0.58 }, + "contamination": { "dcr": 0.05 }, + "clean_vgs": { "baseline": 0.55, "candidate": 0.59 }, + "generalization_gap": 0.07, + "gate": { "outcome": "LIMITED_ROLLOUT", "reasons": ["ood_regressed"] }, + "redaction_status": "redacted_public_safe", + "formal_claim": "Selected mathematical properties behind the contamination-resistant promotion gate are machine-verified in Lean 4. The implementation is property-tested against the formal specification." +} diff --git a/tests/test_examples_run.py b/tests/test_examples_run.py index 212b43f..3b4258f 100644 --- a/tests/test_examples_run.py +++ b/tests/test_examples_run.py @@ -53,3 +53,21 @@ def test_clean_gate_cli_accept_and_reject() -> None: "--new", str(cards / "clean_reject_dcr.json")], capture_output=True, ) assert rej.returncode == 1 + + +def test_demo_clean_gate_accept_and_reject() -> None: + """The 5-minute demo fixtures must produce ACCEPT (0) and REJECT (1).""" + demo = ROOT / "examples" / "demo" + acc = subprocess.run( + ["vlabs", "clean-gate", "--old", str(demo / "baseline.json"), + "--new", str(demo / "candidate.json")], capture_output=True, text=True, + ) + assert acc.returncode == 0, acc.stdout + acc.stderr + assert "ACCEPT" in acc.stdout + rej = subprocess.run( + ["vlabs", "clean-gate", "--old", str(demo / "baseline.json"), + "--new", str(demo / "candidate_overfit.json")], capture_output=True, text=True, + ) + assert rej.returncode == 1, rej.stdout + rej.stderr + assert "REJECT" in rej.stdout + assert "ood_regressed" in rej.stdout and "dcr_increased" in rej.stdout