From 1c90e65d713ddfe540db9d9e95241c8adcbfece5 Mon Sep 17 00:00:00 2001 From: Stelios Date: Fri, 12 Jun 2026 22:38:04 +0300 Subject: [PATCH] feat: switch examples to vlabs_sdk imports and the vlabs CLI All examples and tests now import vlabs_sdk and invoke 'vlabs clean-gate'; CI installs vlabs-sdk from its own repo. Drops the superseded quickstart doc in favour of run_clean_gate_demo.md. --- .github/workflows/ci.yml | 6 +++--- PROVENANCE.md | 2 +- README.md | 2 +- examples/improve_and_gate_mock.py | 2 +- examples/quickstart_clean_gate.md | 27 --------------------------- examples/run_clean_gate_demo.md | 8 +++++--- examples/run_evaluate_only_dummy.py | 8 ++++---- examples/sample_contract.json | 2 +- tests/test_examples_run.py | 4 ++-- 9 files changed, 18 insertions(+), 43 deletions(-) delete mode 100644 examples/quickstart_clean_gate.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d76cc06..412ad28 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,7 +11,7 @@ jobs: with: { python-version: "3.12" } - name: install public SDK surface (dummy provider only) run: | - pip install "verifiable-labs-envs @ git+https://github.com/verifiablelabs/verifiable-labs-envs@main" - pip install "vlabs-prm-eval @ git+https://github.com/verifiablelabs/verifiable-labs-envs@main#subdirectory=tools/vlabs-prm-eval" - pip install pytest + pip install "vlabs-sdk @ git+https://github.com/verifiablelabs/vlabs-sdk@main" + pip install typer pytest + pip install --no-deps "vlabs-prm-eval @ git+https://github.com/verifiablelabs/vlabs-sdk@main#subdirectory=tools/vlabs-prm-eval" - run: pytest tests -q diff --git a/PROVENANCE.md b/PROVENANCE.md index 676999d..9e95e75 100644 --- a/PROVENANCE.md +++ b/PROVENANCE.md @@ -1,7 +1,7 @@ # Provenance Clean import (no history rewrite) from `verifiablelabs/verifiable-labs-envs` -at commit `a0f30dc547a73aaae8608d193f94035192404627` (main). All example code authored fresh for this repo; depends on the public SDK surface only. Everything synthetic. +at commit `762b44e8019af3e89c55bba0f88e9157bb50c5c3` (main). All example code authored fresh for this repo; depends on the public SDK surface only. Everything synthetic. The source monorepo remains canonical until the split flips; this mirror is refreshed by the migration tooling documented in diff --git a/README.md b/README.md index 6b225d0..52f209f 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ evaluation content, no gold answers. ## Contents -- [`examples/quickstart_clean_gate.md`](examples/quickstart_clean_gate.md) — +- [`examples/run_clean_gate_demo.md`](examples/run_clean_gate_demo.md) — run the `clean-gate` CLI against sample metric cards. - [`examples/sample_assurance_card.json`](examples/sample_assurance_card.json) — an illustrative assurance card showing the shape of a gate decision. diff --git a/examples/improve_and_gate_mock.py b/examples/improve_and_gate_mock.py index c3b3a9a..f91e361 100644 --- a/examples/improve_and_gate_mock.py +++ b/examples/improve_and_gate_mock.py @@ -5,7 +5,7 @@ """ from __future__ import annotations -from verifiable_labs_envs.schemas import ScoreSet +from vlabs_sdk.schemas import ScoreSet BASELINE = ScoreSet( public_score=0.78, hidden_score=0.70, ood_score=0.66, adversarial_score=0.60, diff --git a/examples/quickstart_clean_gate.md b/examples/quickstart_clean_gate.md deleted file mode 100644 index 85aef85..0000000 --- a/examples/quickstart_clean_gate.md +++ /dev/null @@ -1,27 +0,0 @@ -# Quickstart: the clean promotion gate CLI - -The `clean-gate` command compares a baseline metric card against a candidate -and prints an ACCEPT / REJECT decision (exit code 0 / 1). It ships with -[verifiable-labs-envs](https://github.com/verifiablelabs/verifiable-labs-envs). - -```bash -git clone https://github.com/verifiablelabs/verifiable-labs-envs -cd verifiable-labs-envs/tools/vlabs-prm-eval -pip install -e . - -# Sample fixtures are included: -vlabs-prm-eval clean-gate \ - --old tests/fixtures/clean/clean_old.json \ - --new tests/fixtures/clean/clean_new_accept.json -# -> ACCEPT (exit 0) - -vlabs-prm-eval clean-gate \ - --old tests/fixtures/clean/clean_old.json \ - --new tests/fixtures/clean/clean_reject_dcr.json -# -> REJECT (exit 1): dcr_increased -``` - -The gate accepts a candidate only when clean verified-generalization score -improves without regressions in contamination risk, hack risk, calibration, -OOD transfer, cost, or latency. The decision logic mirrors the Lean 4 -formal specification and is property-tested against it. diff --git a/examples/run_clean_gate_demo.md b/examples/run_clean_gate_demo.md index 5e364e0..f983e9b 100644 --- a/examples/run_clean_gate_demo.md +++ b/examples/run_clean_gate_demo.md @@ -1,10 +1,12 @@ # Clean promotion gate demo ```bash -pip install "vlabs-prm-eval @ git+https://github.com/verifiablelabs/verifiable-labs-envs@main#subdirectory=tools/vlabs-prm-eval" +pip install "vlabs-sdk @ git+https://github.com/verifiablelabs/vlabs-sdk@main" +pip install typer +pip install --no-deps "vlabs-prm-eval @ git+https://github.com/verifiablelabs/vlabs-sdk@main#subdirectory=tools/vlabs-prm-eval" -vlabs-prm-eval clean-gate --old examples/cards/clean_old.json --new examples/cards/clean_new_accept.json +vlabs clean-gate --old examples/cards/clean_old.json --new examples/cards/clean_new_accept.json # -> ACCEPT (exit 0) -vlabs-prm-eval clean-gate --old examples/cards/clean_old.json --new examples/cards/clean_reject_dcr.json +vlabs clean-gate --old examples/cards/clean_old.json --new examples/cards/clean_reject_dcr.json # -> REJECT (exit 1): dcr_increased ``` diff --git a/examples/run_evaluate_only_dummy.py b/examples/run_evaluate_only_dummy.py index c9e21e1..7f90b2a 100644 --- a/examples/run_evaluate_only_dummy.py +++ b/examples/run_evaluate_only_dummy.py @@ -6,10 +6,10 @@ """ from __future__ import annotations -from verifiable_labs_envs.providers.base import ModelRequest -from verifiable_labs_envs.providers.dummy_provider import DummyProvider -from verifiable_labs_envs.run_config import default_config -from verifiable_labs_envs.schemas import ScoreSet, TransferMetrics +from vlabs_sdk.providers.base import ModelRequest +from vlabs_sdk.providers.dummy_provider import DummyProvider +from vlabs_sdk.run_config import default_config +from vlabs_sdk.schemas import ScoreSet, TransferMetrics SCENARIOS = [ {"split": "public", "prompt": "Customer asks for a refund of $25 bought 3 days ago."}, diff --git a/examples/sample_contract.json b/examples/sample_contract.json index 2e6bb77..54ae489 100644 --- a/examples/sample_contract.json +++ b/examples/sample_contract.json @@ -1,5 +1,5 @@ { - "_comment": "ILLUSTRATIVE evaluation-contract sketch — synthetic; the canonical schema is verifiable_labs_envs.schemas.EvaluationContract.", + "_comment": "ILLUSTRATIVE evaluation-contract sketch — synthetic; the canonical schema is vlabs_sdk.schemas.EvaluationContract.", "goal": "Refund-policy agent answers within policy and resists override attempts.", "capability": "customer_support_refunds", "domain": "e-commerce", diff --git a/tests/test_examples_run.py b/tests/test_examples_run.py index 61f959a..212b43f 100644 --- a/tests/test_examples_run.py +++ b/tests/test_examples_run.py @@ -44,12 +44,12 @@ def test_substrate_mock() -> None: def test_clean_gate_cli_accept_and_reject() -> None: cards = ROOT / "examples" / "cards" ok = subprocess.run( - ["vlabs-prm-eval", "clean-gate", "--old", str(cards / "clean_old.json"), + ["vlabs", "clean-gate", "--old", str(cards / "clean_old.json"), "--new", str(cards / "clean_new_accept.json")], capture_output=True, ) assert ok.returncode == 0 rej = subprocess.run( - ["vlabs-prm-eval", "clean-gate", "--old", str(cards / "clean_old.json"), + ["vlabs", "clean-gate", "--old", str(cards / "clean_old.json"), "--new", str(cards / "clean_reject_dcr.json")], capture_output=True, ) assert rej.returncode == 1