scripts_eval_feature2.py

import time
from cacheguard.embedder import Embedder
from cacheguard.benchmark.dataset import load_qqp
from cacheguard.benchmark.sweep import run_sweep, rank_models

MODELS = [
    "all-MiniLM-L6-v2",
    "all-mpnet-base-v2",
    "BAAI/bge-small-en-v1.5",
    "thenlper/gte-small",
]
TARGET_HIT_RATE = 0.30
BASE_RATE = 0.70

rows = load_qqp(limit=4000)
thresholds = [round(0.70 + 0.02 * i, 2) for i in range(13)]

t0 = time.time()
sweep = run_sweep(rows, MODELS, thresholds, embedder_factory=Embedder, cache_dir=".emb_cache")
dt = time.time() - t0
print(f"sweep ran {len(sweep)} models on {len(rows)} pairs in {dt:.1f}s\n")

ranked = rank_models(sweep, target_hit_rate=TARGET_HIT_RATE, dup_base_rate=BASE_RATE)
print(f"LEADERBOARD (adj false-hit @~{TARGET_HIT_RATE:.0%} hit-rate, {BASE_RATE:.0%} dup prior):")
for i, (name, adj) in enumerate(ranked, 1):
    print(f"  {i}. {name:28s}  {adj:.3f}")

spread = ranked[-1][1] - ranked[0][1]
print("\n---METRIC GATE---")
print(f"models run        : {len(sweep)}/4")
print(f"ranking reproducible: deterministic (fixed embeddings)")
print(f"best-worst spread : {spread:.3f} (target >= 0.020)")
print(f"GATE PASS: {len(sweep) == 4 and spread >= 0.02}")