feat: add LiteLLM judge api_type

pyproject.toml

@@ -16,6 +16,11 @@ dependencies = [
     "rich>=13.0",
 ]
 
+[project.optional-dependencies]
+# Optional: judge api_type "litellm" routes to 100+ providers via one call.
+# Lazy-imported, so the base install stays dependency-light.
+litellm = ["litellm>=1.85.0,<2.0"]
+
 [project.urls]
 Homepage = "https://claw-bench.com"
 Repository = "https://github.com/reacher-z/ClawBench"

src/clawbench/runner/judge.py

@@ -172,6 +172,40 @@ def _call_anthropic_messages(
     return resp["content"][0]["text"]
 
 
+def _call_litellm(model_cfg: dict, model_name: str, system: str, user: str) -> str:
+    """Route the judge through LiteLLM, reaching any of 100+ providers with one
+    call. ``model_name`` is a LiteLLM model string (e.g. ``gpt-4o-mini``,
+    ``anthropic/claude-sonnet-4-6``, ``gemini/gemini-2.5-flash``); set
+    ``base_url`` to point at a LiteLLM proxy."""
+    try:
+        import litellm  # type: ignore
+    except ImportError as e:
+        raise ImportError(
+            "The 'litellm' package is required for api_type 'litellm'. "
+            "Install it with 'pip install clawbench[litellm]' or 'pip install litellm'."
+        ) from e
+
+    kwargs: dict[str, Any] = {}
+    if model_cfg.get("base_url"):
+        kwargs["api_base"] = model_cfg["base_url"]
+    if model_cfg.get("api_key"):
+        kwargs["api_key"] = model_cfg["api_key"]
+    resp = litellm.completion(
+        model=model_name,
+        messages=[
+            {"role": "system", "content": system},
+            {"role": "user", "content": user},
+        ],
+        max_tokens=4096,
+        temperature=0,
+        # Silently drop kwargs a given provider doesn't accept so one call works
+        # across OpenAI, Anthropic, Gemini, Bedrock, etc.
+        drop_params=True,
+        **kwargs,
+    )
+    return resp["choices"][0]["message"].get("content") or ""
+
+
 def _parse_verdict(text: str) -> tuple[bool | None, str]:
     """Best-effort parse of the judge's reply into (match, reason)."""
     text = (text or "").strip()
@@ -224,6 +258,8 @@ def _run_judge(
                 raw = _call_anthropic_messages(
                     model_cfg, judge_model_name, system, user
                 )
+            elif api_type == "litellm":
+                raw = _call_litellm(model_cfg, judge_model_name, system, user)
             else:
                 return {
                     "match": None,

src/clawbench/runner/judge_llm.py

@@ -143,6 +143,40 @@ def _call_anthropic_messages(
     )
 
 
+def _call_litellm(model_cfg: dict, model_name: str, system: str, user: str) -> str:
+    """Route the judge through LiteLLM, reaching any of 100+ providers with one
+    call. ``model_name`` is a LiteLLM model string (e.g. ``gpt-4o-mini``,
+    ``anthropic/claude-sonnet-4-6``, ``gemini/gemini-2.5-flash``); set
+    ``base_url`` to point at a LiteLLM proxy."""
+    try:
+        import litellm  # type: ignore
+    except ImportError as e:
+        raise ImportError(
+            "The 'litellm' package is required for api_type 'litellm'. "
+            "Install it with 'pip install clawbench[litellm]' or 'pip install litellm'."
+        ) from e
+
+    kwargs: dict[str, Any] = {}
+    if model_cfg.get("base_url"):
+        kwargs["api_base"] = model_cfg["base_url"]
+    if model_cfg.get("api_key"):
+        kwargs["api_key"] = model_cfg["api_key"]
+    resp = litellm.completion(
+        model=model_name,
+        messages=[
+            {"role": "system", "content": system},
+            {"role": "user", "content": user},
+        ],
+        max_tokens=800,
+        temperature=0.0,
+        # Silently drop kwargs a given provider doesn't accept so one call works
+        # across OpenAI, Anthropic, Gemini, Bedrock, etc.
+        drop_params=True,
+        **kwargs,
+    )
+    return resp["choices"][0]["message"]["content"] or ""
+
+
 def _parse_verdict(raw: str) -> tuple[bool, str]:
     """Best-effort parse of the judge's reply into (match, reason). Default TRUE on parse failure."""
     try:
@@ -183,6 +217,8 @@ def judge_request(
                 raw = _call_anthropic_messages(
                     model_cfg, judge_model_name, system, user
                 )
+            elif api_type == "litellm":
+                raw = _call_litellm(model_cfg, judge_model_name, system, user)
             else:
                 raise NotImplementedError(
                     f"judge_llm: unsupported api_type {api_type!r}"

src/clawbench/runner/run_support/api_preflight.py

@@ -192,5 +192,26 @@ def preflight_model_api(model_cfg: dict[str, Any], timeout: int = 20) -> None:
             isinstance(resp.get("candidates"), list) and bool(resp["candidates"]),
             "model API returned no Gemini candidates",
         )
+    elif api_type == "litellm":
+        try:
+            import litellm  # type: ignore
+        except ImportError as e:
+            raise ModelApiPreflightError(
+                "api_type 'litellm' requires the 'litellm' package "
+                "(pip install clawbench[litellm])"
+            ) from e
+        resp = litellm.completion(
+            model=model,
+            messages=[{"role": "user", "content": test_content}],
+            max_tokens=4,
+            temperature=0,
+            drop_params=True,
+            api_base=base_url,
+            api_key=key,
+        )
+        _expect_field(
+            bool(getattr(resp, "choices", None)),
+            "model API returned no chat choices",
+        )
     else:
         raise ModelApiPreflightError(f"unsupported api_type {api_type!r}")

src/clawbench/tui.py

@@ -137,6 +137,7 @@ def _is_selected(self, choice):
     "openai-responses",
     "anthropic-messages",
     "google-generative-ai",
+    "litellm",
 ]
 THINKING_LEVELS = ["off", "minimal", "low", "medium", "high", "xhigh", "adaptive"]
 
@@ -145,6 +146,20 @@ def _is_selected(self, choice):
 # example model names so the user doesn't have to remember the exact
 # string for each vendor.
 PROVIDER_PRESETS: dict[str, dict] = {
+    "litellm": {
+        "label": "LiteLLM    (proxy to 100+ providers)",
+        "base_url": "http://localhost:4000/v1",
+        "api_type": "litellm",
+        # With base_url pointing at a LiteLLM proxy, prefix the proxy's model
+        # name with "litellm_proxy/" so the SDK routes through the proxy.
+        # (For direct provider access instead, leave base_url blank and use a
+        # native model string like "anthropic/claude-sonnet-4-6".)
+        "examples": [
+            "litellm_proxy/gpt-4o-mini",
+            "litellm_proxy/claude-sonnet-4-6",
+            "anthropic/claude-sonnet-4-6",
+        ],
+    },
     "anthropic": {
         "label": "Anthropic  (Claude)",
         "base_url": "https://api.anthropic.com",

tests/test_judge_litellm.py

@@ -0,0 +1,97 @@
+"""Tests for the LiteLLM judge api_type (judge.py + judge_llm.py + preflight).
+
+`litellm` is an optional dependency, so we inject a lightweight fake module and
+patch its `completion` per test. This lets the suite run without the real
+package and lets us assert exactly what gets passed to litellm.completion.
+"""
+
+from __future__ import annotations
+
+import sys
+import types
+from typing import Any
+
+import pytest
+
+from clawbench.runner import judge, judge_llm
+from clawbench.runner.run_support import api_preflight
+
+
+class _FakeResp(dict):
+    """Stand-in for litellm's ModelResponse: supports both ["choices"] and .choices."""
+
+    def __init__(self, content: str):
+        super().__init__(choices=[{"message": {"content": content}}])
+        self.choices = self["choices"]
+
+
+def _install_fake_litellm(monkeypatch, content='{"match": true, "reason": "ok"}'):
+    calls: list[dict] = []
+    fake: Any = types.ModuleType("litellm")
+
+    def completion(**kwargs):
+        calls.append(kwargs)
+        return _FakeResp(content)
+
+    fake.completion = completion
+    monkeypatch.setitem(sys.modules, "litellm", fake)
+    return calls
+
+
+CFG = {
+    "base_url": "http://localhost:4000/v1",
+    "api_key": "sk-test",
+    "api_type": "litellm",
+}
+INTERCEPT = {"request": {"url": "https://x/api", "method": "POST", "body": {"a": 1}}}
+
+
+def test_call_litellm_passes_expected_kwargs(monkeypatch) -> None:
+    calls = _install_fake_litellm(monkeypatch)
+    out = judge._call_litellm(CFG, "gpt-4o-mini", "sys", "user")
+    assert out == '{"match": true, "reason": "ok"}'
+    kw = calls[0]
+    assert kw["model"] == "gpt-4o-mini"
+    assert kw["drop_params"] is True
+    assert kw["api_base"] == "http://localhost:4000/v1"
+    assert kw["api_key"] == "sk-test"
+    assert kw["messages"][0]["role"] == "system"
+    assert kw["messages"][1]["role"] == "user"
+
+
+def test_strict_judge_dispatches_litellm(monkeypatch) -> None:
+    _install_fake_litellm(monkeypatch, '{"match": false, "reason": "wrong color"}')
+    r = judge.judge_request(CFG, "gpt-4o-mini", "buy a red shirt", INTERCEPT)
+    assert r["match"] is False
+    assert r["judge_model"] == "gpt-4o-mini"
+
+
+def test_lenient_judge_dispatches_litellm(monkeypatch) -> None:
+    _install_fake_litellm(monkeypatch, '{"match": true, "reason": "no contradiction"}')
+    r = judge_llm.judge_request(
+        CFG, "anthropic/claude-sonnet-4-6", "buy a shirt", INTERCEPT
+    )
+    assert r["match"] is True
+    assert r["rubric"] == "lenient"
+
+
+def test_preflight_accepts_litellm(monkeypatch) -> None:
+    _install_fake_litellm(monkeypatch, "OK")
+    # Should not raise.
+    api_preflight.preflight_model_api({**CFG, "model": "gpt-4o-mini"})
+
+
+def test_call_litellm_omits_credentials_when_absent(monkeypatch) -> None:
+    calls = _install_fake_litellm(monkeypatch)
+    judge._call_litellm({"api_type": "litellm"}, "gpt-4o-mini", "sys", "user")
+    kw = calls[0]
+    # No base_url/api_key configured -> LiteLLM falls back to provider env vars.
+    assert "api_base" not in kw
+    assert "api_key" not in kw
+
+
+def test_call_litellm_raises_without_package(monkeypatch) -> None:
+    # Simulate litellm not installed.
+    monkeypatch.setitem(sys.modules, "litellm", None)
+    with pytest.raises(ImportError, match="litellm"):
+        judge._call_litellm(CFG, "gpt-4o-mini", "sys", "user")