diff --git a/pyproject.toml b/pyproject.toml index 240c7532..8dc19604 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,11 @@ dependencies = [ "rich>=13.0", ] +[project.optional-dependencies] +# Optional: judge api_type "litellm" routes to 100+ providers via one call. +# Lazy-imported, so the base install stays dependency-light. +litellm = ["litellm>=1.85.0,<2.0"] + [project.urls] Homepage = "https://claw-bench.com" Repository = "https://github.com/reacher-z/ClawBench" diff --git a/src/clawbench/runner/judge.py b/src/clawbench/runner/judge.py index 0ba0e4f2..dec0b810 100644 --- a/src/clawbench/runner/judge.py +++ b/src/clawbench/runner/judge.py @@ -172,6 +172,40 @@ def _call_anthropic_messages( return resp["content"][0]["text"] +def _call_litellm(model_cfg: dict, model_name: str, system: str, user: str) -> str: + """Route the judge through LiteLLM, reaching any of 100+ providers with one + call. ``model_name`` is a LiteLLM model string (e.g. ``gpt-4o-mini``, + ``anthropic/claude-sonnet-4-6``, ``gemini/gemini-2.5-flash``); set + ``base_url`` to point at a LiteLLM proxy.""" + try: + import litellm # type: ignore + except ImportError as e: + raise ImportError( + "The 'litellm' package is required for api_type 'litellm'. " + "Install it with 'pip install clawbench[litellm]' or 'pip install litellm'." + ) from e + + kwargs: dict[str, Any] = {} + if model_cfg.get("base_url"): + kwargs["api_base"] = model_cfg["base_url"] + if model_cfg.get("api_key"): + kwargs["api_key"] = model_cfg["api_key"] + resp = litellm.completion( + model=model_name, + messages=[ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ], + max_tokens=4096, + temperature=0, + # Silently drop kwargs a given provider doesn't accept so one call works + # across OpenAI, Anthropic, Gemini, Bedrock, etc. + drop_params=True, + **kwargs, + ) + return resp["choices"][0]["message"].get("content") or "" + + def _parse_verdict(text: str) -> tuple[bool | None, str]: """Best-effort parse of the judge's reply into (match, reason).""" text = (text or "").strip() @@ -224,6 +258,8 @@ def _run_judge( raw = _call_anthropic_messages( model_cfg, judge_model_name, system, user ) + elif api_type == "litellm": + raw = _call_litellm(model_cfg, judge_model_name, system, user) else: return { "match": None, diff --git a/src/clawbench/runner/judge_llm.py b/src/clawbench/runner/judge_llm.py index 59573329..86619d33 100644 --- a/src/clawbench/runner/judge_llm.py +++ b/src/clawbench/runner/judge_llm.py @@ -143,6 +143,40 @@ def _call_anthropic_messages( ) +def _call_litellm(model_cfg: dict, model_name: str, system: str, user: str) -> str: + """Route the judge through LiteLLM, reaching any of 100+ providers with one + call. ``model_name`` is a LiteLLM model string (e.g. ``gpt-4o-mini``, + ``anthropic/claude-sonnet-4-6``, ``gemini/gemini-2.5-flash``); set + ``base_url`` to point at a LiteLLM proxy.""" + try: + import litellm # type: ignore + except ImportError as e: + raise ImportError( + "The 'litellm' package is required for api_type 'litellm'. " + "Install it with 'pip install clawbench[litellm]' or 'pip install litellm'." + ) from e + + kwargs: dict[str, Any] = {} + if model_cfg.get("base_url"): + kwargs["api_base"] = model_cfg["base_url"] + if model_cfg.get("api_key"): + kwargs["api_key"] = model_cfg["api_key"] + resp = litellm.completion( + model=model_name, + messages=[ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ], + max_tokens=800, + temperature=0.0, + # Silently drop kwargs a given provider doesn't accept so one call works + # across OpenAI, Anthropic, Gemini, Bedrock, etc. + drop_params=True, + **kwargs, + ) + return resp["choices"][0]["message"]["content"] or "" + + def _parse_verdict(raw: str) -> tuple[bool, str]: """Best-effort parse of the judge's reply into (match, reason). Default TRUE on parse failure.""" try: @@ -183,6 +217,8 @@ def judge_request( raw = _call_anthropic_messages( model_cfg, judge_model_name, system, user ) + elif api_type == "litellm": + raw = _call_litellm(model_cfg, judge_model_name, system, user) else: raise NotImplementedError( f"judge_llm: unsupported api_type {api_type!r}" diff --git a/src/clawbench/runner/run_support/api_preflight.py b/src/clawbench/runner/run_support/api_preflight.py index 5637b7e6..417ea1bb 100644 --- a/src/clawbench/runner/run_support/api_preflight.py +++ b/src/clawbench/runner/run_support/api_preflight.py @@ -192,5 +192,26 @@ def preflight_model_api(model_cfg: dict[str, Any], timeout: int = 20) -> None: isinstance(resp.get("candidates"), list) and bool(resp["candidates"]), "model API returned no Gemini candidates", ) + elif api_type == "litellm": + try: + import litellm # type: ignore + except ImportError as e: + raise ModelApiPreflightError( + "api_type 'litellm' requires the 'litellm' package " + "(pip install clawbench[litellm])" + ) from e + resp = litellm.completion( + model=model, + messages=[{"role": "user", "content": test_content}], + max_tokens=4, + temperature=0, + drop_params=True, + api_base=base_url, + api_key=key, + ) + _expect_field( + bool(getattr(resp, "choices", None)), + "model API returned no chat choices", + ) else: raise ModelApiPreflightError(f"unsupported api_type {api_type!r}") diff --git a/src/clawbench/tui.py b/src/clawbench/tui.py index b8278c7d..c4448b4d 100644 --- a/src/clawbench/tui.py +++ b/src/clawbench/tui.py @@ -137,6 +137,7 @@ def _is_selected(self, choice): "openai-responses", "anthropic-messages", "google-generative-ai", + "litellm", ] THINKING_LEVELS = ["off", "minimal", "low", "medium", "high", "xhigh", "adaptive"] @@ -145,6 +146,20 @@ def _is_selected(self, choice): # example model names so the user doesn't have to remember the exact # string for each vendor. PROVIDER_PRESETS: dict[str, dict] = { + "litellm": { + "label": "LiteLLM (proxy to 100+ providers)", + "base_url": "http://localhost:4000/v1", + "api_type": "litellm", + # With base_url pointing at a LiteLLM proxy, prefix the proxy's model + # name with "litellm_proxy/" so the SDK routes through the proxy. + # (For direct provider access instead, leave base_url blank and use a + # native model string like "anthropic/claude-sonnet-4-6".) + "examples": [ + "litellm_proxy/gpt-4o-mini", + "litellm_proxy/claude-sonnet-4-6", + "anthropic/claude-sonnet-4-6", + ], + }, "anthropic": { "label": "Anthropic (Claude)", "base_url": "https://api.anthropic.com", diff --git a/tests/test_judge_litellm.py b/tests/test_judge_litellm.py new file mode 100644 index 00000000..bb407d82 --- /dev/null +++ b/tests/test_judge_litellm.py @@ -0,0 +1,97 @@ +"""Tests for the LiteLLM judge api_type (judge.py + judge_llm.py + preflight). + +`litellm` is an optional dependency, so we inject a lightweight fake module and +patch its `completion` per test. This lets the suite run without the real +package and lets us assert exactly what gets passed to litellm.completion. +""" + +from __future__ import annotations + +import sys +import types +from typing import Any + +import pytest + +from clawbench.runner import judge, judge_llm +from clawbench.runner.run_support import api_preflight + + +class _FakeResp(dict): + """Stand-in for litellm's ModelResponse: supports both ["choices"] and .choices.""" + + def __init__(self, content: str): + super().__init__(choices=[{"message": {"content": content}}]) + self.choices = self["choices"] + + +def _install_fake_litellm(monkeypatch, content='{"match": true, "reason": "ok"}'): + calls: list[dict] = [] + fake: Any = types.ModuleType("litellm") + + def completion(**kwargs): + calls.append(kwargs) + return _FakeResp(content) + + fake.completion = completion + monkeypatch.setitem(sys.modules, "litellm", fake) + return calls + + +CFG = { + "base_url": "http://localhost:4000/v1", + "api_key": "sk-test", + "api_type": "litellm", +} +INTERCEPT = {"request": {"url": "https://x/api", "method": "POST", "body": {"a": 1}}} + + +def test_call_litellm_passes_expected_kwargs(monkeypatch) -> None: + calls = _install_fake_litellm(monkeypatch) + out = judge._call_litellm(CFG, "gpt-4o-mini", "sys", "user") + assert out == '{"match": true, "reason": "ok"}' + kw = calls[0] + assert kw["model"] == "gpt-4o-mini" + assert kw["drop_params"] is True + assert kw["api_base"] == "http://localhost:4000/v1" + assert kw["api_key"] == "sk-test" + assert kw["messages"][0]["role"] == "system" + assert kw["messages"][1]["role"] == "user" + + +def test_strict_judge_dispatches_litellm(monkeypatch) -> None: + _install_fake_litellm(monkeypatch, '{"match": false, "reason": "wrong color"}') + r = judge.judge_request(CFG, "gpt-4o-mini", "buy a red shirt", INTERCEPT) + assert r["match"] is False + assert r["judge_model"] == "gpt-4o-mini" + + +def test_lenient_judge_dispatches_litellm(monkeypatch) -> None: + _install_fake_litellm(monkeypatch, '{"match": true, "reason": "no contradiction"}') + r = judge_llm.judge_request( + CFG, "anthropic/claude-sonnet-4-6", "buy a shirt", INTERCEPT + ) + assert r["match"] is True + assert r["rubric"] == "lenient" + + +def test_preflight_accepts_litellm(monkeypatch) -> None: + _install_fake_litellm(monkeypatch, "OK") + # Should not raise. + api_preflight.preflight_model_api({**CFG, "model": "gpt-4o-mini"}) + + +def test_call_litellm_omits_credentials_when_absent(monkeypatch) -> None: + calls = _install_fake_litellm(monkeypatch) + judge._call_litellm({"api_type": "litellm"}, "gpt-4o-mini", "sys", "user") + kw = calls[0] + # No base_url/api_key configured -> LiteLLM falls back to provider env vars. + assert "api_base" not in kw + assert "api_key" not in kw + + +def test_call_litellm_raises_without_package(monkeypatch) -> None: + # Simulate litellm not installed. + monkeypatch.setitem(sys.modules, "litellm", None) + with pytest.raises(ImportError, match="litellm"): + judge._call_litellm(CFG, "gpt-4o-mini", "sys", "user")