diff --git a/src/guidellm/__main__.py b/src/guidellm/__main__.py index 85b7cfc0c..1c08a6128 100644 --- a/src/guidellm/__main__.py +++ b/src/guidellm/__main__.py @@ -175,7 +175,8 @@ def benchmark(): default=BenchmarkGenerativeTextArgs.get_default("backend_kwargs"), help=( "JSON string of arguments to pass to the backend. E.g., " - '\'{"api_key": "apikey-*", "verify": false}\'' + '\'{"api_key": "apikey-*", "verify": false}\' or ' + '\'{"extras": {"body": {"temperature": 0.6, "top_p": 0.95}}}\' for sampling params.' ), ) @click.option( diff --git a/src/guidellm/backends/openai/http.py b/src/guidellm/backends/openai/http.py index 9e33f54af..ded568215 100644 --- a/src/guidellm/backends/openai/http.py +++ b/src/guidellm/backends/openai/http.py @@ -74,6 +74,13 @@ class OpenAIHttpBackendArgs(BackendArgs): ) }, ) + extras: dict[str, Any] | None = Field( + default=None, + description=( + "Optional request arguments to merge into each request (e.g. body fields). " + 'Use {"body": {"temperature": 0.6, "top_p": 0.95}} to set sampling params.' + ), + ) @field_validator("request_format") @classmethod