From 9652179d690eba63fbaac7ae13648f3f638a652e Mon Sep 17 00:00:00 2001 From: harivilasp Date: Mon, 6 Jul 2026 20:05:24 -0700 Subject: [PATCH 1/2] feat: benchmark proxy payload scaling Signed-off-by: harivilasp --- crates/agentic-server/benches/proxy_bench.rs | 142 ++++++++++-------- crates/agentic-server/tests/responses_test.rs | 32 +++- 2 files changed, 112 insertions(+), 62 deletions(-) diff --git a/crates/agentic-server/benches/proxy_bench.rs b/crates/agentic-server/benches/proxy_bench.rs index 8628da0..7a36cf0 100644 --- a/crates/agentic-server/benches/proxy_bench.rs +++ b/crates/agentic-server/benches/proxy_bench.rs @@ -1,4 +1,5 @@ use std::convert::Infallible; +use std::sync::Arc; use axum::body::Body; use axum::extract::Request; @@ -6,7 +7,7 @@ use axum::response::{IntoResponse, Response}; use axum::routing::{get, post}; use axum::{Router, serve}; use bytes::Bytes; -use criterion::{Criterion, criterion_group}; +use criterion::{BenchmarkId, Criterion, criterion_group}; use futures::stream; use http::StatusCode; use tokio::net::TcpListener; @@ -18,7 +19,9 @@ use agentic_core::executor::{ConversationHandler, ExecutionContext, ResponseHand use agentic_core::proxy::ProxyState; use agentic_core::storage::{ConversationStore, ResponseStore}; use agentic_server::app::{AppState, ServerConfig, build_router}; -use std::sync::Arc; + +const CONTENT_TYPE_JSON: &str = "application/json"; +const PAYLOAD_SIZES: [usize; 3] = [1024, 10 * 1024, 100 * 1024]; fn bench_config(llm_url: &str) -> Config { Config { @@ -58,7 +61,7 @@ async fn responses_handler(req: Request) -> Response { } let out = r#"{"id":"resp_bench","object":"response","status":"completed"}"#; - (StatusCode::OK, [("content-type", "application/json")], out).into_response() + (StatusCode::OK, [("content-type", CONTENT_TYPE_JSON)], out).into_response() } async fn spawn_llm() -> String { @@ -102,56 +105,51 @@ async fn spawn_gateway(config: Config) -> String { format!("http://{addr}") } -fn proxy_benchmarks(c: &mut Criterion) { - let rt = Runtime::new().unwrap(); +fn responses_url(base_url: &str) -> String { + format!("{base_url}/v1/responses") +} - let (llm_url, gateway_url) = rt.block_on(async { - let llm_url = spawn_llm().await; - let config = bench_config(&llm_url); - let gateway_url = spawn_gateway(config).await; - (llm_url, gateway_url) +fn request_body(prompt_bytes: usize, stream: bool) -> Bytes { + let prompt = "x".repeat(prompt_bytes); + let body = serde_json::json!({ + "model": "bench-model", + "input": [{"role": "user", "content": prompt}], + "store": false, + "stream": stream }); + Bytes::from(serde_json::to_vec(&body).expect("benchmark request body serializes")) +} - let client = reqwest::Client::new(); +async fn post_response(client: reqwest::Client, url: String, body: Bytes) -> usize { + let resp = client + .post(url) + .header("content-type", CONTENT_TYPE_JSON) + .body(body) + .send() + .await + .expect("benchmark request succeeds"); - let non_stream_body = serde_json::json!({ - "model": "bench-model", - "input": [{"role": "user", "content": "hello"}] - }); - let stream_body = serde_json::json!({ - "model": "bench-model", - "input": [{"role": "user", "content": "hello"}], - "stream": true - }); + resp.bytes().await.expect("benchmark response body").len() +} + +fn bench_single_request(c: &mut Criterion, rt: &Runtime, client: &reqwest::Client, llm_url: &str, gateway_url: &str) { + let non_stream_body = request_body(5, false); + let stream_body = request_body(5, true); let mut group = c.benchmark_group("non_stream"); group.bench_function("direct", |b| { - let url = format!("{llm_url}/v1/responses"); + let url = responses_url(llm_url); let body = non_stream_body.clone(); - b.to_async(&rt).iter(|| { - let client = client.clone(); - let url = url.clone(); - let body = body.clone(); - async move { - let resp = client.post(&url).json(&body).send().await.unwrap(); - resp.bytes().await.unwrap() - } - }); + b.to_async(rt) + .iter(|| post_response(client.clone(), url.clone(), body.clone())); }); group.bench_function("proxied", |b| { - let url = format!("{gateway_url}/v1/responses"); + let url = responses_url(gateway_url); let body = non_stream_body.clone(); - b.to_async(&rt).iter(|| { - let client = client.clone(); - let url = url.clone(); - let body = body.clone(); - async move { - let resp = client.post(&url).json(&body).send().await.unwrap(); - resp.bytes().await.unwrap() - } - }); + b.to_async(rt) + .iter(|| post_response(client.clone(), url.clone(), body.clone())); }); group.finish(); @@ -159,34 +157,58 @@ fn proxy_benchmarks(c: &mut Criterion) { let mut group = c.benchmark_group("stream"); group.bench_function("direct", |b| { - let url = format!("{llm_url}/v1/responses"); + let url = responses_url(llm_url); let body = stream_body.clone(); - b.to_async(&rt).iter(|| { - let client = client.clone(); - let url = url.clone(); - let body = body.clone(); - async move { - let resp = client.post(&url).json(&body).send().await.unwrap(); - resp.bytes().await.unwrap() - } - }); + b.to_async(rt) + .iter(|| post_response(client.clone(), url.clone(), body.clone())); }); group.bench_function("proxied", |b| { - let url = format!("{gateway_url}/v1/responses"); + let url = responses_url(gateway_url); let body = stream_body.clone(); - b.to_async(&rt).iter(|| { - let client = client.clone(); - let url = url.clone(); - let body = body.clone(); - async move { - let resp = client.post(&url).json(&body).send().await.unwrap(); - resp.bytes().await.unwrap() - } - }); + b.to_async(rt) + .iter(|| post_response(client.clone(), url.clone(), body.clone())); }); group.finish(); } +fn bench_body_size(c: &mut Criterion, rt: &Runtime, client: &reqwest::Client, llm_url: &str, gateway_url: &str) { + let mut group = c.benchmark_group("non_stream/body_size"); + + for size in PAYLOAD_SIZES { + group.bench_with_input(BenchmarkId::new("direct", size), &size, |b, &size| { + let url = responses_url(llm_url); + let body = request_body(size, false); + b.to_async(rt) + .iter(|| post_response(client.clone(), url.clone(), body.clone())); + }); + + group.bench_with_input(BenchmarkId::new("proxied", size), &size, |b, &size| { + let url = responses_url(gateway_url); + let body = request_body(size, false); + b.to_async(rt) + .iter(|| post_response(client.clone(), url.clone(), body.clone())); + }); + } + + group.finish(); +} + +fn proxy_benchmarks(c: &mut Criterion) { + let rt = Runtime::new().unwrap(); + + let (llm_url, gateway_url) = rt.block_on(async { + let llm_url = spawn_llm().await; + let config = bench_config(&llm_url); + let gateway_url = spawn_gateway(config).await; + (llm_url, gateway_url) + }); + + let client = reqwest::Client::new(); + + bench_single_request(c, &rt, &client, &llm_url, &gateway_url); + bench_body_size(c, &rt, &client, &llm_url, &gateway_url); +} + criterion_group!(proxy_benches, proxy_benchmarks); diff --git a/crates/agentic-server/tests/responses_test.rs b/crates/agentic-server/tests/responses_test.rs index ab64791..52894be 100644 --- a/crates/agentic-server/tests/responses_test.rs +++ b/crates/agentic-server/tests/responses_test.rs @@ -1,12 +1,11 @@ mod common; -use std::sync::Arc; - use axum::Router; use axum::body::Bytes; use axum::response::IntoResponse; use axum::routing::post; use http::StatusCode; +use std::sync::Arc; use tokio::net::TcpListener; use tokio::sync::Mutex; @@ -133,6 +132,35 @@ async fn test_store_false_with_web_search_reaches_executor() { assert_eq!(requests[0]["tools"][0]["name"], "web_search"); } +#[tokio::test] +async fn test_store_false_proxies_large_json_body_to_vllm() { + // Arrange + let (llm_url, requests, _h1) = spawn_mock_vllm_json_capture().await; + let (gw_url, _h2) = spawn_gateway(test_state(&test_config(&llm_url))).await; + let prompt = "x".repeat(100 * 1024); + + // Act + let resp = reqwest::Client::new() + .post(format!("{gw_url}/v1/responses")) + .json(&serde_json::json!({ + "model": "test", + "input": [{"type": "message", "role": "user", "content": prompt}], + "store": false, + "stream": false + })) + .send() + .await + .unwrap(); + + // Assert — the gateway keeps this below-limit request on the proxy path. + assert_eq!(resp.status(), 200); + let requests = requests.lock().await; + assert_eq!(requests.len(), 1); + assert_eq!(requests[0]["store"], false); + assert_eq!(requests[0]["stream"], false); + assert_eq!(requests[0]["input"][0]["content"].as_str().unwrap().len(), 100 * 1024); +} + #[tokio::test] async fn test_store_false_proxies_sse_to_vllm() { // Arrange From df1f4637be0a01c3d90befe160fdd69f6b594aa1 Mon Sep 17 00:00:00 2001 From: harivilasp Date: Tue, 7 Jul 2026 20:58:33 -0700 Subject: [PATCH 2/2] clarify proxy benchmark prompt sizing Signed-off-by: harivilasp --- crates/agentic-server/benches/proxy_bench.rs | 47 ++++++++++++-------- 1 file changed, 28 insertions(+), 19 deletions(-) diff --git a/crates/agentic-server/benches/proxy_bench.rs b/crates/agentic-server/benches/proxy_bench.rs index 7a36cf0..f3ccfd5 100644 --- a/crates/agentic-server/benches/proxy_bench.rs +++ b/crates/agentic-server/benches/proxy_bench.rs @@ -21,7 +21,7 @@ use agentic_core::storage::{ConversationStore, ResponseStore}; use agentic_server::app::{AppState, ServerConfig, build_router}; const CONTENT_TYPE_JSON: &str = "application/json"; -const PAYLOAD_SIZES: [usize; 3] = [1024, 10 * 1024, 100 * 1024]; +const PROMPT_SIZES: [usize; 3] = [1024, 10 * 1024, 100 * 1024]; fn bench_config(llm_url: &str) -> Config { Config { @@ -60,6 +60,7 @@ async fn responses_handler(req: Request) -> Response { .into_response(); } + // Keep the mock response fixed so prompt-size runs isolate request upload overhead. let out = r#"{"id":"resp_bench","object":"response","status":"completed"}"#; (StatusCode::OK, [("content-type", CONTENT_TYPE_JSON)], out).into_response() } @@ -173,23 +174,31 @@ fn bench_single_request(c: &mut Criterion, rt: &Runtime, client: &reqwest::Clien group.finish(); } -fn bench_body_size(c: &mut Criterion, rt: &Runtime, client: &reqwest::Client, llm_url: &str, gateway_url: &str) { - let mut group = c.benchmark_group("non_stream/body_size"); - - for size in PAYLOAD_SIZES { - group.bench_with_input(BenchmarkId::new("direct", size), &size, |b, &size| { - let url = responses_url(llm_url); - let body = request_body(size, false); - b.to_async(rt) - .iter(|| post_response(client.clone(), url.clone(), body.clone())); - }); - - group.bench_with_input(BenchmarkId::new("proxied", size), &size, |b, &size| { - let url = responses_url(gateway_url); - let body = request_body(size, false); - b.to_async(rt) - .iter(|| post_response(client.clone(), url.clone(), body.clone())); - }); +fn bench_prompt_size(c: &mut Criterion, rt: &Runtime, client: &reqwest::Client, llm_url: &str, gateway_url: &str) { + let mut group = c.benchmark_group("non_stream/prompt_bytes"); + + for prompt_bytes in PROMPT_SIZES { + group.bench_with_input( + BenchmarkId::new("direct", prompt_bytes), + &prompt_bytes, + |b, &prompt_bytes| { + let url = responses_url(llm_url); + let body = request_body(prompt_bytes, false); + b.to_async(rt) + .iter(|| post_response(client.clone(), url.clone(), body.clone())); + }, + ); + + group.bench_with_input( + BenchmarkId::new("proxied", prompt_bytes), + &prompt_bytes, + |b, &prompt_bytes| { + let url = responses_url(gateway_url); + let body = request_body(prompt_bytes, false); + b.to_async(rt) + .iter(|| post_response(client.clone(), url.clone(), body.clone())); + }, + ); } group.finish(); @@ -208,7 +217,7 @@ fn proxy_benchmarks(c: &mut Criterion) { let client = reqwest::Client::new(); bench_single_request(c, &rt, &client, &llm_url, &gateway_url); - bench_body_size(c, &rt, &client, &llm_url, &gateway_url); + bench_prompt_size(c, &rt, &client, &llm_url, &gateway_url); } criterion_group!(proxy_benches, proxy_benchmarks);