diff --git a/AIGateway/src/routers/prompts.py b/AIGateway/src/routers/prompts.py index a30065c268..d10fb24ddc 100644 --- a/AIGateway/src/routers/prompts.py +++ b/AIGateway/src/routers/prompts.py @@ -377,27 +377,48 @@ async def delete_test_dataset( return {"deleted": True} -@router.post("/test", status_code=501) +@router.post("/test") async def test_prompt(request: Request, body: TestPromptRequest): """ - Test a prompt by resolving variables and proxying to the LLM completion - endpoint. The LLM proxy integration lives in the Express backend for now; - this stub returns 501 until the proxy is co-located in the AIGateway - service. + Test a prompt by resolving variables and streaming the LLM response. + Returns SSE stream compatible with the frontend's streamPromptTest(). """ + import json as _json + + from fastapi.responses import StreamingResponse + from services.proxy_service import resolve_endpoint_for_key + from services.llm_service import stream_chat_completion + verify_internal_key(request) + org_id = get_org_id(request) - # Resolve variables so callers can at least validate substitution locally + # Resolve variables in the prompt content resolved_content = crud.resolve_variables( body.content, body.variables or {}, ) - raise HTTPException( - status_code=501, - detail=( - "test-prompt requires proxy integration — " - "LLM proxy currently runs in the Express backend. " - "Resolved content is available but cannot be forwarded yet." - ), - ) + # Resolve the endpoint to get provider, model, and API key + try: + endpoint = await resolve_endpoint_for_key( + organization_id=org_id, + endpoint_slug=body.endpoint_slug, + allowed_endpoint_ids=[], + ) + except ValueError as e: + raise HTTPException(status_code=404, detail=str(e)) + + # Stream the LLM response + async def _stream(): + try: + async for chunk_str in stream_chat_completion( + model=endpoint["model"], + messages=resolved_content, + api_key=endpoint["decrypted_key"], + ): + yield chunk_str + except Exception as e: + yield f"data: {_json.dumps({'error': str(e)})}\n\n" + yield "data: [DONE]\n\n" + + return StreamingResponse(_stream(), media_type="text/event-stream") diff --git a/AIGateway/src/routers/tenant_chat.py b/AIGateway/src/routers/tenant_chat.py index 7457d17b82..70a7aa8580 100644 --- a/AIGateway/src/routers/tenant_chat.py +++ b/AIGateway/src/routers/tenant_chat.py @@ -127,7 +127,10 @@ async def get_providers(request: Request): provider = info.get("litellm_provider", "unknown") if provider not in providers: providers[provider] = [] - providers[provider].append(model_name) + providers[provider].append({ + "id": model_name, + "mode": info.get("mode", "chat"), + }) return { "data": { diff --git a/Clients/src/presentation/pages/AIGateway/Prompts/ComparePanel.tsx b/Clients/src/presentation/pages/AIGateway/Prompts/ComparePanel.tsx index 4eb5e2edad..74615f6d64 100644 --- a/Clients/src/presentation/pages/AIGateway/Prompts/ComparePanel.tsx +++ b/Clients/src/presentation/pages/AIGateway/Prompts/ComparePanel.tsx @@ -115,6 +115,7 @@ export default function ComparePanel({ onChange={(e) => setVersionA(Number(e.target.value))} items={versionItems} placeholder="Select version" + getOptionValue={(item) => item._id} sx={{ flex: 1 }} /> setModel(e.target.value as string)} - items={allModelItems} - placeholder="Select model" - sx={{ flex: 1 }} - getOptionValue={(item) => item._id} - /> + + { + setModelSearch(e.target.value); + setModelDropdownOpen(e.target.value.length >= 2); + if (!e.target.value) setModel(""); + }} + onFocus={() => { if (modelSearch.length >= 2) setModelDropdownOpen(true); }} + onBlur={() => setTimeout(() => setModelDropdownOpen(false), 200)} + sx={{ minWidth: "unset" }} + /> + {modelDropdownOpen && modelSearch.length >= 2 && (() => { + const q = modelSearch.toLowerCase(); + const filtered = allModelItems.filter((m) => m.name.toLowerCase().includes(q)).slice(0, 20); + if (filtered.length === 0) return null; + return ( + + {filtered.map((m) => ( + e.preventDefault()} + onClick={() => { setModel(m._id); setModelSearch(""); setModelDropdownOpen(false); }} + sx={{ + px: "12px", py: "8px", cursor: "pointer", fontSize: 12, + "&:hover": { bgcolor: palette.background.alt }, + bgcolor: model === m._id ? `${palette.brand.primary}08` : "transparent", + }} + > + {m.name} + + ))} + + ); + })()} + { setTempConfig({ ...config }); setIsConfigOpen(true); }} @@ -650,39 +685,37 @@ export default function PromptEditorPage() { ))} + {/* Endpoint selector + variables (shared across all tabs) */} + + setSelectedEndpoint(e.target.value as string)} - items={endpoints.map((e: any) => ({ _id: e.slug, name: `${e.display_name} (${e.slug})` }))} - placeholder="Select endpoint" - sx={{ width: "100%" }} - /> - 0 ? "16px" : 0}> - Pick an endpoint to route test requests through. The model and API key come from the endpoint configuration. - - {detectedVars.length > 0 && ( - - Variables - {detectedVars.map((v) => ( - setVariableValues((prev) => ({ ...prev, [v]: e.target.value }))} - placeholder={`Value for {{${v}}}`} - /> - ))} - - )} - - {/* Chat area */} {chatMessages.length === 0 && ( diff --git a/Clients/src/presentation/pages/AIGateway/shared.ts b/Clients/src/presentation/pages/AIGateway/shared.ts index d20e3b7762..814f696d17 100644 --- a/Clients/src/presentation/pages/AIGateway/shared.ts +++ b/Clients/src/presentation/pages/AIGateway/shared.ts @@ -245,7 +245,8 @@ export async function streamPromptTest( ): Promise { const startTime = Date.now(); - const response = await fetch(`${GATEWAY_API_URL}/ai-gateway/prompts/test`, { + // Use relative URL to go through Vite proxy (avoids CORS with fetch) + const response = await fetch("/api/ai-gateway/prompts/test", { method: "POST", headers: { "Content-Type": "application/json",