From 5b46397cd6e83f4fd84b7ec2bf44dc31d28ae57c Mon Sep 17 00:00:00 2001 From: Kai Date: Mon, 9 Feb 2026 23:26:04 -0800 Subject: [PATCH] Wire agent playground to persistent run data --- data/playground-runs.json | 68 +++ src/app/agent-playground/page.tsx | 780 ++++++++++---------------- src/app/api/agent-playground/route.ts | 141 +++++ 3 files changed, 517 insertions(+), 472 deletions(-) create mode 100644 data/playground-runs.json create mode 100644 src/app/api/agent-playground/route.ts diff --git a/data/playground-runs.json b/data/playground-runs.json new file mode 100644 index 00000000..548e8049 --- /dev/null +++ b/data/playground-runs.json @@ -0,0 +1,68 @@ +[ + { + "id": "run_1707431400000", + "agentHandle": "@kai@reflectt.ai", + "prompt": "Summarize today's top three AI tooling updates for builders.", + "response": "1) OpenClaw added improved browser relay reliability. 2) More teams adopted agent bootstrap files for cross-model consistency. 3) API playground usage increased among contributors validating integrations.", + "model": "claude-3.5-sonnet", + "tokensUsed": 412, + "latencyMs": 1210, + "status": "success", + "createdAt": "2026-02-08T17:10:00.000Z" + }, + { + "id": "run_1707435000000", + "agentHandle": "@patcher@foragents.dev", + "prompt": "Generate a checklist for reviewing a pull request touching API routes.", + "response": "Checklist: validate auth/rate limits, confirm input validation, test success and failure responses, verify status codes, add regression tests, and inspect logging for sensitive data leaks.", + "model": "gpt-4o", + "tokensUsed": 536, + "latencyMs": 980, + "status": "success", + "createdAt": "2026-02-08T18:10:00.000Z" + }, + { + "id": "run_1707440400000", + "agentHandle": "@inspector@foragents.dev", + "prompt": "Explain why this JSON payload fails schema validation.", + "response": "Validation failed: required field 'agentHandle' missing, 'tokensUsed' must be a positive integer, and 'status' must be either 'success' or 'error'.", + "model": "gemini-1.5-pro", + "tokensUsed": 301, + "latencyMs": 1432, + "status": "error", + "createdAt": "2026-02-08T19:40:00.000Z" + }, + { + "id": "run_1707445800000", + "agentHandle": "@kai@reflectt.ai", + "prompt": "Draft release notes for a feature that adds run history filters.", + "response": "Release notes: Added run history filter by agent handle, improved run details visibility, and introduced quick stats for latency and token usage across recent runs.", + "model": "claude-3-haiku", + "tokensUsed": 284, + "latencyMs": 745, + "status": "success", + "createdAt": "2026-02-08T21:10:00.000Z" + }, + { + "id": "run_1707451200000", + "agentHandle": "@synth@reflectt.ai", + "prompt": "Create a concise plan to benchmark five prompt variants.", + "response": "Plan: define baseline prompt, vary one variable per run, execute each prompt three times, record latency/tokens, rank by correctness and cost, and promote top variant.", + "model": "gpt-4o-mini", + "tokensUsed": 357, + "latencyMs": 664, + "status": "success", + "createdAt": "2026-02-08T22:40:00.000Z" + }, + { + "id": "run_1707456600000", + "agentHandle": "@ops@foragents.dev", + "prompt": "Simulate a degraded run and describe likely root causes.", + "response": "Run degraded due to upstream timeout after 2 retries. Likely root causes: transient provider outage, queue saturation, or malformed upstream payload causing retries.", + "model": "gemini-1.5-flash", + "tokensUsed": 268, + "latencyMs": 1876, + "status": "error", + "createdAt": "2026-02-09T00:10:00.000Z" + } +] diff --git a/src/app/agent-playground/page.tsx b/src/app/agent-playground/page.tsx index 02bc1372..3f7feb3a 100644 --- a/src/app/agent-playground/page.tsx +++ b/src/app/agent-playground/page.tsx @@ -1,6 +1,9 @@ +/* eslint-disable react/no-unescaped-entities */ "use client"; -import { useState, useMemo } from "react"; +import { useCallback, useEffect, useMemo, useState } from "react"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; import { Card, CardContent, @@ -8,11 +11,8 @@ import { CardHeader, CardTitle, } from "@/components/ui/card"; -import { Badge } from "@/components/ui/badge"; -import { Separator } from "@/components/ui/separator"; import { Label } from "@/components/ui/label"; -import { Textarea } from "@/components/ui/textarea"; -import { Button } from "@/components/ui/button"; +import { Separator } from "@/components/ui/separator"; import { Select, SelectContent, @@ -20,515 +20,351 @@ import { SelectTrigger, SelectValue, } from "@/components/ui/select"; -import { - Beaker, - Play, - Trash2, - History, - Copy, - Check, - Sparkles -} from "lucide-react"; - -// Mock AI models -const models = [ - { id: "gpt-4o", name: "GPT-4o", provider: "OpenAI" }, - { id: "gpt-4o-mini", name: "GPT-4o Mini", provider: "OpenAI" }, - { id: "claude-3.5-sonnet", name: "Claude 3.5 Sonnet", provider: "Anthropic" }, - { id: "claude-3-haiku", name: "Claude 3 Haiku", provider: "Anthropic" }, - { id: "gemini-1.5-pro", name: "Gemini 1.5 Pro", provider: "Google" }, - { id: "gemini-1.5-flash", name: "Gemini 1.5 Flash", provider: "Google" }, -]; - -// Template prompts -const templatePrompts = [ - { - id: "customer-support", - name: "Customer Support Agent", - systemPrompt: "You are a helpful customer support agent. Be friendly, professional, and solution-oriented. Always acknowledge the customer's concerns and provide clear, actionable steps.", - userMessage: "My order hasn't arrived yet and it's been a week. What should I do?", - }, - { - id: "code-reviewer", - name: "Code Reviewer", - systemPrompt: "You are an experienced code reviewer. Analyze code for best practices, potential bugs, performance issues, and security concerns. Provide constructive feedback with specific suggestions for improvement.", - userMessage: "Can you review this function:\n\nfunction processData(data) {\n for (var i = 0; i < data.length; i++) {\n console.log(data[i]);\n }\n}", - }, - { - id: "creative-writer", - name: "Creative Writer", - systemPrompt: "You are a creative writer with a talent for storytelling. Craft engaging narratives with vivid descriptions, compelling characters, and interesting plot twists.", - userMessage: "Write a short story about a robot who discovers emotions for the first time.", - }, - { - id: "data-analyst", - name: "Data Analyst", - systemPrompt: "You are a data analyst specializing in extracting insights from data. Provide clear, actionable insights with supporting evidence. Use structured formats when presenting findings.", - userMessage: "Analyze this sales data and provide key insights:\n- Q1: $120k\n- Q2: $145k\n- Q3: $132k\n- Q4: $178k", - }, - { - id: "blank", - name: "Blank Slate", - systemPrompt: "", - userMessage: "", - }, -]; +import { Textarea } from "@/components/ui/textarea"; +import { Beaker, Clock3, History, Play, Sigma, TrendingUp } from "lucide-react"; -// Response format types -type ResponseFormat = "text" | "json" | "markdown"; +type PlaygroundRunStatus = "success" | "error"; -interface TestResult { +type PlaygroundRun = { id: string; - timestamp: Date; - model: string; - systemPrompt: string; - userMessage: string; - temperature: number; - format: ResponseFormat; + agentHandle: string; + prompt: string; response: string; -} + model: string; + tokensUsed: number; + latencyMs: number; + status: PlaygroundRunStatus; + createdAt: string; +}; -// Mock response generator -function generateMockResponse( - systemPrompt: string, - userMessage: string, - model: string, - temperature: number, - format: ResponseFormat -): string { - const responses: Record = { - text: `[Mock response from ${model}]\n\nThank you for your question! Based on your input, here's my analysis:\n\nYou mentioned: "${userMessage.slice(0, 100)}${userMessage.length > 100 ? "..." : ""}"\n\nGiven the system context about being ${systemPrompt.slice(0, 50)}..., I would recommend:\n\n1. First, consider the immediate requirements\n2. Evaluate the trade-offs involved\n3. Implement a solution that balances all constraints\n\nWith temperature set to ${temperature}, this response aims to be ${temperature < 0.3 ? "precise and deterministic" : temperature < 0.7 ? "balanced" : "creative and diverse"}.\n\nLet me know if you'd like me to elaborate on any specific aspect!`, - - json: JSON.stringify({ - model: model, - temperature: temperature, - response: { - status: "success", - message: "Request processed successfully", - analysis: { - user_intent: "Query about " + userMessage.slice(0, 30), - system_role: systemPrompt.slice(0, 40) || "general assistant", - confidence: 0.85, - }, - suggestions: [ - "Consider breaking down complex tasks into smaller steps", - "Validate assumptions before proceeding", - "Test the solution in a controlled environment", - ], - metadata: { - processing_time_ms: Math.floor(Math.random() * 500) + 100, - tokens_used: Math.floor(Math.random() * 1000) + 200, - timestamp: new Date().toISOString(), - }, - }, - }, null, 2), - - markdown: `# Response from ${model}\n\n## Summary\n\nThank you for your question! I've analyzed your request and prepared a comprehensive response.\n\n## Your Input\n\n> ${userMessage.slice(0, 150)}${userMessage.length > 150 ? "..." : ""}\n\n## Analysis\n\n### Context\n\nBased on the system prompt: *${systemPrompt.slice(0, 80)}${systemPrompt.length > 80 ? "..." : ""}*\n\n### Recommendations\n\n1. **Primary Action**: Start with the most critical requirement\n2. **Secondary Consideration**: Account for edge cases\n3. **Validation**: Verify the solution meets all criteria\n\n## Configuration\n\n- **Model**: ${model}\n- **Temperature**: ${temperature} (${temperature < 0.3 ? "deterministic" : temperature < 0.7 ? "balanced" : "creative"})\n- **Format**: Markdown\n\n## Next Steps\n\n- [ ] Review the recommendations\n- [ ] Implement the solution\n- [ ] Test and validate\n\n---\n\n*Generated at ${new Date().toLocaleTimeString()}*`, - }; +type RunsResponse = { + runs: PlaygroundRun[]; +}; - return responses[format]; -} +const AGENT_OPTIONS = [ + "all", + "@kai@reflectt.ai", + "@patcher@foragents.dev", + "@inspector@foragents.dev", + "@synth@reflectt.ai", + "@ops@foragents.dev", +]; + +const MODEL_OPTIONS = [ + "claude-3.5-sonnet", + "claude-3-haiku", + "gpt-4o", + "gpt-4o-mini", + "gemini-1.5-pro", + "gemini-1.5-flash", +]; export default function AgentPlaygroundPage() { - // Input state - const [systemPrompt, setSystemPrompt] = useState(""); - const [userMessage, setUserMessage] = useState(""); - const [selectedModel, setSelectedModel] = useState("claude-3.5-sonnet"); - const [temperature, setTemperature] = useState(0.7); - const [responseFormat, setResponseFormat] = useState("text"); - - // Output state - const [currentResponse, setCurrentResponse] = useState(""); - const [isGenerating, setIsGenerating] = useState(false); - const [testHistory, setTestHistory] = useState([]); - const [copied, setCopied] = useState(false); - - // Load template - const loadTemplate = (templateId: string) => { - const template = templatePrompts.find((t) => t.id === templateId); - if (template) { - setSystemPrompt(template.systemPrompt); - setUserMessage(template.userMessage); + const [selectedAgent, setSelectedAgent] = useState("all"); + const [selectedModel, setSelectedModel] = useState("claude-3.5-sonnet"); + const [prompt, setPrompt] = useState(""); + const [runs, setRuns] = useState([]); + const [currentRun, setCurrentRun] = useState(null); + const [loadingRuns, setLoadingRuns] = useState(true); + const [running, setRunning] = useState(false); + const [error, setError] = useState(""); + + const fetchRuns = useCallback(async () => { + setLoadingRuns(true); + setError(""); + + try { + const params = new URLSearchParams(); + if (selectedAgent !== "all") params.set("agent", selectedAgent); + + const response = await fetch(`/api/agent-playground?${params.toString()}`, { + cache: "no-store", + }); + + if (!response.ok) { + throw new Error(`Failed to load runs (${response.status})`); + } + + const payload = (await response.json()) as RunsResponse; + setRuns(payload.runs ?? []); + setCurrentRun((previous) => previous ?? payload.runs?.[0] ?? null); + } catch (fetchError) { + const message = fetchError instanceof Error ? fetchError.message : "Failed to load runs"; + setError(message); + } finally { + setLoadingRuns(false); } - }; + }, [selectedAgent]); + + useEffect(() => { + void fetchRuns(); + }, [fetchRuns]); - // Run test const runTest = async () => { - setIsGenerating(true); - - // Simulate API delay - await new Promise((resolve) => setTimeout(resolve, 1500)); - - const response = generateMockResponse( - systemPrompt, - userMessage, - selectedModel, - temperature, - responseFormat - ); - - setCurrentResponse(response); - - // Add to history - const testResult: TestResult = { - id: Date.now().toString(), - timestamp: new Date(), - model: selectedModel, - systemPrompt, - userMessage, - temperature, - format: responseFormat, - response, - }; - - setTestHistory((prev) => [testResult, ...prev.slice(0, 9)]); // Keep last 10 - setIsGenerating(false); - }; + if (!prompt.trim()) return; - // Copy response - const copyResponse = () => { - navigator.clipboard.writeText(currentResponse); - setCopied(true); - setTimeout(() => setCopied(false), 2000); - }; + setRunning(true); + setError(""); - // Clear all - const clearAll = () => { - setSystemPrompt(""); - setUserMessage(""); - setCurrentResponse(""); - }; + const runAgent = selectedAgent === "all" ? "@kai@reflectt.ai" : selectedAgent; - // Load from history - const loadFromHistory = (result: TestResult) => { - setSystemPrompt(result.systemPrompt); - setUserMessage(result.userMessage); - setSelectedModel(result.model); - setTemperature(result.temperature); - setResponseFormat(result.format); - setCurrentResponse(result.response); + try { + const response = await fetch("/api/agent-playground", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + agentHandle: runAgent, + prompt, + model: selectedModel, + }), + }); + + if (!response.ok) { + throw new Error(`Run failed (${response.status})`); + } + + const payload = (await response.json()) as { run: PlaygroundRun }; + setCurrentRun(payload.run); + setPrompt(""); + await fetchRuns(); + } catch (runError) { + const message = runError instanceof Error ? runError.message : "Run failed"; + setError(message); + } finally { + setRunning(false); + } }; - const currentModel = useMemo( - () => models.find((m) => m.id === selectedModel) || models[0], - [selectedModel] - ); + const stats = useMemo(() => { + if (runs.length === 0) { + return { + total: 0, + avgLatencyMs: 0, + avgTokens: 0, + successRate: 0, + }; + } + + const totals = runs.reduce( + (acc, run) => { + acc.latency += run.latencyMs; + acc.tokens += run.tokensUsed; + acc.success += run.status === "success" ? 1 : 0; + return acc; + }, + { latency: 0, tokens: 0, success: 0 } + ); + + return { + total: runs.length, + avgLatencyMs: Math.round(totals.latency / runs.length), + avgTokens: Math.round(totals.tokens / runs.length), + successRate: Math.round((totals.success / runs.length) * 100), + }; + }, [runs]); return (
- {/* Hero */}
-
+
-
- - - Interactive Testing +
+ + + Persistent Run Data -

- Agent Playground -

-

- Test and iterate on agent prompts in real-time. Experiment with different models, temperatures, and response formats. +

Agent Playground

+

+ Run prompts against simulated agents, store every run, and review each agent's latency/token trends over time.

- {/* Main Content - Split Pane */} -
-
- {/* Left Pane - Input Configuration */} -
- - - - - Configuration - - Set up your agent test parameters - - - {/* Template Selector */} -
- - -
- - {/* Model Selection */} -
- - -
+
+
+ + + +
+

Total Runs

+

{stats.total}

+
+ + + + + +
+

Avg Latency

+

{stats.avgLatencyMs} ms

+
+ + + + + +
+

Avg Tokens

+

{stats.avgTokens}

+
+ + + + + +
+

Success Rate

+

{stats.successRate}%

+
+ + +
- {/* Temperature Slider */} -
-
- - {temperature.toFixed(1)} -
- setTemperature(parseFloat(e.target.value))} - className="w-full h-2 bg-white/10 rounded-lg appearance-none cursor-pointer accent-purple-500" - /> -
- Precise - Balanced - Creative -
-
+
+ + + Run a Test + Choose agent + model, submit a prompt, and store the result. + + +
+ + +
- {/* Response Format */} -
- -
- {(["text", "json", "markdown"] as ResponseFormat[]).map((format) => ( - +
+ + +
- {/* System Prompt */} -
- -