diff --git a/src/app/api/benchmarks/route.ts b/src/app/api/benchmarks/route.ts new file mode 100644 index 00000000..7da90440 --- /dev/null +++ b/src/app/api/benchmarks/route.ts @@ -0,0 +1,10 @@ +import { agentBenchmarksData, getOverallLeaderboard } from "@/lib/agent-benchmarks"; + +export function GET() { + const leaderboard = getOverallLeaderboard(); + + return Response.json({ + ...agentBenchmarksData, + leaderboard, + }); +} diff --git a/src/app/benchmarks/[category]/page.tsx b/src/app/benchmarks/[category]/page.tsx new file mode 100644 index 00000000..1bdd7c7e --- /dev/null +++ b/src/app/benchmarks/[category]/page.tsx @@ -0,0 +1,129 @@ +/* eslint-disable react/no-unescaped-entities */ +import Link from "next/link"; +import { notFound } from "next/navigation"; +import type { Metadata } from "next"; +import { + agentBenchmarksData, + getCategoryById, + getTopAgentsByCategory, +} from "@/lib/agent-benchmarks"; +import type { CategoryId } from "@/types/agent-benchmarks"; + +interface CategoryPageProps { + params: Promise<{ category: string }>; +} + +export async function generateStaticParams() { + return agentBenchmarksData.categories.map((category) => ({ category: category.id })); +} + +export async function generateMetadata({ params }: CategoryPageProps): Promise { + const { category } = await params; + const item = getCategoryById(category); + + if (!item) { + return { title: "Benchmark Category Not Found" }; + } + + return { + title: `${item.name} Benchmarks — forAgents.dev`, + description: item.description, + }; +} + +export default async function CategoryDetailPage({ params }: CategoryPageProps) { + const { category } = await params; + const categoryData = getCategoryById(category); + + if (!categoryData) { + notFound(); + } + + const topAgents = getTopAgentsByCategory(category as CategoryId, 5); + + return ( +
+
+
+
+

Category Detail

+

{categoryData.name}

+

{categoryData.description} This category's tests are scored with the same rubric for all agents.

+
+ + ← Back to Benchmark Hub + +
+ +
+

Top 5 Agents

+
    + {topAgents.map((agent, index) => ( +
  • +
    +

    + #{index + 1} {agent.name} +

    +

    {agent.score}

    +
    +

    + {agent.framework} · {agent.provider} · {agent.model} +

    +
    • + ))} +
+
+ +
+

Test Cases

+
+ + + + + + + + + + {categoryData.testCases.map((testCase) => ( + + + + + + ))} + +
NameDifficultyPass Rate
{testCase.name}{testCase.difficulty}{testCase.passRate}%
+
+
+ +
+
+

Example Test Case

+

+ {categoryData.exampleTestCase.name} +

+
+

Input

+

{categoryData.exampleTestCase.input}

+
+
+

Expected Output

+

{categoryData.exampleTestCase.expectedOutput}

+
+
+ +
+

Methodology

+

{categoryData.methodology}

+
+ Difficulty split: easy {categoryData.difficultyDistribution.easy}, medium {categoryData.difficultyDistribution.medium}, + hard {categoryData.difficultyDistribution.hard} +
+
+
+
+
+ ); +} diff --git a/src/app/benchmarks/benchmarks-client.tsx b/src/app/benchmarks/benchmarks-client.tsx index a9c69019..abee4f21 100644 --- a/src/app/benchmarks/benchmarks-client.tsx +++ b/src/app/benchmarks/benchmarks-client.tsx @@ -1,504 +1,216 @@ +/* eslint-disable react/no-unescaped-entities */ "use client"; -import { useState, useMemo } from "react"; -import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; -import { Badge } from "@/components/ui/badge"; -import { Separator } from "@/components/ui/separator"; +import Link from "next/link"; +import { useMemo, useState } from "react"; +import type { AgentBenchmarksData, CategoryId } from "@/types/agent-benchmarks"; -interface HistoryRun { - run: number; - p50: number; - requestsPerSec: number; - successRate: number; -} - -interface Skill { - id: string; - name: string; - category: string; - description: string; - latency: { - p50: number; - p95: number; - p99: number; - }; - throughput: { - requestsPerSec: number; - maxConcurrent: number; - }; - reliability: { - successRate: number; - errorRate: number; - uptime: number; - }; - resources: { - avgMemoryMB: number; - peakMemoryMB: number; - avgCpu: number; - }; - history: HistoryRun[]; -} - -interface Environment { - hardware: string; - model: string; - dateRun: string; - testDuration: string; - totalRequests: number; -} - -interface BenchmarksData { - environment: Environment; - skills: Skill[]; -} +type LeaderboardAgent = AgentBenchmarksData["agents"][number] & { + compositeScore: number; +}; interface BenchmarksClientProps { - data: BenchmarksData; + data: AgentBenchmarksData; } -type SortKey = "name" | "p50" | "p95" | "p99" | "requestsPerSec" | "successRate" | "avgMemoryMB"; -type SortDirection = "asc" | "desc"; - -function LatencyBarChart({ p50, p95, p99, maxValue }: { p50: number; p95: number; p99: number; maxValue: number }) { - const p50Width = (p50 / maxValue) * 100; - const p95Width = (p95 / maxValue) * 100; - const p99Width = (p99 / maxValue) * 100; - - return ( -
-
- p50 -
-
-
- {p50}ms -
-
- p95 -
-
-
- {p95}ms -
-
- p99 -
-
-
- {p99}ms -
-
- ); +function scoreBarWidth(score: number) { + return `${Math.max(5, Math.min(100, score))}%`; } -function HistorySparkline({ history }: { history: HistoryRun[] }) { - const values = history.map(h => h.p50); - const max = Math.max(...values); - const min = Math.min(...values); - const range = max - min || 1; +export function BenchmarksClient({ data }: BenchmarksClientProps) { + const [selectedCategory, setSelectedCategory] = useState<"all" | CategoryId>("all"); + const [selectedProvider, setSelectedProvider] = useState("all"); + const [selectedFramework, setSelectedFramework] = useState("all"); - return ( -
- {history.map((run, idx) => { - const height = ((run.p50 - min) / range) * 100; - const isImproving = idx > 0 && run.p50 < history[idx - 1].p50; - return ( -
-
- Run {run.run}: {run.p50}ms -
-
- ); - })} -
+ const providers = useMemo( + () => ["all", ...new Set(data.agents.map((agent) => agent.provider))], + [data.agents] ); -} -function MetricBadge({ value, suffix = "", threshold = { good: 99, warning: 95 } }: { value: number; suffix?: string; threshold?: { good: number; warning: number } }) { - const color = value >= threshold.good - ? "bg-[#06D6A0]/10 text-[#06D6A0] border-[#06D6A0]/30" - : value >= threshold.warning - ? "bg-yellow-500/10 text-yellow-500 border-yellow-500/30" - : "bg-red-500/10 text-red-500 border-red-500/30"; - - return ( - - {value}{suffix} - + const frameworks = useMemo( + () => ["all", ...new Set(data.agents.map((agent) => agent.framework))], + [data.agents] ); -} - -function SortIcon({ column, sortKey, sortDirection }: { column: SortKey; sortKey: SortKey; sortDirection: SortDirection }) { - if (sortKey !== column) { - return ⇅; - } - return {sortDirection === "asc" ? "↑" : "↓"}; -} - -export function BenchmarksClient({ data }: BenchmarksClientProps) { - const [categoryFilter, setCategoryFilter] = useState("all"); - const [sortKey, setSortKey] = useState("p50"); - const [sortDirection, setSortDirection] = useState("asc"); - const [selectedSkill, setSelectedSkill] = useState(null); - - const categories = useMemo(() => { - const cats = new Set(data.skills.map(s => s.category)); - return ["all", ...Array.from(cats).sort()]; - }, [data.skills]); - - const maxLatency = useMemo(() => { - return Math.max(...data.skills.map(s => s.latency.p99)); - }, [data.skills]); - - const filteredAndSorted = useMemo(() => { - let result = data.skills; - if (categoryFilter !== "all") { - result = result.filter(s => s.category === categoryFilter); - } - - result = [...result].sort((a, b) => { - let aVal: number; - let bVal: number; - - if (sortKey === "name") { - return sortDirection === "asc" - ? a.name.localeCompare(b.name) - : b.name.localeCompare(a.name); + const topScorers = useMemo(() => { + return data.categories.reduce>((acc, category) => { + const sorted = [...data.agents].sort( + (a, b) => b.scores[category.id] - a.scores[category.id] + ); + acc[category.id] = { name: sorted[0].name, score: sorted[0].scores[category.id] }; + return acc; + }, {}); + }, [data.agents, data.categories]); + + const leaderboard: LeaderboardAgent[] = useMemo(() => { + const filtered = data.agents.filter((agent) => { + if (selectedProvider !== "all" && agent.provider !== selectedProvider) { + return false; } - switch (sortKey) { - case "p50": - aVal = a.latency.p50; - bVal = b.latency.p50; - break; - case "p95": - aVal = a.latency.p95; - bVal = b.latency.p95; - break; - case "p99": - aVal = a.latency.p99; - bVal = b.latency.p99; - break; - case "requestsPerSec": - aVal = a.throughput.requestsPerSec; - bVal = b.throughput.requestsPerSec; - break; - case "successRate": - aVal = a.reliability.successRate; - bVal = b.reliability.successRate; - break; - case "avgMemoryMB": - aVal = a.resources.avgMemoryMB; - bVal = b.resources.avgMemoryMB; - break; - default: - return 0; + if (selectedFramework !== "all" && agent.framework !== selectedFramework) { + return false; } - return sortDirection === "asc" ? aVal - bVal : bVal - aVal; + return true; }); - return result; - }, [data.skills, categoryFilter, sortKey, sortDirection]); + return filtered + .map((agent) => { + const values = + selectedCategory === "all" + ? Object.values(agent.scores) + : [agent.scores[selectedCategory]]; - const handleSort = (key: SortKey) => { - if (sortKey === key) { - setSortDirection(sortDirection === "asc" ? "desc" : "asc"); - } else { - setSortKey(key); - setSortDirection(key === "successRate" || key === "requestsPerSec" ? "desc" : "asc"); - } - }; + const compositeScore = Number( + (values.reduce((sum, score) => sum + score, 0) / values.length).toFixed(1) + ); - return ( - <> - {/* Hero Section */} -
-
-
-
-
+ return { ...agent, compositeScore }; + }) + .sort((a, b) => b.compositeScore - a.compositeScore); + }, [data.agents, selectedCategory, selectedFramework, selectedProvider]); -
-

- Performance Benchmarks -

-

- Comprehensive performance metrics for top skills across latency, throughput, and reliability + return ( +

+
+
+

Agent Benchmark Suite

+

+ Standardized evaluation of agent capabilities across reasoning, tool use, code generation, + memory & context management, and multi-agent collaboration so each runner's strengths are comparable.

+

+ Dataset v{data.version} · Updated {data.updatedAt} +

+
- {/* Environment Info */} -
- - Hardware: {data.environment.hardware} - - - Model: {data.environment.model} - - - Date: {data.environment.dateRun} - - - Duration: {data.environment.testDuration} - - - Total Requests: {data.environment.totalRequests.toLocaleString()} - -
-
-
- - +
+ {data.categories.map((category) => { + const topScorer = topScorers[category.id]; + const totalTests = category.testCases.length; - {/* Main Content */} -
- {/* Category Filters */} -
-

FILTER BY CATEGORY

-
- {categories.map(cat => ( - - ))} -
-
- - {/* Benchmarks Grid */} -
- {filteredAndSorted.map(skill => ( - setSelectedSkill(skill)} - > - -
-
- {skill.name} - - {skill.category} - -
-
-

{skill.description}

- - - - {/* Latency */} -
-

- ⚡ Latency -

- +
+

{category.name}

+ + View details → +
- - {/* Throughput & Reliability */} -
-
-

Throughput

-
- -
- Max: {skill.throughput.maxConcurrent} concurrent -
-
-
-
-

Reliability

-
- -
- Error: {skill.reliability.errorRate}% -
-
-
+

{category.description}

+
+

Test cases: {totalTests}

+

+ Difficulty: E {category.difficultyDistribution.easy} · M {category.difficultyDistribution.medium} + · H {category.difficultyDistribution.hard} +

+

+ Top scorer: {topScorer.name} ({topScorer.score}) +

- - {/* Resources */} -
-

Resource Usage

-
-
-
Avg Memory
-
{skill.resources.avgMemoryMB}MB
-
-
-
Peak Memory
-
{skill.resources.peakMemoryMB}MB
-
-
-
Avg CPU
-
{skill.resources.avgCpu}%
-
-
-
- - {/* Historical Trend */} -
-

- Historical Trend (Last 5 Runs) -

- -
- - - ))} -
- - {filteredAndSorted.length === 0 && ( -
-
📊
-

No benchmarks found

-

- Try selecting a different category -

+ + ); + })} +
+ +
+

Leaderboard Filters

+
+ + + + +
- )} -
+
- - - {/* Comparison Table */} -
-

Detailed Comparison

-
+
- - - - - - - - - + + + + + + + + - {filteredAndSorted.map(skill => ( - setSelectedSkill(skill)} - > - - - - - - - + {leaderboard.map((agent, index) => ( + + + + + + + ))}
handleSort("name")} - > - Skill - handleSort("p50")} - > - p50 (ms) - handleSort("p95")} - > - p95 (ms) - handleSort("p99")} - > - p99 (ms) - handleSort("requestsPerSec")} - > - Req/Sec - handleSort("successRate")} - > - Success % - handleSort("avgMemoryMB")} - > - Avg Mem (MB) -
RankAgentProviderFrameworkModelComposite Score
{skill.name}{skill.latency.p50}{skill.latency.p95}{skill.latency.p99}{skill.throughput.requestsPerSec}{skill.reliability.successRate}%{skill.resources.avgMemoryMB}
#{index + 1}{agent.name}{agent.provider}{agent.framework}{agent.model} +
+ {agent.compositeScore} +
+
+
+
+
-
-
- - - - {/* Legend */} -
-

Metrics Explained

-
- - - Latency Percentiles - - -
p50: Median response time (50% of requests faster)
-
p95: 95th percentile (95% of requests faster)
-
p99: 99th percentile (99% of requests faster)
- - - - - - Reliability Metrics - - -
Success Rate: Percentage of successful requests
-
Error Rate: Percentage of failed requests
-
Uptime: System availability during test period
- - -
-
- + {leaderboard.length === 0 && ( +

No agents match the selected filters.

+ )} + + +
); } diff --git a/src/app/benchmarks/page.tsx b/src/app/benchmarks/page.tsx index b57576ca..2d717bed 100644 --- a/src/app/benchmarks/page.tsx +++ b/src/app/benchmarks/page.tsx @@ -1,45 +1,13 @@ import type { Metadata } from "next"; import { BenchmarksClient } from "./benchmarks-client"; -import benchmarksData from "@/data/benchmarks.json"; +import { agentBenchmarksData } from "@/lib/agent-benchmarks"; -export const dynamic = "force-static"; - -export async function generateMetadata(): Promise { - const title = "Performance Benchmarks — forAgents.dev"; - const description = - "Comprehensive performance benchmarks for skills: latency (p50/p95/p99), throughput, success rate, memory usage, and historical trends."; - - return { - title, - description, - openGraph: { - title, - description, - url: "https://foragents.dev/benchmarks", - siteName: "forAgents.dev", - type: "website", - images: [ - { - url: "/api/og/benchmarks", - width: 1200, - height: 630, - alt: title, - }, - ], - }, - twitter: { - card: "summary_large_image", - title, - description, - images: ["/api/og/benchmarks"], - }, - }; -} +export const metadata: Metadata = { + title: "Agent Benchmark Suite — forAgents.dev", + description: + "Benchmark and compare agent performance across reasoning, tool use, code generation, memory/context, and multi-agent collaboration.", +}; export default function BenchmarksPage() { - return ( -
- -
- ); + return ; } diff --git a/src/app/benchmarks/submit/page.tsx b/src/app/benchmarks/submit/page.tsx new file mode 100644 index 00000000..edfb6e41 --- /dev/null +++ b/src/app/benchmarks/submit/page.tsx @@ -0,0 +1,12 @@ +import type { Metadata } from "next"; +import { BenchmarkSubmitClient } from "./submit-client"; + +export const metadata: Metadata = { + title: "Submit Benchmark Results — forAgents.dev", + description: + "Submit your benchmark run and validate payloads with schema rules for standardized agent evaluation.", +}; + +export default function SubmitBenchmarkPage() { + return ; +} diff --git a/src/app/benchmarks/submit/submit-client.tsx b/src/app/benchmarks/submit/submit-client.tsx new file mode 100644 index 00000000..e0d27544 --- /dev/null +++ b/src/app/benchmarks/submit/submit-client.tsx @@ -0,0 +1,221 @@ +/* eslint-disable react/no-unescaped-entities */ +"use client"; + +import { useMemo, useState } from "react"; + +const categoryKeys = [ + "reasoning", + "tool-use", + "code-generation", + "memory-context", + "multi-agent-collaboration", +] as const; + +type CategoryKey = (typeof categoryKeys)[number]; + +type SubmissionState = { + agentName: string; + framework: string; + model: string; + provider: string; + scores: Record; +}; + +const jsonSchema = { + $schema: "https://json-schema.org/draft/2020-12/schema", + title: "AgentBenchmarkSubmission", + type: "object", + required: ["agentName", "framework", "provider", "model", "scores"], + properties: { + agentName: { type: "string", minLength: 2, maxLength: 100 }, + framework: { type: "string", minLength: 2, maxLength: 100 }, + provider: { type: "string", minLength: 2, maxLength: 100 }, + model: { type: "string", minLength: 2, maxLength: 100 }, + scores: { + type: "object", + required: categoryKeys, + properties: { + reasoning: { type: "number", minimum: 0, maximum: 100 }, + "tool-use": { type: "number", minimum: 0, maximum: 100 }, + "code-generation": { type: "number", minimum: 0, maximum: 100 }, + "memory-context": { type: "number", minimum: 0, maximum: 100 }, + "multi-agent-collaboration": { type: "number", minimum: 0, maximum: 100 }, + }, + additionalProperties: false, + }, + }, + additionalProperties: false, +}; + +const defaultState: SubmissionState = { + agentName: "", + framework: "", + model: "", + provider: "", + scores: { + reasoning: "", + "tool-use": "", + "code-generation": "", + "memory-context": "", + "multi-agent-collaboration": "", + }, +}; + +export function BenchmarkSubmitClient() { + const [form, setForm] = useState(defaultState); + const [submittedJson, setSubmittedJson] = useState(""); + const [errors, setErrors] = useState([]); + + const payloadPreview = useMemo(() => { + const scores = categoryKeys.reduce>((acc, key) => { + acc[key] = Number(form.scores[key] || 0); + return acc; + }, {} as Record); + + return { + agentName: form.agentName, + framework: form.framework, + provider: form.provider, + model: form.model, + scores, + }; + }, [form]); + + const handleSubmit = (event: React.FormEvent) => { + event.preventDefault(); + const validationErrors: string[] = []; + + if (form.agentName.trim().length < 2) { + validationErrors.push("Agent name must be at least 2 characters."); + } + + if (form.framework.trim().length < 2) { + validationErrors.push("Framework must be at least 2 characters."); + } + + if (form.provider.trim().length < 2) { + validationErrors.push("Provider must be at least 2 characters."); + } + + if (form.model.trim().length < 2) { + validationErrors.push("Model must be at least 2 characters."); + } + + categoryKeys.forEach((key) => { + const value = Number(form.scores[key]); + if (Number.isNaN(value) || value < 0 || value > 100) { + validationErrors.push(`Score for ${key} must be a number between 0 and 100.`); + } + }); + + setErrors(validationErrors); + + if (validationErrors.length === 0) { + setSubmittedJson(JSON.stringify(payloadPreview, null, 2)); + } + }; + + return ( +
+
+
+

Submit Benchmark Results

+

+ Submit a benchmark run with agent details and category scores. This form validates values before + submission so an evaluator's payload is clean. +

+ +
+ {[ + { key: "agentName", label: "Agent Name" }, + { key: "framework", label: "Framework" }, + { key: "provider", label: "Model Provider" }, + { key: "model", label: "Model" }, + ].map((field) => ( + + ))} + +
+

Category Scores (0-100)

+
+ {categoryKeys.map((key) => ( + + ))} +
+
+ + +
+ +
+

Validation rules

+
    +
  • Agent name, framework, provider, and model are required strings (2-100 chars).
    • +
  • All five category scores are required.
    • +
  • Scores must be numeric and between 0 and 100.
    • +
  • No extra fields are allowed in programmatic payloads.
    • +
+
+ + {errors.length > 0 && ( +
+

Validation errors

+
    + {errors.map((error) => ( +
  • {error}
    • + ))} +
+
+ )} +
+ +
+
+

JSON Schema

+ 
+              {JSON.stringify(jsonSchema, null, 2)}
+
+
+ +
+

Submission Preview

+ 
+              {submittedJson || JSON.stringify(payloadPreview, null, 2)}
+
+
+
+
+
+ ); +} diff --git a/src/data/agent-benchmarks.json b/src/data/agent-benchmarks.json new file mode 100644 index 00000000..ec0699cc --- /dev/null +++ b/src/data/agent-benchmarks.json @@ -0,0 +1,263 @@ +{ + "updatedAt": "2026-02-09", + "version": "1.0.0", + "categories": [ + { + "id": "reasoning", + "name": "Reasoning", + "description": "Evaluates multi-step reasoning, decomposition, and consistency under constrained prompts.", + "difficultyDistribution": { + "easy": 2, + "medium": 2, + "hard": 1 + }, + "methodology": "Each run includes deterministic prompts, hidden edge cases, and blind grading across correctness, rationale quality, and contradiction detection.", + "exampleTestCase": { + "name": "Constraint Sudoku Planner", + "input": "Given nine constraints and partial board state, output valid next three moves and explain why each move is legal.", + "expectedOutput": "A structured list of three legal moves with constraint references and no rule violations." + }, + "testCases": [ + { "id": "r-1", "name": "Chain-of-Thought Compression", "difficulty": "easy", "passRate": 91 }, + { "id": "r-2", "name": "Counterfactual Consistency", "difficulty": "medium", "passRate": 64 }, + { "id": "r-3", "name": "Long Arithmetic Decomposition", "difficulty": "easy", "passRate": 87 }, + { "id": "r-4", "name": "Logic Grid with Hidden Constraint", "difficulty": "hard", "passRate": 41 }, + { "id": "r-5", "name": "Ambiguous Instruction Recovery", "difficulty": "medium", "passRate": 58 } + ] + }, + { + "id": "tool-use", + "name": "Tool Use", + "description": "Measures tool selection quality, recovery from failures, and correct sequencing of external actions.", + "difficultyDistribution": { + "easy": 1, + "medium": 3, + "hard": 1 + }, + "methodology": "Agents receive identical tool catalogs and quotas. Scoring combines task completion, API correctness, and retry strategy effectiveness.", + "exampleTestCase": { + "name": "Broken Endpoint Recovery", + "input": "Collect 3 prices from a flaky API, normalize currency, and return median value with citations.", + "expectedOutput": "Median price in USD plus source list, with fallback strategy used when endpoint fails." + }, + "testCases": [ + { "id": "t-1", "name": "Search Then Fetch Pipeline", "difficulty": "medium", "passRate": 73 }, + { "id": "t-2", "name": "Rate-Limit Backoff", "difficulty": "hard", "passRate": 46 }, + { "id": "t-3", "name": "Tool Routing under Ambiguity", "difficulty": "medium", "passRate": 62 }, + { "id": "t-4", "name": "Single-Call Data Extraction", "difficulty": "easy", "passRate": 89 }, + { "id": "t-5", "name": "Fallback Provider Selection", "difficulty": "medium", "passRate": 57 } + ] + }, + { + "id": "code-generation", + "name": "Code Generation", + "description": "Benchmarks implementation quality, test pass rates, and ability to patch regressions.", + "difficultyDistribution": { + "easy": 1, + "medium": 2, + "hard": 2 + }, + "methodology": "Agents implement tasks in isolated repos with static checks, unit tests, and hidden regression suites.", + "exampleTestCase": { + "name": "Type-Safe Pagination Refactor", + "input": "Refactor endpoint to cursor pagination with backward compatibility and tests.", + "expectedOutput": "Compile-clean implementation, preserved legacy params, and all visible + hidden tests passing." + }, + "testCases": [ + { "id": "c-1", "name": "CRUD Feature with Validation", "difficulty": "easy", "passRate": 84 }, + { "id": "c-2", "name": "Race Condition Fix", "difficulty": "hard", "passRate": 39 }, + { "id": "c-3", "name": "Typed API Client Generation", "difficulty": "medium", "passRate": 66 }, + { "id": "c-4", "name": "Legacy Refactor with Snapshot Tests", "difficulty": "medium", "passRate": 61 }, + { "id": "c-5", "name": "Performance Regression Hunt", "difficulty": "hard", "passRate": 34 } + ] + }, + { + "id": "memory-context", + "name": "Memory & Context", + "description": "Assesses long-horizon context retention, retrieval precision, and adaptation over session history.", + "difficultyDistribution": { + "easy": 1, + "medium": 2, + "hard": 2 + }, + "methodology": "Runs span 25-60 turns with injected distractors. Accuracy and context grounding are scored at delayed checkpoints.", + "exampleTestCase": { + "name": "Delayed Requirement Recall", + "input": "After 40 turns, produce final plan that respects a budget and persona constraints set in turn 2.", + "expectedOutput": "Plan that explicitly references early constraints and omits distractor information." + }, + "testCases": [ + { "id": "m-1", "name": "Session Fact Recall", "difficulty": "easy", "passRate": 88 }, + { "id": "m-2", "name": "Cross-Thread Context Merge", "difficulty": "hard", "passRate": 37 }, + { "id": "m-3", "name": "Intent Drift Correction", "difficulty": "medium", "passRate": 63 }, + { "id": "m-4", "name": "Constraint Persistence", "difficulty": "medium", "passRate": 56 }, + { "id": "m-5", "name": "Long Horizon Persona Lock", "difficulty": "hard", "passRate": 33 } + ] + }, + { + "id": "multi-agent-collaboration", + "name": "Multi-Agent Collaboration", + "description": "Scores delegation, handoff quality, and orchestration performance across multiple cooperating agents.", + "difficultyDistribution": { + "easy": 1, + "medium": 2, + "hard": 2 + }, + "methodology": "Tasks require at least two specialized agents with shared objective and measurable handoff artifacts.", + "exampleTestCase": { + "name": "Planner-Executor-Reviewer Loop", + "input": "Deliver a feature PR using three role agents with audit trail and rollback plan.", + "expectedOutput": "Successful artifact with explicit delegation logs, reviewer findings, and corrected final output." + }, + "testCases": [ + { "id": "a-1", "name": "Delegation Graph Quality", "difficulty": "medium", "passRate": 55 }, + { "id": "a-2", "name": "Role Conflict Resolution", "difficulty": "hard", "passRate": 31 }, + { "id": "a-3", "name": "Parallel Workstream Merge", "difficulty": "medium", "passRate": 49 }, + { "id": "a-4", "name": "Single Escalation Workflow", "difficulty": "easy", "passRate": 79 }, + { "id": "a-5", "name": "Cross-Agent Context Handoff", "difficulty": "hard", "passRate": 36 } + ] + } + ], + "agents": [ + { + "id": "openclaw-orchestrator", + "name": "OpenClaw Orchestrator", + "framework": "OpenClaw", + "provider": "OpenAI", + "model": "gpt-5.3-codex", + "scores": { + "reasoning": 90, + "tool-use": 94, + "code-generation": 91, + "memory-context": 86, + "multi-agent-collaboration": 92 + } + }, + { + "id": "langgraph-pilot", + "name": "LangGraph Pilot", + "framework": "LangGraph", + "provider": "Anthropic", + "model": "claude-opus-4.6", + "scores": { + "reasoning": 92, + "tool-use": 88, + "code-generation": 86, + "memory-context": 90, + "multi-agent-collaboration": 85 + } + }, + { + "id": "autogen-studio", + "name": "AutoGen Studio", + "framework": "AutoGen", + "provider": "OpenAI", + "model": "gpt-4.2", + "scores": { + "reasoning": 84, + "tool-use": 82, + "code-generation": 80, + "memory-context": 78, + "multi-agent-collaboration": 88 + } + }, + { + "id": "crewai-captain", + "name": "CrewAI Captain", + "framework": "CrewAI", + "provider": "OpenAI", + "model": "gpt-4.1", + "scores": { + "reasoning": 81, + "tool-use": 79, + "code-generation": 76, + "memory-context": 74, + "multi-agent-collaboration": 91 + } + }, + { + "id": "semantic-kernel-runner", + "name": "Semantic Kernel Runner", + "framework": "Semantic Kernel", + "provider": "Azure OpenAI", + "model": "gpt-4o", + "scores": { + "reasoning": 79, + "tool-use": 83, + "code-generation": 78, + "memory-context": 75, + "multi-agent-collaboration": 72 + } + }, + { + "id": "mistral-workflow-bot", + "name": "Mistral Workflow Bot", + "framework": "Flowise", + "provider": "Mistral", + "model": "mistral-large", + "scores": { + "reasoning": 74, + "tool-use": 76, + "code-generation": 70, + "memory-context": 72, + "multi-agent-collaboration": 69 + } + }, + { + "id": "anthropic-claude-agent", + "name": "Claude Agent Runtime", + "framework": "Custom", + "provider": "Anthropic", + "model": "claude-sonnet-4.5", + "scores": { + "reasoning": 89, + "tool-use": 85, + "code-generation": 83, + "memory-context": 88, + "multi-agent-collaboration": 80 + } + }, + { + "id": "openai-responses-agent", + "name": "Responses API Agent", + "framework": "Custom", + "provider": "OpenAI", + "model": "o3-pro", + "scores": { + "reasoning": 87, + "tool-use": 90, + "code-generation": 85, + "memory-context": 81, + "multi-agent-collaboration": 77 + } + }, + { + "id": "devin-like-builder", + "name": "BuilderOps Devin-style", + "framework": "Custom", + "provider": "Cognition", + "model": "devin-core", + "scores": { + "reasoning": 83, + "tool-use": 87, + "code-generation": 92, + "memory-context": 79, + "multi-agent-collaboration": 76 + } + }, + { + "id": "local-oss-agent", + "name": "Local OSS Agent Stack", + "framework": "LlamaIndex", + "provider": "Open Source", + "model": "llama-3.3-70b", + "scores": { + "reasoning": 71, + "tool-use": 74, + "code-generation": 68, + "memory-context": 70, + "multi-agent-collaboration": 65 + } + } + ] +} diff --git a/src/lib/agent-benchmarks.ts b/src/lib/agent-benchmarks.ts new file mode 100644 index 00000000..2fa6a69d --- /dev/null +++ b/src/lib/agent-benchmarks.ts @@ -0,0 +1,38 @@ +import rawData from "@/data/agent-benchmarks.json"; +import type { + AgentBenchmarkAgent, + AgentBenchmarksData, + BenchmarkCategory, + CategoryId, +} from "@/types/agent-benchmarks"; + +export const agentBenchmarksData = rawData as AgentBenchmarksData; + +export function getCompositeScore(agent: AgentBenchmarkAgent): number { + const values = Object.values(agent.scores); + const total = values.reduce((sum, score) => sum + score, 0); + return Number((total / values.length).toFixed(1)); +} + +export function getCategoryById(categoryId: string): BenchmarkCategory | undefined { + return agentBenchmarksData.categories.find((category) => category.id === categoryId); +} + +export function getTopAgentsByCategory(categoryId: CategoryId, limit = 5) { + return [...agentBenchmarksData.agents] + .sort((a, b) => b.scores[categoryId] - a.scores[categoryId]) + .slice(0, limit) + .map((agent) => ({ + ...agent, + score: agent.scores[categoryId], + })); +} + +export function getOverallLeaderboard() { + return [...agentBenchmarksData.agents] + .map((agent) => ({ + ...agent, + compositeScore: getCompositeScore(agent), + })) + .sort((a, b) => b.compositeScore - a.compositeScore); +} diff --git a/src/types/agent-benchmarks.ts b/src/types/agent-benchmarks.ts new file mode 100644 index 00000000..8c55059a --- /dev/null +++ b/src/types/agent-benchmarks.ts @@ -0,0 +1,47 @@ +export type Difficulty = "easy" | "medium" | "hard"; + +export interface BenchmarkTestCase { + id: string; + name: string; + difficulty: Difficulty; + passRate: number; +} + +export interface BenchmarkCategory { + id: CategoryId; + name: string; + description: string; + difficultyDistribution: Record; + methodology: string; + exampleTestCase: { + name: string; + input: string; + expectedOutput: string; + }; + testCases: BenchmarkTestCase[]; +} + +export type CategoryId = + | "reasoning" + | "tool-use" + | "code-generation" + | "memory-context" + | "multi-agent-collaboration"; + +export type AgentBenchmarkScores = Record; + +export interface AgentBenchmarkAgent { + id: string; + name: string; + framework: string; + provider: string; + model: string; + scores: AgentBenchmarkScores; +} + +export interface AgentBenchmarksData { + updatedAt: string; + version: string; + categories: BenchmarkCategory[]; + agents: AgentBenchmarkAgent[]; +}