diff --git a/src/app/observability-guide/logging/page.tsx b/src/app/observability-guide/logging/page.tsx new file mode 100644 index 00000000..44e64b54 --- /dev/null +++ b/src/app/observability-guide/logging/page.tsx @@ -0,0 +1,714 @@ +"use client"; + +import { useState } from "react"; +import Link from "next/link"; +import { FileText, AlertCircle, Info, AlertTriangle, XOctagon } from "lucide-react"; + +interface LogExample { + title: string; + description: string; + level: "debug" | "info" | "warn" | "error"; + good: string; + bad?: string; +} + +const LOG_EXAMPLES: LogExample[] = [ + { + title: "Tool Call Execution", + description: "Log every tool invocation with parameters, duration, and result status", + level: "info", + good: `logger.info({ + correlationId: "req-abc123", + userId: "user-456", + action: "tool_call", + tool: "web_search", + params: { query: "best practices", maxResults: 5 }, + durationMs: 234, + status: "success", + resultSize: 1024, +}, "Tool call completed");`, + bad: `console.log("web search done");`, + }, + { + title: "Agent Decision Point", + description: "Log reasoning steps and why specific tools or approaches were chosen", + level: "info", + good: `logger.info({ + correlationId: "req-abc123", + action: "reasoning", + thought: "User wants weather, need location first", + selectedTool: "get_location", + alternatives: ["web_search", "ask_user"], + confidence: 0.92, +}, "Agent decision");`, + bad: `console.log("Choosing get_location tool");`, + }, + { + title: "Error Handling", + description: "Capture full error context including stack, retry attempts, and impact", + level: "error", + good: `logger.error({ + correlationId: "req-abc123", + action: "tool_call", + tool: "database_query", + error: { + name: error.name, + message: error.message, + stack: error.stack, + code: error.code, + }, + retryAttempt: 2, + maxRetries: 3, + userImpact: "Request failed, fallback to cached data", +}, "Tool execution failed");`, + bad: `console.error("DB query failed:", error);`, + }, + { + title: "Context Switch", + description: "Log when agent changes conversation context or workspace", + level: "info", + good: `logger.info({ + correlationId: "req-abc123", + action: "context_switch", + from: { workspace: "/project-a", sessionId: "sess-111" }, + to: { workspace: "/project-b", sessionId: "sess-222" }, + reason: "User explicitly requested project switch", + preservedContext: ["user_preferences", "auth_token"], +}, "Context switched");`, + bad: `console.log("Switched to project-b");`, + }, + { + title: "Token Usage", + description: "Track LLM token consumption for cost and performance analysis", + level: "info", + good: `logger.info({ + correlationId: "req-abc123", + action: "llm_call", + model: "gpt-4-turbo", + operation: "completion", + tokens: { + prompt: 1234, + completion: 567, + total: 1801, + }, + cost: 0.0234, // USD + durationMs: 1820, + cached: false, +}, "LLM request completed");`, + bad: `console.log("GPT-4 call done");`, + }, + { + title: "Security Event", + description: "Always log potential security violations for audit trails", + level: "warn", + good: `logger.warn({ + correlationId: "req-abc123", + userId: "user-789", + action: "security_violation", + type: "prompt_injection_detected", + input: sanitizedInput.substring(0, 200), // Truncated for privacy + patterns: ["ignore previous instructions"], + blocked: true, + severity: "high", +}, "Potential prompt injection blocked");`, + bad: `console.log("Blocked suspicious input");`, + }, +]; + +const LOG_LEVELS = [ + { + level: "DEBUG", + icon: Info, + color: "text-gray-600 bg-gray-100 border-gray-300", + when: "Development/troubleshooting only", + examples: [ + "Variable state changes", + "Loop iterations", + "Detailed function entry/exit", + "Interim computation results", + ], + production: false, + }, + { + level: "INFO", + icon: Info, + color: "text-blue-600 bg-blue-100 border-blue-300", + when: "Normal operations worth recording", + examples: [ + "Tool calls started/completed", + "Agent decisions and reasoning", + "Context switches", + "Session start/end", + ], + production: true, + }, + { + level: "WARN", + icon: AlertTriangle, + color: "text-yellow-600 bg-yellow-100 border-yellow-300", + when: "Unexpected but recoverable situations", + examples: [ + "Retry attempts after transient failures", + "Deprecated API usage", + "Approaching rate limits", + "Potential security issues (blocked)", + ], + production: true, + }, + { + level: "ERROR", + icon: XOctagon, + color: "text-red-600 bg-red-100 border-red-300", + when: "Failures that prevent operation completion", + examples: [ + "Unhandled exceptions", + "Tool execution failures", + "External API errors", + "Data validation failures", + ], + production: true, + }, +]; + +const CORRELATION_STRATEGIES = [ + { + strategy: "Request ID Propagation", + description: "Generate a unique ID for each user request and propagate it through all logs, traces, and metrics", + code: `import { randomUUID } from 'crypto'; +import { AsyncLocalStorage } from 'async_hooks'; + +const asyncLocalStorage = new AsyncLocalStorage(); + +// Middleware to create correlation context +app.use((req, res, next) => { + const correlationId = req.headers['x-correlation-id'] || randomUUID(); + asyncLocalStorage.run({ correlationId }, () => { + req.correlationId = correlationId; + res.setHeader('x-correlation-id', correlationId); + next(); + }); +}); + +// Logger automatically includes correlation ID +function createLogger() { + return { + info: (data, msg) => { + const context = asyncLocalStorage.getStore(); + console.log(JSON.stringify({ + ...data, + correlationId: context?.correlationId, + timestamp: new Date().toISOString(), + level: 'info', + message: msg, + })); + }, + }; +}`, + }, + { + strategy: "Trace Context Injection", + description: "Use OpenTelemetry to automatically inject trace IDs into logs for correlation", + code: `import { trace } from '@opentelemetry/api'; +import pino from 'pino'; + +const logger = pino({ + mixin: () => { + const span = trace.getActiveSpan(); + if (!span) return {}; + + const spanContext = span.spanContext(); + return { + traceId: spanContext.traceId, + spanId: spanContext.spanId, + traceFlags: spanContext.traceFlags, + }; + }, +}); + +// Now all logs automatically include trace/span IDs +logger.info({ action: 'tool_call', tool: 'search' }, 'Executing tool'); +// Output includes: { ..., traceId: "abc123...", spanId: "def456..." }`, + }, + { + strategy: "Hierarchical Context", + description: "Maintain parent-child relationships in logs for multi-agent or nested operations", + code: `class ExecutionContext { + constructor(parentId = null) { + this.id = randomUUID(); + this.parentId = parentId; + this.startTime = Date.now(); + } + + createChild() { + return new ExecutionContext(this.id); + } + + toLogContext() { + return { + executionId: this.id, + parentExecutionId: this.parentId, + executionDepth: this.getDepth(), + executionDurationMs: Date.now() - this.startTime, + }; + } +} + +// Usage +const rootCtx = new ExecutionContext(); +logger.info({ ...rootCtx.toLogContext(), action: 'main_task' }, 'Starting task'); + +const childCtx = rootCtx.createChild(); +logger.info({ ...childCtx.toLogContext(), action: 'sub_task' }, 'Delegated to sub-agent');`, + }, +]; + +const LOG_AGGREGATION = [ + { + name: "Structured Format", + description: "Always use JSON for logs. Never plain text.", + example: { + good: `{ "timestamp": "2026-02-09T21:00:00Z", "level": "info", "correlationId": "req-123", "action": "tool_call", "tool": "search", "durationMs": 234 }`, + bad: `[INFO] 2026-02-09 21:00:00 - Tool search completed in 234ms`, + }, + why: "JSON logs are machine-parseable, filterable, and can be queried in log aggregation tools. Plain text requires regex parsing.", + }, + { + name: "Standard Fields", + description: "Use consistent field names across all logs", + fields: [ + { name: "timestamp", type: "ISO8601 string", required: true }, + { name: "level", type: "debug|info|warn|error", required: true }, + { name: "correlationId", type: "string (UUID)", required: true }, + { name: "action", type: "string", required: false, description: "What happened (tool_call, reasoning, etc.)" }, + { name: "userId", type: "string", required: false }, + { name: "sessionId", type: "string", required: false }, + { name: "message", type: "string", required: true }, + ], + }, + { + name: "Sampling & Volume Control", + description: "High-volume logs (like debug) should be sampled in production", + code: `const shouldSample = (level: string, sampleRate: number) => { + if (level === 'error') return true; // Always log errors + if (level === 'debug' && process.env.NODE_ENV === 'production') { + return Math.random() < sampleRate; // Sample 1% of debug logs + } + return true; +}; + +logger.debug = (data, msg) => { + if (!shouldSample('debug', 0.01)) return; + // ... log normally +};`, + }, +]; + +const BEST_PRACTICES = [ + { + practice: "Log at Boundaries", + description: "Always log when crossing system boundaries (API calls, database queries, external tool invocations)", + why: "These are the most common failure points and debugging blind spots.", + }, + { + practice: "Avoid Logging Secrets", + description: "Never log API keys, passwords, tokens, or PII. Redact sensitive fields.", + code: `const redact = (obj: any) => { + const sensitive = ['password', 'apiKey', 'token', 'secret', 'ssn', 'creditCard']; + const redacted = { ...obj }; + for (const key of Object.keys(redacted)) { + if (sensitive.some(s => key.toLowerCase().includes(s))) { + redacted[key] = '[REDACTED]'; + } + } + return redacted; +}; + +logger.info(redact(userData), "User authenticated");`, + }, + { + practice: "Include Error Context", + description: "When logging errors, include the full context needed to reproduce the issue", + code: `try { + const result = await tool.execute(params); +} catch (error) { + logger.error({ + correlationId, + action: 'tool_execution', + tool: tool.name, + params: redact(params), // Params that caused the error + error: { + name: error.name, + message: error.message, + stack: error.stack, + }, + retryable: isRetryable(error), + userImpact: 'Request failed', + }, 'Tool execution failed'); + throw error; +}`, + }, + { + practice: "Use Log Levels Consistently", + description: "Don't use ERROR for warnings or INFO for errors. Be consistent across your codebase.", + why: "Alerts and dashboards depend on accurate log levels. Mis-leveled logs cause alert fatigue or missed incidents.", + }, + { + practice: "Make Logs Searchable", + description: "Use consistent field names and values that you can query in log aggregation tools", + examples: [ + "action: 'tool_call' (not 'executing tool', 'tool exec', 'calling tool')", + "status: 'success'|'failure' (not 'ok', 'done', 'error')", + ], + }, +]; + +export default function LoggingPage() { + const [selectedLevel, setSelectedLevel] = useState("all"); + + const filteredExamples = + selectedLevel === "all" ? LOG_EXAMPLES : LOG_EXAMPLES.filter((ex) => ex.level === selectedLevel); + + return ( +
+ {/* Header */} +
+ + ← Back to Observability Guide + +
+ +

Logging Best Practices

+
+

+ Structured logging strategies for AI agents. Learn what to log, how to format it, and how to make logs + actionable. +

+
+ + {/* What to Log */} +
+

What to Log in AI Agents

+
+

+ Golden Rule: Log everything needed to answer these questions: +

+
    +
  • + • + What did the agent decide to do? +
    • +
  • + • + Why did it make that decision? +
    • +
  • + • + Which tools were called and with what parameters? +
    • +
  • + • + What was the outcome (success, failure, partial)? +
    • +
  • + • + How long did each operation take? +
    • +
  • + • + What was the cost (tokens, API calls)? +
    • +
+
+ +
+
+

Log Examples by Category

+
+ + + + +
+
+ +
+ {filteredExamples.map((example) => ( +
+
+

{example.title}

+ + {example.level} + +
+

{example.description}

+ +
+ {example.bad && ( +
+
❌ Bad:
+ 
+                        {example.bad}
+
+
+ )} +
+
✅ Good:
+ 
+                      {example.good}
+
+
+
+
+ ))} +
+
+
+ + {/* Log Levels */} +
+

Log Levels: When to Use Each

+
+ {LOG_LEVELS.map((level) => { + const Icon = level.icon; + return ( +
+
+ +
+
+

{level.level}

+ {level.production ? ( + + Production enabled + + ) : ( + Dev only + )} +
+

+ When to use: {level.when} +

+
+

Examples:

+
    + {level.examples.map((ex) => ( +
  • + • + {ex} +
    • + ))} +
+
+
+
+
+ ); + })} +
+
+ + {/* Correlation IDs */} +
+

+ Correlation IDs: Tracing Requests End-to-End +

+
+

+ Why correlation IDs matter: Without them, you can't answer "What happened during this specific + user request?" when you have thousands of concurrent requests. +

+

+ A correlation ID (or request ID) is a unique identifier that connects all logs, metrics, and traces for a + single user request, even across multiple services or agents. +

+
+ +
+ {CORRELATION_STRATEGIES.map((strategy) => ( +
+

{strategy.strategy}

+

{strategy.description}

+ 
+                {strategy.code}
+
+
+ ))} +
+
+ + {/* Log Aggregation */} +
+

Log Aggregation & Structured Formats

+ + {LOG_AGGREGATION.map((topic, idx) => ( +
0 ? "mt-8" : ""}`}> +

{topic.name}

+

{topic.description}

+ + {topic.example && ( +
+
+
✅ Good (JSON):
+ 
+                    {topic.example.good}
+
+
+
+
❌ Bad (Plain text):
+ 
+                    {topic.example.bad}
+
+
+ {topic.why && ( +
+

+ Why: {topic.why} +

+
+ )} +
+ )} + + {topic.fields && ( +
+ + + + + + + + + + + {topic.fields.map((field) => ( + + + + + + + ))} + +
FieldTypeRequiredDescription
{field.name}{field.type} + {field.required ? ( + Yes + ) : ( + No + )} + {field.description || "—"}
+
+ )} + + {topic.code && ( +
+ 
+                  {topic.code}
+
+
+ )} +
+ ))} +
+ + {/* Best Practices */} +
+

Best Practices Checklist

+
+ {BEST_PRACTICES.map((item) => ( +
+

{item.practice}

+

{item.description}

+ {item.why && ( +
+

+ Why: {item.why} +

+
+ )} + {item.code && ( + 
+                  {item.code}
+
+ )} + {item.examples && ( +
+

Examples:

+
    + {item.examples.map((ex) => ( +
  • + • + {ex} +
    • + ))} +
+
+ )} +
+ ))} +
+
+ + {/* Related Guides */} +
+

Related Guides

+
+ +

📊 Metrics & Dashboards

+

+ Track agent performance with key metrics, instrumentation patterns, and alerting +

+ + +

🔍 Distributed Tracing

+

+ Trace multi-agent workflows with OpenTelemetry and span instrumentation +

+ +
+
+
+ ); +} diff --git a/src/app/observability-guide/metrics/page.tsx b/src/app/observability-guide/metrics/page.tsx new file mode 100644 index 00000000..e21b71ca --- /dev/null +++ b/src/app/observability-guide/metrics/page.tsx @@ -0,0 +1,732 @@ +"use client"; + +import { useState } from "react"; +import Link from "next/link"; +import { LineChart, TrendingUp, AlertCircle, Activity } from "lucide-react"; + +interface Metric { + name: string; + category: "performance" | "cost" | "reliability" | "business"; + description: string; + type: "counter" | "gauge" | "histogram"; + labels?: string[]; + alertThreshold?: string; + code: string; +} + +const KEY_METRICS: Metric[] = [ + { + name: "agent_requests_total", + category: "performance", + description: "Total number of agent requests (success + failure)", + type: "counter", + labels: ["status", "userId", "agentType"], + code: `import { Counter } from 'prom-client'; + +const requestCounter = new Counter({ + name: 'agent_requests_total', + help: 'Total agent requests', + labelNames: ['status', 'userId', 'agentType'], +}); + +// Instrument +requestCounter.inc({ status: 'success', userId: 'user-123', agentType: 'assistant' });`, + }, + { + name: "agent_request_duration_seconds", + category: "performance", + description: "Agent request latency (p50, p95, p99)", + type: "histogram", + labels: ["agentType", "operation"], + alertThreshold: "p95 > 5s", + code: `import { Histogram } from 'prom-client'; + +const requestDuration = new Histogram({ + name: 'agent_request_duration_seconds', + help: 'Agent request latency', + labelNames: ['agentType', 'operation'], + buckets: [0.1, 0.5, 1, 2, 5, 10, 30], // seconds +}); + +// Instrument +const start = Date.now(); +const result = await agent.execute(request); +requestDuration.observe( + { agentType: 'assistant', operation: 'chat' }, + (Date.now() - start) / 1000 +);`, + }, + { + name: "agent_llm_tokens_total", + category: "cost", + description: "Total LLM tokens consumed (prompt + completion)", + type: "counter", + labels: ["model", "operation", "cached"], + code: `const tokenCounter = new Counter({ + name: 'agent_llm_tokens_total', + help: 'Total LLM tokens consumed', + labelNames: ['model', 'operation', 'cached'], +}); + +// After LLM call +tokenCounter.inc( + { model: 'gpt-4-turbo', operation: 'completion', cached: 'false' }, + response.usage.total_tokens +);`, + }, + { + name: "agent_llm_cost_usd", + category: "cost", + description: "Estimated LLM cost in USD", + type: "counter", + labels: ["model", "userId"], + alertThreshold: "Daily cost > $100", + code: `const costCounter = new Counter({ + name: 'agent_llm_cost_usd', + help: 'Estimated LLM cost', + labelNames: ['model', 'userId'], +}); + +// Calculate cost based on token pricing +const cost = calculateCost(response.usage, 'gpt-4-turbo'); +costCounter.inc({ model: 'gpt-4-turbo', userId: 'user-123' }, cost);`, + }, + { + name: "agent_tool_calls_total", + category: "performance", + description: "Tool invocations by tool name and status", + type: "counter", + labels: ["tool", "status"], + code: `const toolCallCounter = new Counter({ + name: 'agent_tool_calls_total', + help: 'Tool invocations', + labelNames: ['tool', 'status'], +}); + +try { + await tools[toolName].execute(params); + toolCallCounter.inc({ tool: toolName, status: 'success' }); +} catch (error) { + toolCallCounter.inc({ tool: toolName, status: 'failure' }); + throw error; +}`, + }, + { + name: "agent_tool_duration_seconds", + category: "performance", + description: "Tool execution latency", + type: "histogram", + labels: ["tool"], + alertThreshold: "p99 > 10s", + code: `const toolDuration = new Histogram({ + name: 'agent_tool_duration_seconds', + help: 'Tool execution latency', + labelNames: ['tool'], + buckets: [0.05, 0.1, 0.5, 1, 2, 5, 10], +}); + +const start = Date.now(); +const result = await tools[toolName].execute(params); +toolDuration.observe({ tool: toolName }, (Date.now() - start) / 1000);`, + }, + { + name: "agent_errors_total", + category: "reliability", + description: "Total errors by type and severity", + type: "counter", + labels: ["errorType", "severity", "recoverable"], + alertThreshold: "Error rate > 5%", + code: `const errorCounter = new Counter({ + name: 'agent_errors_total', + help: 'Total errors', + labelNames: ['errorType', 'severity', 'recoverable'], +}); + +catch (error) { + errorCounter.inc({ + errorType: error.name, + severity: getSeverity(error), + recoverable: isRetryable(error) ? 'true' : 'false', + }); +}`, + }, + { + name: "agent_queue_depth", + category: "performance", + description: "Current number of queued agent requests", + type: "gauge", + labels: ["priority"], + alertThreshold: "Queue depth > 100", + code: `import { Gauge } from 'prom-client'; + +const queueDepth = new Gauge({ + name: 'agent_queue_depth', + help: 'Queued agent requests', + labelNames: ['priority'], +}); + +// Update as queue changes +queueDepth.set({ priority: 'high' }, highPriorityQueue.length); +queueDepth.set({ priority: 'normal' }, normalPriorityQueue.length);`, + }, + { + name: "agent_success_rate", + category: "reliability", + description: "Percentage of successful requests (business metric)", + type: "gauge", + labels: ["agentType"], + alertThreshold: "Success rate < 95%", + code: `// Derived from agent_requests_total +// Calculate in your monitoring tool (Prometheus, Datadog, etc.) +// Prometheus query: +// rate(agent_requests_total{status="success"}[5m]) / +// rate(agent_requests_total[5m])`, + }, + { + name: "agent_context_switches_total", + category: "business", + description: "Number of times agent switches context (workspace, project, etc.)", + type: "counter", + labels: ["fromContext", "toContext"], + code: `const contextSwitchCounter = new Counter({ + name: 'agent_context_switches_total', + help: 'Context switches', + labelNames: ['fromContext', 'toContext'], +}); + +contextSwitchCounter.inc({ + fromContext: 'project-a', + toContext: 'project-b', +});`, + }, +]; + +const DASHBOARD_PATTERNS = [ + { + name: "Agent Health Overview", + description: "Single-pane view of overall agent health", + panels: [ + { metric: "Request Rate", query: "rate(agent_requests_total[5m])", viz: "Line chart" }, + { metric: "Success Rate", query: "success / total requests", viz: "Single stat (95%+)" }, + { metric: "Error Rate", query: "rate(agent_errors_total[5m])", viz: "Line chart (alert if > 5%)" }, + { metric: "p95 Latency", query: "histogram_quantile(0.95, agent_request_duration_seconds)", viz: "Line chart" }, + { metric: "Queue Depth", query: "agent_queue_depth", viz: "Gauge" }, + { metric: "Active Users", query: "count(distinct userId)", viz: "Single stat" }, + ], + }, + { + name: "Cost & Token Usage", + description: "Track LLM costs and optimize spend", + panels: [ + { metric: "Daily Cost", query: "increase(agent_llm_cost_usd[1d])", viz: "Bar chart by model" }, + { metric: "Tokens/Request", query: "avg(tokens) by model", viz: "Line chart" }, + { metric: "Cost per User", query: "sum(cost) by userId", viz: "Table (top 10)" }, + { metric: "Cache Hit Rate", query: "cached / total tokens", viz: "Single stat" }, + { metric: "Model Distribution", query: "count by model", viz: "Pie chart" }, + ], + }, + { + name: "Tool Performance", + description: "Analyze tool usage and bottlenecks", + panels: [ + { metric: "Tool Call Volume", query: "rate(agent_tool_calls_total[5m]) by tool", viz: "Stacked area" }, + { metric: "Tool Latency", query: "histogram_quantile(0.95, tool_duration) by tool", viz: "Bar chart" }, + { metric: "Tool Error Rate", query: "errors / total by tool", viz: "Heatmap" }, + { metric: "Slowest Tools", query: "topk(5, p95 latency by tool)", viz: "Table" }, + ], + }, + { + name: "User Experience", + description: "User-centric metrics for product teams", + panels: [ + { metric: "Session Duration", query: "avg(session_duration_seconds)", viz: "Line chart" }, + { metric: "Messages per Session", query: "avg(messages)", viz: "Histogram" }, + { metric: "User Satisfaction", query: "thumbs_up / (thumbs_up + thumbs_down)", viz: "Single stat" }, + { metric: "Retry Rate", query: "retries / total requests", viz: "Line chart" }, + ], + }, +]; + +const ALERT_RULES = [ + { + name: "High Error Rate", + severity: "critical", + condition: "Error rate > 5% for 5 minutes", + query: "rate(agent_errors_total[5m]) / rate(agent_requests_total[5m]) > 0.05", + action: "Page on-call engineer", + why: "Users are experiencing failures. Immediate investigation required.", + }, + { + name: "Slow Response Time", + severity: "warning", + condition: "p95 latency > 5 seconds for 10 minutes", + query: "histogram_quantile(0.95, rate(agent_request_duration_seconds_bucket[5m])) > 5", + action: "Notify team Slack channel", + why: "Degraded user experience. May indicate scaling issues or slow LLM responses.", + }, + { + name: "Queue Buildup", + severity: "warning", + condition: "Queue depth > 100 for 5 minutes", + query: "agent_queue_depth > 100", + action: "Auto-scale workers or notify ops team", + why: "Requests are backing up. May need to add capacity or shed load.", + }, + { + name: "Cost Spike", + severity: "warning", + condition: "Hourly cost > 2x baseline", + query: "increase(agent_llm_cost_usd[1h]) > 2 * avg_over_time(increase(agent_llm_cost_usd[1h])[7d])", + action: "Notify finance/eng leads", + why: "Unexpected cost increase. Could indicate a runaway agent, attack, or usage spike.", + }, + { + name: "Tool Failure Spike", + severity: "high", + condition: "Tool error rate > 10% for any tool", + query: "rate(agent_tool_calls_total{status='failure'}[5m]) / rate(agent_tool_calls_total[5m]) > 0.10", + action: "Alert on-call, investigate tool health", + why: "A critical tool is failing. May cascade to agent failures.", + }, + { + name: "Zero Traffic", + severity: "critical", + condition: "No requests for 5 minutes", + query: "rate(agent_requests_total[5m]) == 0", + action: "Page on-call (potential outage)", + why: "Complete service outage. Users cannot reach the agent.", + }, +]; + +const INSTRUMENTATION_CODE = { + expressMiddleware: `import express from 'express'; +import { register, collectDefaultMetrics } from 'prom-client'; + +collectDefaultMetrics(); // CPU, memory, etc. + +const app = express(); + +// Metrics endpoint +app.get('/metrics', async (req, res) => { + res.set('Content-Type', register.contentType); + res.end(await register.metrics()); +}); + +// Request metrics middleware +app.use((req, res, next) => { + const start = Date.now(); + res.on('finish', () => { + const duration = (Date.now() - start) / 1000; + requestDuration.observe( + { method: req.method, route: req.route?.path, status: res.statusCode }, + duration + ); + requestCounter.inc({ method: req.method, status: res.statusCode }); + }); + next(); +});`, + + llmInstrumentation: `async function callLLM(prompt: string, model: string) { + const start = Date.now(); + const operationId = randomUUID(); + + try { + const response = await openai.chat.completions.create({ + model, + messages: [{ role: 'user', content: prompt }], + }); + + // Record metrics + const durationSec = (Date.now() - start) / 1000; + llmLatency.observe({ model, operation: 'completion' }, durationSec); + + const tokens = response.usage; + tokenCounter.inc({ model, type: 'prompt' }, tokens.prompt_tokens); + tokenCounter.inc({ model, type: 'completion' }, tokens.completion_tokens); + + const cost = calculateCost(tokens, model); + costCounter.inc({ model }, cost); + + llmCallCounter.inc({ model, status: 'success' }); + + return response; + } catch (error) { + llmCallCounter.inc({ model, status: 'failure' }); + throw error; + } +}`, + + customBusinessMetrics: `// Track business-specific metrics +const userFeedbackGauge = new Gauge({ + name: 'agent_user_satisfaction', + help: 'User satisfaction score (1-5)', + labelNames: ['agentType'], +}); + +const sessionLengthHistogram = new Histogram({ + name: 'agent_session_duration_seconds', + help: 'User session duration', + buckets: [60, 300, 600, 1800, 3600], // 1min to 1hr +}); + +// Instrument +userFeedbackGauge.set({ agentType: 'assistant' }, averageRating); +sessionLengthHistogram.observe(sessionDurationSec);`, +}; + +export default function MetricsPage() { + const [selectedCategory, setSelectedCategory] = useState("all"); + + const filteredMetrics = + selectedCategory === "all" ? KEY_METRICS : KEY_METRICS.filter((m) => m.category === selectedCategory); + + return ( +
+ {/* Header */} +
+ + ← Back to Observability Guide + +
+ +

Metrics & Dashboards

+
+

+ Key performance indicators for AI agents. Learn what to measure, how to instrument, and how to build + actionable dashboards. +

+
+ + {/* Why Metrics Matter */} +
+

Why Metrics Matter

+
+

+ Logs tell you what happened. Metrics tell you how often, how + fast, and how much. +

+
+
+

Metrics enable:

+
    +
  • • Real-time dashboards and alerts
    • +
  • • Performance regression detection
    • +
  • • Capacity planning and autoscaling
    • +
  • • Cost tracking and optimization
    • +
  • • SLA/SLO compliance monitoring
    • +
+
+
+

Without metrics, you can't answer:

+
    +
  • • Is the agent slower than yesterday?
    • +
  • • Are we within budget for LLM costs?
    • +
  • • Which tool is the bottleneck?
    • +
  • • Can we handle 10x traffic?
    • +
+
+
+
+
+ + {/* Metric Types */} +
+

Metric Types

+
+
+

Counter

+

+ Monotonically increasing value. Only goes up (or resets to 0). +

+

+ Examples: Total requests, total errors, total tokens +

+

+ Query: rate(counter[5m]) for per-second rate +

+
+
+

Gauge

+

Current value that can go up or down.

+

+ Examples: Queue depth, active users, memory usage +

+

+ Query: gauge (current value) +

+
+
+

Histogram

+

+ Distribution of values, calculates percentiles (p50, p95, p99). +

+

+ Examples: Request latency, tool duration, token count per request +

+

+ Query: histogram_quantile(0.95, ...) +

+
+
+
+ + {/* Key Metrics */} +
+
+

Key Agent Metrics

+
+ + + + + +
+
+ +
+ {filteredMetrics.map((metric) => ( +
+
+
+

{metric.name}

+

{metric.description}

+
+
+ + {metric.type} + + + {metric.category} + +
+
+ + {metric.labels && ( +
+ Labels: + {metric.labels.join(", ")} +
+ )} + + {metric.alertThreshold && ( +
+ ⚠️ Alert Threshold: + {metric.alertThreshold} +
+ )} + + 
+                {metric.code}
+
+
+ ))} +
+
+ + {/* Dashboard Patterns */} +
+

Dashboard Design Patterns

+
+ {DASHBOARD_PATTERNS.map((dashboard) => ( +
+

{dashboard.name}

+

{dashboard.description}

+
+ + + + + + + + + + {dashboard.panels.map((panel, idx) => ( + + + + + + ))} + +
PanelQuery/MetricVisualization
{panel.metric}{panel.query}{panel.viz}
+
+
+ ))} +
+
+ + {/* Alert Rules */} +
+

Alerting Best Practices

+
+

+ Golden Rule: Only alert on symptoms users care about (errors, latency, downtime). +

+

+ Don't alert on causes (high CPU, memory) unless they directly impact user experience. +

+
+ +
+ {ALERT_RULES.map((rule) => ( +
+
+
+

{rule.name}

+

{rule.condition}

+
+ + {rule.severity} + +
+ +
+ Prometheus Query: + 
+                  {rule.query}
+
+
+ +
+
+ Action: +

{rule.action}

+
+
+ Why it matters: +

{rule.why}

+
+
+
+ ))} +
+
+ + {/* Instrumentation Code */} +
+

Instrumentation Code Examples

+ +
+
+

Express.js Middleware (Automatic Instrumentation)

+ 
+              {INSTRUMENTATION_CODE.expressMiddleware}
+
+
+ +
+

+ LLM Call Instrumentation +

+ 
+              {INSTRUMENTATION_CODE.llmInstrumentation}
+
+
+ +
+

Custom Business Metrics

+ 
+              {INSTRUMENTATION_CODE.customBusinessMetrics}
+
+
+
+
+ + {/* Related Guides */} +
+

Related Guides

+
+ +

📝 Logging Best Practices

+

+ Structured logging, correlation IDs, and what to log for debugging +

+ + +

🔍 Distributed Tracing

+

+ Trace multi-agent workflows with OpenTelemetry and span instrumentation +

+ +
+
+
+ ); +} diff --git a/src/app/observability-guide/page.tsx b/src/app/observability-guide/page.tsx new file mode 100644 index 00000000..3fcda530 --- /dev/null +++ b/src/app/observability-guide/page.tsx @@ -0,0 +1,608 @@ +"use client"; + +import { useState } from "react"; +import Link from "next/link"; +import { Activity, Eye, LineChart, Network, CheckCircle, XCircle } from "lucide-react"; + +interface ChecklistItem { + id: string; + title: string; + priority: "critical" | "high" | "medium" | "low"; + category: string; + description: string; + guide: string; +} + +const CHECKLIST: ChecklistItem[] = [ + { + id: "structured-logging", + title: "Structured Logging", + priority: "critical", + category: "Logging", + description: "Implement JSON-formatted logs with correlation IDs for all agent decisions and tool calls", + guide: "/observability-guide/logging", + }, + { + id: "error-tracking", + title: "Error Tracking & Alerts", + priority: "critical", + category: "Metrics", + description: "Track error rates, failures, and set up alerts for degraded agent performance", + guide: "/observability-guide/metrics", + }, + { + id: "distributed-tracing", + title: "Distributed Tracing", + priority: "high", + category: "Tracing", + description: "Instrument multi-agent workflows with OpenTelemetry for end-to-end visibility", + guide: "/observability-guide/tracing", + }, + { + id: "token-usage-metrics", + title: "Token Usage Tracking", + priority: "high", + category: "Metrics", + description: "Monitor token consumption, costs, and model latency across all LLM calls", + guide: "/observability-guide/metrics#token-tracking", + }, + { + id: "correlation-ids", + title: "Request Correlation", + priority: "high", + category: "Logging", + description: "Propagate correlation IDs through all log statements for request tracing", + guide: "/observability-guide/logging#correlation", + }, + { + id: "decision-logging", + title: "Decision Audit Trail", + priority: "medium", + category: "Logging", + description: "Log all agent reasoning steps, tool selections, and parameter choices", + guide: "/observability-guide/logging#decisions", + }, + { + id: "latency-percentiles", + title: "Latency Percentiles", + priority: "medium", + category: "Metrics", + description: "Track p50, p95, p99 latencies for agent responses and tool executions", + guide: "/observability-guide/metrics#latency", + }, + { + id: "span-instrumentation", + title: "Span Instrumentation", + priority: "medium", + category: "Tracing", + description: "Create spans for all tool calls, LLM requests, and context switches", + guide: "/observability-guide/tracing#spans", + }, +]; + +const OBSERVABILITY_PILLARS = [ + { + name: "Logs", + icon: "📝", + color: "blue", + description: "Timestamped records of discrete events", + useCases: [ + "Debugging agent decisions and failures", + "Audit trails for compliance", + "Root cause analysis", + "Security incident investigation", + ], + keyMetrics: ["Log volume", "Error log rate", "Log ingestion lag"], + guide: "/observability-guide/logging", + }, + { + name: "Metrics", + icon: "📊", + color: "green", + description: "Numerical measurements aggregated over time", + useCases: [ + "Performance monitoring and alerting", + "Capacity planning", + "Cost tracking (tokens, API calls)", + "SLA/SLO tracking", + ], + keyMetrics: ["Latency percentiles", "Error rates", "Token usage", "Success rates"], + guide: "/observability-guide/metrics", + }, + { + name: "Traces", + icon: "🔍", + color: "purple", + description: "End-to-end request flow through distributed systems", + useCases: [ + "Multi-agent workflow visibility", + "Latency bottleneck identification", + "Dependency mapping", + "Tool call chain analysis", + ], + keyMetrics: ["Trace completion rate", "Span count", "Critical path latency"], + guide: "/observability-guide/tracing", + }, +]; + +const MATURITY_LEVELS = [ + { + level: 1, + name: "Reactive", + description: "Basic logging to console/files, manual debugging", + characteristics: [ + "Print statements and console.log debugging", + "No structured logging", + "Manual log grepping", + "No metrics or dashboards", + ], + limitations: "Can't answer: Why did the agent fail? What's the p95 latency? Which tool call is slow?", + }, + { + level: 2, + name: "Aware", + description: "Centralized logging, basic metrics", + characteristics: [ + "Logs aggregated in central system (e.g., CloudWatch, Datadog)", + "JSON structured logging with some context", + "Basic metrics (error count, request count)", + "Manual dashboard creation", + ], + limitations: "Limited correlation between logs and metrics. No distributed tracing. Reactive debugging.", + }, + { + level: 3, + name: "Proactive", + description: "Full observability with logs, metrics, traces", + characteristics: [ + "Structured logging with correlation IDs", + "Comprehensive metrics and alerting", + "Distributed tracing with OpenTelemetry", + "Pre-built dashboards for agent health", + ], + capabilities: "Can answer: What failed? Where? Why? How often? What's the user impact?", + }, + { + level: 4, + name: "Predictive", + description: "AI-powered observability, anomaly detection", + characteristics: [ + "Automated anomaly detection on metrics", + "Predictive alerting (issues before they impact users)", + "Auto-remediation for known failure patterns", + "Continuous profiling and optimization", + ], + capabilities: + "System self-heals and auto-scales. Proactive alerts before user impact. Agent performance continuously optimized.", + }, +]; + +const TOOL_LANDSCAPE = [ + { + category: "All-in-One Platforms", + tools: [ + { name: "Datadog", features: ["Logs", "Metrics", "Traces", "APM"], agentFriendly: true }, + { name: "New Relic", features: ["Logs", "Metrics", "Traces", "AI Monitoring"], agentFriendly: true }, + { name: "Dynatrace", features: ["Full-stack observability", "AI ops"], agentFriendly: false }, + ], + }, + { + category: "Open Source", + tools: [ + { name: "Grafana Stack", features: ["Loki (logs)", "Prometheus (metrics)", "Tempo (traces)"], agentFriendly: true }, + { name: "Elastic Stack", features: ["Elasticsearch", "Logstash", "Kibana"], agentFriendly: true }, + { name: "OpenTelemetry", features: ["Vendor-neutral instrumentation"], agentFriendly: true }, + ], + }, + { + category: "Specialized", + tools: [ + { name: "Langfuse", features: ["LLM observability", "Token tracking", "Prompt management"], agentFriendly: true }, + { name: "LangSmith", features: ["LangChain tracing", "Eval workflows"], agentFriendly: true }, + { name: "Helicone", features: ["LLM proxy", "Cost tracking", "Caching"], agentFriendly: true }, + ], + }, +]; + +const QUICK_START_CODE = [ + { + step: 1, + title: "Structured Logging Setup", + code: `// Use a structured logging library +import pino from 'pino'; + +const logger = pino({ + level: process.env.LOG_LEVEL || 'info', + formatters: { + level: (label) => ({ level: label }), + }, +}); + +// Log with context and correlation +logger.info({ + correlationId: req.id, + userId: req.user.id, + action: 'tool_call', + tool: 'web_search', + query: 'observability best practices', + durationMs: 234, +}, 'Tool call completed');`, + }, + { + step: 2, + title: "Basic Metrics with Prometheus", + code: `import { Counter, Histogram } from 'prom-client'; + +// Define metrics +const toolCallCounter = new Counter({ + name: 'agent_tool_calls_total', + help: 'Total number of tool calls', + labelNames: ['tool', 'status'], +}); + +const llmLatency = new Histogram({ + name: 'agent_llm_latency_seconds', + help: 'LLM request latency', + labelNames: ['model', 'operation'], + buckets: [0.1, 0.5, 1, 2, 5, 10], +}); + +// Instrument your code +const start = Date.now(); +const result = await callLLM(prompt); +llmLatency.observe({ model: 'gpt-4', operation: 'completion' }, (Date.now() - start) / 1000);`, + }, + { + step: 3, + title: "Distributed Tracing with OpenTelemetry", + code: `import { trace, SpanStatusCode } from '@opentelemetry/api'; + +const tracer = trace.getTracer('agent-runtime'); + +async function executeToolCall(toolName: string, params: any) { + return tracer.startActiveSpan(\`tool.\${toolName}\`, async (span) => { + try { + span.setAttribute('tool.name', toolName); + span.setAttribute('tool.params', JSON.stringify(params)); + + const result = await tools[toolName](params); + + span.setStatus({ code: SpanStatusCode.OK }); + span.setAttribute('tool.result.size', JSON.stringify(result).length); + + return result; + } catch (error) { + span.setStatus({ code: SpanStatusCode.ERROR, message: error.message }); + span.recordException(error); + throw error; + } finally { + span.end(); + } + }); +}`, + }, +]; + +export default function ObservabilityGuidePage() { + const [selectedPriority, setSelectedPriority] = useState("all"); + const [completedItems, setCompletedItems] = useState>(new Set()); + + const priorityColors = { + critical: "text-red-600 bg-red-50 border-red-200", + high: "text-orange-600 bg-orange-50 border-orange-200", + medium: "text-yellow-600 bg-yellow-50 border-yellow-200", + low: "text-blue-600 bg-blue-50 border-blue-200", + }; + + const filteredChecklist = + selectedPriority === "all" + ? CHECKLIST + : CHECKLIST.filter((item) => item.priority === selectedPriority); + + const toggleComplete = (id: string) => { + const newCompleted = new Set(completedItems); + if (newCompleted.has(id)) { + newCompleted.delete(id); + } else { + newCompleted.add(id); + } + setCompletedItems(newCompleted); + }; + + return ( +
+ {/* Header */} +
+
+ +

Agent Observability & Monitoring

+
+

+ Comprehensive observability guide for AI agents. Monitor, debug, and optimize agent behavior with logs, + metrics, and distributed tracing. +

+
+ + {/* Three Pillars */} +
+

The Three Pillars of Observability

+
+ {OBSERVABILITY_PILLARS.map((pillar) => ( +
+
+ {pillar.icon} +

{pillar.name}

+
+

{pillar.description}

+
+

Use Cases:

+
    + {pillar.useCases.map((useCase) => ( +
  • + • + {useCase} +
    • + ))} +
+
+
+

Key Metrics:

+
+ {pillar.keyMetrics.map((metric) => ( + + {metric} + + ))} +
+
+ + Deep Dive → + +
+ ))} +
+
+ + {/* Why Observability Matters for Agents */} +
+

Why Observability Matters for AI Agents

+
+

+ AI agents are non-deterministic black boxes that make autonomous decisions, call external + tools, and chain complex reasoning steps. Without observability, you're flying blind: +

+
+
+

❌ Without Observability

+
    +
  • • "The agent failed" — but why?
    • +
  • • Debugging requires code changes & redeployment
    • +
  • • Can't measure performance improvements
    • +
  • • Users report issues before you know they exist
    • +
  • • No visibility into cost (tokens, API calls)
    • +
+
+
+

✅ With Observability

+
    +
  • • Real-time dashboards show agent health
    • +
  • • Trace requests end-to-end across tools
    • +
  • • Debug production issues without redeployment
    • +
  • • Alerts trigger before users notice degradation
    • +
  • • Optimize costs with token/latency metrics
    • +
+
+
+

+ Rule of thumb: If you can't answer "Why did this agent request fail?" in under 2 minutes, + your observability is insufficient. +

+
+
+ + {/* Maturity Model */} +
+

Observability Maturity Model

+
+ {MATURITY_LEVELS.map((level) => ( +
+
+
+ {level.level} +
+
+

{level.name}

+

{level.description}

+
+
+

Characteristics:

+
    + {level.characteristics.map((char) => ( +
  • + • + {char} +
    • + ))} +
+
+
+

+ {level.capabilities ? "Capabilities:" : "Limitations:"} +

+

+ {level.capabilities || level.limitations} +

+
+
+
+
+
+ ))} +
+
+ + {/* Tool Landscape */} +
+

Observability Tool Landscape

+
+ {TOOL_LANDSCAPE.map((category) => ( +
+

{category.category}

+
+ {category.tools.map((tool) => ( +
+
+

{tool.name}

+ {tool.agentFriendly && ( + Agent-friendly + )} +
+
+ {tool.features.map((feature) => ( + + {feature} + + ))} +
+
+ ))} +
+
+ ))} +
+
+ + {/* Quick Start Checklist */} +
+
+

Quick Start Checklist

+
+ + + + +
+
+ +
+ {filteredChecklist.map((item) => ( +
+
+ +
+
+

{item.title}

+ + {item.priority} + + {item.category} +
+

{item.description}

+ + View Guide → + +
+
+
+ ))} +
+
+ + {/* Quick Start Code Examples */} +
+

Quick Start: 3 Essential Steps

+
+ {QUICK_START_CODE.map((item) => ( +
+
+
+ {item.step} +
+
+

{item.title}

+ 
+                    {item.code}
+
+
+
+
+ ))} +
+
+ + {/* Deep Dive Links */} +
+

Deep Dive Guides

+
+ +

📝 Logging Best Practices

+

+ Structured logging, correlation IDs, log levels, what to log, aggregation strategies +

+ + +

📊 Metrics & Dashboards

+

+ Key agent metrics, instrumentation patterns, dashboard design, alerting thresholds +

+ + +

🔍 Distributed Tracing

+

+ Multi-agent tracing, OpenTelemetry setup, span design, trace visualization +

+ +
+
+
+ ); +}