feat(observability): measure response tokens and cache status

scripts/lint-tool-schemas.mjs

@@ -33,6 +33,14 @@ const TOOL_NAME_RE = /^[a-z][a-z0-9_]{2,63}$/;
 
 const BASELINE_PATH = resolve(__dirname, 'lint-tool-schemas.baseline.json');
 
+async function readStdin() {
+  let data = '';
+  process.stdin.setEncoding('utf8');
+  for await (const chunk of process.stdin) data += chunk;
+  return data;
+}
+
+
 // ── parse CLI args ─────────────────────────────────────────────────────────
 const args = process.argv.slice(2);
 const updateBaseline = args.includes('--update-baseline');
@@ -46,7 +54,7 @@ if (!inputFile) {
 // ── load tools list ────────────────────────────────────────────────────────
 let tools;
 try {
-  tools = JSON.parse(readFileSync(inputFile === '-' ? 0 : resolve(inputFile), 'utf8'));
+  tools = JSON.parse(inputFile === '-' ? await readStdin() : readFileSync(resolve(inputFile), 'utf8'));
 } catch (err) {
   process.stderr.write(`Error reading tools list: ${err.message}\n`);
   process.exit(2);

src/index.ts

@@ -110,8 +110,10 @@ program
       const server = new MCPServer(undefined, { initialToolTier: 3 });
       registerAllTools(server);
       const manifest = server.getToolManifest();
-      process.stdout.write(JSON.stringify(manifest.tools) + '\n');
-      process.exit(0);
+      await new Promise<void>((resolve) => {
+        process.stdout.write(JSON.stringify(manifest.tools) + '\n', () => resolve());
+      });
+      return;
     }
 
     let port = parseInt(options.port, 10);

src/mcp-server.ts

@@ -76,6 +76,62 @@ const SKIP_RECORDING_TOOLS = new Set([
  * Detect if an error is a Chrome/CDP connection error that may be recoverable
  * by reconnecting to the browser.
  */
+export function estimateOutputTokensFromChars(chars: number): number {
+  // Heuristic only; intentionally avoids provider-specific tokenizer deps.
+  return Math.max(0, Math.ceil(chars / 4));
+}
+
+function stringifyResultPayload(result: MCPResult): string {
+  try {
+    return JSON.stringify(result);
+  } catch {
+    return Array.isArray(result.content)
+      ? result.content.map((c) => c.text ?? c.data ?? '').join('')
+      : '';
+  }
+}
+
+const CACHE_STATUS_LABELS = new Set(['HIT', 'MISS', 'BYPASS', 'ERROR']);
+const CACHE_KEY_VERSION_LABEL_RE = /^v?\d{1,3}$/i;
+
+function normalizeCacheStatusLabel(raw: string): string {
+  const normalized = raw.trim().toUpperCase();
+  return CACHE_STATUS_LABELS.has(normalized) ? normalized : 'UNKNOWN';
+}
+
+function normalizeCacheKeyVersionLabel(raw: unknown): string {
+  if (raw === undefined || raw === null || raw === '') return 'unknown';
+  const normalized = String(raw).trim();
+  if (normalized === '') return 'unknown';
+  return CACHE_KEY_VERSION_LABEL_RE.test(normalized) ? normalized : 'other';
+}
+
+export function extractCacheStatus(result: MCPResult): { status: string; keyVersion: string } | null {
+  const raw = (result as Record<string, unknown>)._cache
+    ?? (result as Record<string, unknown>).cache
+    ?? (result as Record<string, unknown>).cacheStatus;
+  if (typeof raw === 'string') {
+    return { status: normalizeCacheStatusLabel(raw), keyVersion: 'unknown' };
+  }
+  if (raw && typeof raw === 'object') {
+    const obj = raw as Record<string, unknown>;
+    const status = typeof obj.status === 'string' ? obj.status : typeof obj.cacheStatus === 'string' ? obj.cacheStatus : null;
+    if (!status) return null;
+    const keyVersion = obj.keyVersion ?? obj.version ?? 'unknown';
+    return {
+      status: normalizeCacheStatusLabel(status),
+      keyVersion: normalizeCacheKeyVersionLabel(keyVersion),
+    };
+  }
+  if (result.structuredContent && typeof result.structuredContent.cacheStatus === 'string') {
+    return {
+      status: normalizeCacheStatusLabel(result.structuredContent.cacheStatus),
+      keyVersion: normalizeCacheKeyVersionLabel(result.structuredContent.cacheKeyVersion),
+    };
+  }
+  return null;
+}
+
 export function isConnectionError(error: unknown): boolean {
   if (error instanceof OpenChromeConnectionError) return true;
   const message = formatError(error);
@@ -1236,7 +1292,7 @@ export class MCPServer {
         } catch {
           // best-effort
         }
-        return {
+        const deniedResult: MCPResult = {
           content: [
             {
               type: 'text',
@@ -1245,6 +1301,8 @@ export class MCPServer {
           ],
           isError: true,
         };
+        this.recordToolOutputObservability(toolName, deniedResult);
+        return deniedResult;
       }
     }
 
@@ -1271,7 +1329,7 @@ export class MCPServer {
       } catch {
         // best-effort
       }
-      return {
+      const forbiddenResult: MCPResult = {
         content: [
           {
             type: 'text',
@@ -1280,6 +1338,8 @@ export class MCPServer {
         ],
         isError: true,
       };
+      this.recordToolOutputObservability(toolName, forbiddenResult);
+      return forbiddenResult;
     }
 
     // Handle the expand_tools meta-tool before normal tool lookup
@@ -1305,9 +1365,11 @@ export class MCPServer {
         text += `\n\nNewly available tools:\n${JSON.stringify(newTools, null, 2)}\n\nYou can now call these tools directly by name.`;
       }
 
-      return {
+      const expandResult: MCPResult = {
         content: [{ type: 'text', text }],
       };
+      this.recordToolOutputObservability(toolName, expandResult);
+      return expandResult;
     }
 
     const tool = this.tools.get(toolName);
@@ -1320,10 +1382,12 @@ export class MCPServer {
     if (requiredFields && requiredFields.length > 0) {
       const missing = requiredFields.filter((field) => !(field in toolArgs) || toolArgs[field] === undefined || toolArgs[field] === null);
       if (missing.length > 0) {
-        return {
+        const missingArgsResult: MCPResult = {
           content: [{ type: 'text', text: `Error: Missing required argument(s): ${missing.join(', ')}` }],
           isError: true,
         };
+        this.recordToolOutputObservability(toolName, missingArgsResult);
+        return missingArgsResult;
       }
     }
 
@@ -1392,7 +1456,7 @@ export class MCPServer {
       if (!rateResult.allowed) {
         console.error(`[MCPServer] Rate limit exceeded for session ${sessionId}, retry after ${rateResult.retryAfterSec}s`);
         try { getMetricsCollector().inc('openchrome_rate_limit_rejections_total', withTenantLabel({ tool: toolName })); } catch { /* best-effort */ }
-        return {
+        const rateLimitResult: MCPResult = {
           content: [
             {
               type: 'text',
@@ -1401,6 +1465,8 @@ export class MCPServer {
           ],
           isError: true,
         };
+        this.recordToolOutputObservability(toolName, rateLimitResult);
+        return rateLimitResult;
       }
     }
 
@@ -1440,7 +1506,7 @@ export class MCPServer {
         });
 
         if (reconnectResult !== 'reconnected') {
-          return {
+          const reconnectResultPayload: MCPResult = {
             content: [
               {
                 type: 'text',
@@ -1449,6 +1515,8 @@ export class MCPServer {
             ],
             isError: true,
           };
+          this.recordToolOutputObservability(toolName, reconnectResultPayload);
+          return reconnectResultPayload;
         }
         console.error(`[MCPServer] Reconnection complete, proceeding with "${toolName}"`);
       }
@@ -1812,7 +1880,7 @@ export class MCPServer {
         }
       }
 
-      if (compressionConfig?.enabled && compressionConfig?.trackSavings) {
+      if (compressionConfig?.enabled && compressionConfig?.trackSavings && !(result as Record<string, unknown>)._compression) {
         (result as Record<string, unknown>)._compression = {
           level: compressionConfig.level ?? 'light',
           verbosity,
@@ -1825,7 +1893,9 @@ export class MCPServer {
       // the substituted input, returned it inside a JSON blob, or surfaced
       // it via an error message) with `${SECRET:NAME}` placeholders. No-op
       // when --secrets was not passed.
-      return redactSecrets(result);
+      const finalResult = redactSecrets(result);
+      this.recordToolOutputObservability(toolName, finalResult);
+      return finalResult;
     } catch (error) {
       const message = formatError(error);
       const abortReason = isClientDisconnect(error) ? 'client_disconnect' : null;
@@ -1978,7 +2048,45 @@ export class MCPServer {
 
       // Secrets redaction (#834) — see success path. Error messages can
       // include the literal value (e.g. "type ... failed for value X").
-      return redactSecrets(errResult);
+      const finalErrResult = redactSecrets(errResult);
+      this.recordToolOutputObservability(toolName, finalErrResult);
+      return finalErrResult;
+    }
+  }
+
+
+  private recordToolOutputObservability(toolName: string, result: MCPResult): void {
+    try {
+      const metrics = getMetricsCollector();
+      const payload = stringifyResultPayload(result);
+      const bytes = Buffer.byteLength(payload, 'utf8');
+      metrics.observe('openchrome_tool_output_bytes', withTenantLabel({ tool: toolName }), bytes);
+      metrics.observe('openchrome_tool_estimated_tokens', withTenantLabel({ tool: toolName }), estimateOutputTokensFromChars(payload.length));
+
+      const compression = (result as Record<string, unknown>)._compression;
+      if (compression && typeof compression === 'object') {
+        const originalChars = (compression as Record<string, unknown>).originalChars;
+        const compressedChars = (compression as Record<string, unknown>).compressedChars;
+        const level = String((compression as Record<string, unknown>).level ?? 'unknown');
+        if (typeof originalChars === 'number' && typeof compressedChars === 'number' && originalChars > compressedChars) {
+          metrics.observe(
+            'openchrome_tool_compression_saved_bytes',
+            withTenantLabel({ tool: toolName, mode: level }),
+            originalChars - compressedChars,
+          );
+        }
+      }
+
+      const cache = extractCacheStatus(result);
+      if (cache) {
+        metrics.inc('openchrome_cache_status_total', withTenantLabel({
+          tool: toolName,
+          status: cache.status,
+          key_version: cache.keyVersion,
+        }));
+      }
+    } catch {
+      // Metrics are best-effort and must never affect tool responses.
     }
   }
 

src/metrics/collector.ts

@@ -247,6 +247,13 @@ export function getMetricsCollector(): MetricsCollector {
     instance.registerCounter('openchrome_tool_calls_total', 'Total MCP tool calls');
     instance.registerHistogram('openchrome_tool_duration_seconds', 'Tool call duration in seconds',
       [0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120]);
+    instance.registerHistogram('openchrome_tool_output_bytes', 'Final MCP tool result payload size in bytes',
+      [128, 512, 1024, 4096, 16384, 65536, 262144, 1048576]);
+    instance.registerHistogram('openchrome_tool_estimated_tokens', 'Estimated MCP tool result output tokens (chars / 4 heuristic)',
+      [32, 128, 256, 1024, 4096, 16384, 65536, 262144]);
+    instance.registerHistogram('openchrome_tool_compression_saved_bytes', 'Estimated response bytes saved by response compression or delta modes',
+      [128, 512, 1024, 4096, 16384, 65536, 262144]);
+    instance.registerCounter('openchrome_cache_status_total', 'Cache status observations by tool and key version');
     instance.registerCounter('openchrome_reconnect_total', 'Total successful CDP reconnections');
     instance.registerGauge('openchrome_heap_bytes', 'Node.js heap usage in bytes');
     instance.registerGauge('openchrome_active_sessions', 'Current active MCP sessions');

src/tools/crawl-cancel.ts

@@ -19,7 +19,7 @@ const definition: MCPToolDefinition = {
   inputSchema: {
     type: 'object',
     properties: {
-      jobId: { type: 'string', description: 'Job id returned by crawl_start.' },
+      jobId: { type: 'string', description: 'REQUIRED Job id returned by crawl_start.' },
     },
     required: ['jobId'],
   },

src/tools/crawl-start.ts

@@ -20,7 +20,7 @@ const definition: MCPToolDefinition = {
   inputSchema: {
     type: 'object',
     properties: {
-      url: { type: 'string', description: 'Starting URL to crawl' },
+      url: { type: 'string', description: 'REQUIRED Starting URL to crawl' },
       max_depth: { type: 'number', description: 'Max link-follow depth. Default: 2' },
       max_pages: { type: 'number', description: 'Max pages to crawl. Default: 20' },
       scope: { type: 'string', description: 'URL glob limiting which URLs to follow. Default: same origin.' },

src/tools/crawl-status.ts

@@ -24,7 +24,7 @@ const definition: MCPToolDefinition = {
   inputSchema: {
     type: 'object',
     properties: {
-      jobId: { type: 'string', description: 'Job id returned by crawl_start.' },
+      jobId: { type: 'string', description: 'REQUIRED Job id returned by crawl_start.' },
       advance: {
         type: 'number',
         description: 'Max pages to fetch in this call. Default OC_CRAWL_ADVANCE_DEFAULT (5). Use 0 for read-only.',

src/tools/read-page.ts

@@ -599,8 +599,14 @@ const handler: ToolHandler = async (
               const statsLine = `[page_stats] url: ${result.pageStats.url} | title: ${result.pageStats.title} | scroll: ${result.pageStats.scrollX},${result.pageStats.scrollY} | viewport: ${result.pageStats.viewportWidth}x${result.pageStats.viewportHeight} | docSize: ${result.pageStats.scrollWidth}x${result.pageStats.scrollHeight}\n\n`;
               const includePaginationDom = args.includePagination !== false;
               const domPaginationSection = includePaginationDom ? formatPaginationSection(await detectPagination(page, tabId)) : '';
+              const compressedText = statsLine + delta.content + nodeRefsBlock + domPaginationSection;
               return {
-                content: [{ type: 'text', text: statsLine + delta.content + nodeRefsBlock + domPaginationSection }],
+                content: [{ type: 'text', text: compressedText }],
+                _compression: {
+                  level: 'delta',
+                  originalChars: outputText.length,
+                  compressedChars: compressedText.length,
+                },
               };
             }
             // If not delta (too many changes), fall through to full response