From e8f2529e9e1cce1d07796297e6e02459f5cefefd Mon Sep 17 00:00:00 2001 From: Joshua Menke Date: Thu, 21 May 2026 15:22:17 +0200 Subject: [PATCH] fix: align transcript real-turn index with UserPromptSubmit (0.2.12) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Synthetic user-line triples for client-only slash commands (/model, /clear) inflated transcript-reader's real-turn index, drifting off-by-N from the collector's turnIndex. With UserPromptSubmit lacking prompt_id, ordinal fallback returned the wrong turn — early turns lost outputs entirely, later turns carried earlier turns' responses. Pre-pass now marks any prompt_id with content as client-only and skips those user lines (plus isMeta:true). Also accept camelCase promptId as defense-in-depth. Co-Authored-By: Claude Sonnet 4.6 --- .claude-plugin/marketplace.json | 2 +- .claude-plugin/plugin.json | 2 +- CHANGELOG.md | 6 +++++ package.json | 2 +- scripts/server.js | 2 +- scripts/transcript-reader.js | 46 ++++++++++++++++++++++++++++++++- src/server.ts | 2 +- src/transcript-reader.ts | 44 ++++++++++++++++++++++++++++++- src/types.ts | 4 ++- tests/transcript-reader.test.ts | 38 +++++++++++++++++++++++++++ 10 files changed, 140 insertions(+), 8 deletions(-) diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 2925bdb..8cd1d9d 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -8,7 +8,7 @@ { "name": "claude-code-ai-observability", "description": "Realtime Sentry traces with per-turn tokens, cost, subagent and error instrumentation, and rich auto-tagging (session / git / host). Forked from sergical/claude-code-sentry-monitor (MIT).", - "version": "0.2.11", + "version": "0.2.12", "author": { "name": "Joshkop" }, diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index d09307a..7b6a9bb 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "claude-code-ai-observability", "description": "Comprehensive AI Agent Observability plugin for Claude Code — realtime Sentry traces with per-turn tokens, cost, subagent and error instrumentation, and rich auto-tagging (session / git / host). Forked from sergical/claude-code-sentry-monitor (MIT).", - "version": "0.2.11", + "version": "0.2.12", "author": { "name": "Joshkop" }, diff --git a/CHANGELOG.md b/CHANGELOG.md index fe05226..acfa849 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). +## [0.2.12] - 2026-05-21 + +### Fixed + +- **AI Conversations: inputs and outputs now align with the right turn.** Claude Code's transcript records client-only slash commands like `/model` and `/clear` as a synthetic user-line triple (``, ``, ``), and Claude Code does not fire `UserPromptSubmit` for them. The transcript reader was counting every one of those lines as a real turn, so the collector's `turnIndex` (which only counts real `UserPromptSubmit` events) drifted off-by-N from the transcript's real-turn ordinal. Because real `UserPromptSubmit` hook payloads don't carry `prompt_id`, `selectTurn` fell back to ordinal — and the ordinal pointed at the wrong transcript turn. Visible symptom: early turns showed `gen_ai.input.messages` but the chat span had no `gen_ai.output.messages` (`claude_code.usage_extraction.status = turn_had_no_usage`), and a later turn carried an *earlier* turn's response (e.g. turn 3's chat span output was actually the assistant response to turn 0). `readTranscript` now skips `isMeta:true` lines and every user line belonging to a prompt_id that has any `` content, restoring 1:1 alignment with `UserPromptSubmit`. Defense-in-depth: `UserPromptSubmit` events are now read from both `prompt_id` and `promptId`. + ## [0.2.11] - 2026-05-21 ### Fixed diff --git a/package.json b/package.json index 5a33a67..e39a568 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "claude-code-ai-observability", - "version": "0.2.11", + "version": "0.2.12", "description": "Comprehensive AI Agent Observability plugin for Claude Code — realtime Sentry traces with per-turn tokens, cost, subagent and error instrumentation, and rich auto-tagging (session / git / host).", "type": "module", "scripts": { diff --git a/scripts/server.js b/scripts/server.js index 4941e61..e248b7b 100644 --- a/scripts/server.js +++ b/scripts/server.js @@ -311,7 +311,7 @@ export function startServer(sentry, config, baseAutoTags) { const record = getOrCreateSession(event); void closeCurrentTurn(record).then(() => { record.turnIndex += 1; - record.currentPromptId = event.prompt_id ?? null; + record.currentPromptId = event.prompt_id ?? event.promptId ?? null; const prompt = event.prompt ?? event.message ?? null; record.currentTurnPrompt = prompt; record.currentTurnStart = Date.now() / 1000; diff --git a/scripts/transcript-reader.js b/scripts/transcript-reader.js index 0d0df5e..ded6492 100644 --- a/scripts/transcript-reader.js +++ b/scripts/transcript-reader.js @@ -38,6 +38,21 @@ function isToolResultUserLine(content) { function promptIdOf(l) { return l.promptId ?? l.prompt_id ?? null; } +function isMetaOf(l) { + return l.isMeta === true || l.is_meta === true; +} +// Client-only slash commands like /model and /clear are surfaced in the +// transcript as a synthetic user-line triple (, +// , ) but never fire UserPromptSubmit, +// so they must NOT count as real turns. Detect them by the unmistakable +// wrappers — these only appear on client-side commands; +// model-bound slash commands (e.g. /superpowers:foo) have +// without any sibling. +function isLocalCommandText(text) { + if (!text) + return false; + return text.startsWith("") || text.startsWith(""); +} function collectSessionDims(l, into) { if (into.permissionMode === undefined) { into.permissionMode = l.permissionMode ?? l.permission_mode; @@ -91,6 +106,24 @@ export function readTranscript(path) { if (recognizedTypeLines === 0) { return legacyResult(path, "no recognizable transcript line types"); } + // Pre-pass: any prompt_id that has at least one line is + // a client-only slash command (/model, /clear, …). Every user line under + // that prompt_id must be excluded from real-turn segmentation; otherwise + // the synthetic triple bumps the real-turn ordinal off-by-N from the + // collector's turnIndex (which only counts true UserPromptSubmit events). + const clientOnlyPromptIds = new Set(); + for (const l of parsed) { + if (l.type !== "user") + continue; + if (l.isSidechain === true) + continue; + const pid = promptIdOf(l); + if (!pid) + continue; + if (isLocalCommandText(textFromContent(l.message?.content))) { + clientOnlyPromptIds.add(pid); + } + } const turns = []; const byPromptId = new Map(); const session = {}; @@ -104,10 +137,21 @@ export function readTranscript(path) { continue; if (isToolResultUserLine(l.message?.content)) continue; + // Skip caveat / meta lines — these are Claude Code annotations, not + // user input, and never fire UserPromptSubmit. + if (isMetaOf(l)) + continue; + const pid = promptIdOf(l); + // Skip every line in a client-only slash-command group (see pre-pass). + if (pid && clientOnlyPromptIds.has(pid)) + continue; + // Defensive: also skip a bare line that somehow lacks + // a prompt_id (older Claude Code builds). + if (isLocalCommandText(textFromContent(l.message?.content))) + continue; if (current) turns.push(current); realIndex += 1; - const pid = promptIdOf(l); current = emptyTurn(pid, realIndex); const t = textFromContent(l.message?.content); if (t) diff --git a/src/server.ts b/src/server.ts index ce4e915..6f17049 100644 --- a/src/server.ts +++ b/src/server.ts @@ -371,7 +371,7 @@ export function startServer( const record = getOrCreateSession(event); void closeCurrentTurn(record).then(() => { record.turnIndex += 1; - record.currentPromptId = event.prompt_id ?? null; + record.currentPromptId = event.prompt_id ?? event.promptId ?? null; const prompt = event.prompt ?? event.message ?? null; record.currentTurnPrompt = prompt; record.currentTurnStart = Date.now() / 1000; diff --git a/src/transcript-reader.ts b/src/transcript-reader.ts index af9a577..0d40dc4 100644 --- a/src/transcript-reader.ts +++ b/src/transcript-reader.ts @@ -29,6 +29,8 @@ export interface TranscriptReadResult { interface Line { type?: string; isSidechain?: boolean; + isMeta?: boolean; + is_meta?: boolean; promptId?: string; prompt_id?: string; permissionMode?: string; @@ -85,6 +87,22 @@ function promptIdOf(l: Line): string | null { return l.promptId ?? l.prompt_id ?? null; } +function isMetaOf(l: Line): boolean { + return l.isMeta === true || l.is_meta === true; +} + +// Client-only slash commands like /model and /clear are surfaced in the +// transcript as a synthetic user-line triple (, +// , ) but never fire UserPromptSubmit, +// so they must NOT count as real turns. Detect them by the unmistakable +// wrappers — these only appear on client-side commands; +// model-bound slash commands (e.g. /superpowers:foo) have +// without any sibling. +function isLocalCommandText(text: string | null): boolean { + if (!text) return false; + return text.startsWith("") || text.startsWith(""); +} + function collectSessionDims(l: Line, into: SessionDimensions): void { if (into.permissionMode === undefined) { into.permissionMode = l.permissionMode ?? l.permission_mode; @@ -139,6 +157,22 @@ export function readTranscript(path: string): TranscriptReadResult { return legacyResult(path, "no recognizable transcript line types"); } + // Pre-pass: any prompt_id that has at least one line is + // a client-only slash command (/model, /clear, …). Every user line under + // that prompt_id must be excluded from real-turn segmentation; otherwise + // the synthetic triple bumps the real-turn ordinal off-by-N from the + // collector's turnIndex (which only counts true UserPromptSubmit events). + const clientOnlyPromptIds = new Set(); + for (const l of parsed) { + if (l.type !== "user") continue; + if (l.isSidechain === true) continue; + const pid = promptIdOf(l); + if (!pid) continue; + if (isLocalCommandText(textFromContent(l.message?.content))) { + clientOnlyPromptIds.add(pid); + } + } + const turns: RealTurn[] = []; const byPromptId = new Map(); const session: SessionDimensions = {}; @@ -151,9 +185,17 @@ export function readTranscript(path: string): TranscriptReadResult { if (l.type === "user") { if (l.isSidechain === true) continue; if (isToolResultUserLine(l.message?.content)) continue; + // Skip caveat / meta lines — these are Claude Code annotations, not + // user input, and never fire UserPromptSubmit. + if (isMetaOf(l)) continue; + const pid = promptIdOf(l); + // Skip every line in a client-only slash-command group (see pre-pass). + if (pid && clientOnlyPromptIds.has(pid)) continue; + // Defensive: also skip a bare line that somehow lacks + // a prompt_id (older Claude Code builds). + if (isLocalCommandText(textFromContent(l.message?.content))) continue; if (current) turns.push(current); realIndex += 1; - const pid = promptIdOf(l); current = emptyTurn(pid, realIndex); const t = textFromContent(l.message?.content); if (t) current.prompt = t; diff --git a/src/types.ts b/src/types.ts index b76db86..c5857a1 100644 --- a/src/types.ts +++ b/src/types.ts @@ -117,8 +117,10 @@ export interface UserPromptSubmitEvent extends AiobsEnvelope { message?: string; /** C1: stable id of this prompt, used to correlate the collector's open * turn to the transcript's real-turn line. Optional — absent on older - * Claude Code; collector falls back to ordinal among real turns. */ + * Claude Code; collector falls back to ordinal among real turns. Some + * builds emit camelCase `promptId`, so both are accepted at read time. */ prompt_id?: string; + promptId?: string; } export interface PreToolUseEvent extends AiobsEnvelope { diff --git a/tests/transcript-reader.test.ts b/tests/transcript-reader.test.ts index dbfdc71..04678f0 100644 --- a/tests/transcript-reader.test.ts +++ b/tests/transcript-reader.test.ts @@ -56,6 +56,44 @@ describe("readTranscript — turn segmentation (C1)", () => { }); }); +describe("readTranscript — synthetic slash-command lines do not count as real turns", () => { + // Claude Code emits synthetic user lines for client-only slash commands like + // /model and /clear — typically a triple: (isMeta:true), + // /foo, . These never fire UserPromptSubmit, + // so they must be excluded from the real-turn index, or the collector's + // turnIndex (which counts UserPromptSubmit events) drifts off-by-N from + // transcript ordinals and selectTurn returns the wrong turn. + it("skips /model-style synthetic triples and aligns ordinals with real prompts", () => { + const p = make( + [ + // /model client-only command: 3 synthetic user lines sharing one promptId + JSON.stringify({ type: "user", isMeta: true, promptId: "cmd1", message: { content: "Caveat: ..." } }), + JSON.stringify({ type: "user", promptId: "cmd1", message: { content: "/model\n" } }), + JSON.stringify({ type: "user", promptId: "cmd1", message: { content: "Set model to Sonnet" } }), + // First real prompt — UserPromptSubmit would fire here, collector turnIndex=0 + JSON.stringify({ type: "user", promptId: "real1", message: { content: "Hello how are you" } }), + JSON.stringify({ type: "assistant", message: { model: "m", usage: { input_tokens: 100, output_tokens: 10 }, content: [{ type: "text", text: "Doing well, thanks!" }] } }), + // Second real prompt — collector turnIndex=1 + JSON.stringify({ type: "user", promptId: "real2", message: { content: "another reply please" } }), + JSON.stringify({ type: "assistant", message: { model: "m", usage: { input_tokens: 200, output_tokens: 20 }, content: [{ type: "text", text: "Here you go." }] } }), + ].join("\n"), + ); + const r = readTranscript(p); + expect(r.degraded).toBe(false); + // Only 2 REAL turns — the 3 /model synthetic lines collapse to 0. + expect(r.turns).toHaveLength(2); + // Ordinal 0 must be "Hello", not the caveat — this is what selectTurn + // returns when UserPromptSubmit lacks prompt_id and falls back to ordinal. + expect(r.turns[0].promptId).toBe("real1"); + expect(r.turns[0].response).toBe("Doing well, thanks!"); + expect(r.turns[1].promptId).toBe("real2"); + expect(r.turns[1].response).toBe("Here you go."); + // byPromptId must NOT contain the synthetic prompt_id at all. + expect(r.byPromptId.has("cmd1")).toBe(false); + expect(selectTurn(r, undefined, 0).turn!.response).toBe("Doing well, thanks!"); + }); +}); + describe("readTranscript — missing/empty file is not parse-degraded", () => { it("missing file → empty turns, degraded false (not format drift)", () => { const r = readTranscript("/tmp/__aiobs_no_such_transcript__.jsonl");