feat(task): add run-state divergence comparison and metrics (#2047)

* feat(task): add run-state divergence comparison and metrics

* fix(task): key divergence dedupe on the event payload and test the schema

* fix(task): sort event keys in the divergence key

Author: edelauna

packages/types/src/__tests__/telemetry.test.ts

@@ -13,6 +13,8 @@ import {
 	ConsecutiveMistakeError,
 	isConsecutiveMistakeError,
 	extractConsecutiveMistakeErrorProperties,
+	rooCodeTelemetryEventSchema,
+	TelemetryEventName,
 } from "../telemetry.js"
 
 describe("telemetry error utilities", () => {
@@ -575,3 +577,79 @@ describe("telemetry error utilities", () => {
 		})
 	})
 })
+
+describe("rooCodeTelemetryEventSchema run-state events", () => {
+	const appProperties = {
+		appName: "zoo-code",
+		appVersion: "1.0.0",
+		vscodeVersion: "1.100.0",
+		platform: "linux",
+		editorName: "vscode",
+		language: "en",
+		mode: "code",
+	}
+	const runStateProperties = {
+		...appProperties,
+		taskId: "task-1",
+		kind: "rejection",
+		eventTag: "askStarted",
+		mismatched: [],
+		phase: "running",
+		streamTag: "none",
+		abort: true,
+	}
+
+	it.each([TelemetryEventName.RUN_STATE_DIVERGENCE, TelemetryEventName.RUN_STATE_KNOWN_RACE])(
+		"keeps the run-state properties of %s",
+		(type) => {
+			const result = rooCodeTelemetryEventSchema.safeParse({
+				type,
+				properties: { ...runStateProperties, raceId: "R-9", mismatched: ["abortReason"] },
+			})
+
+			expect(result.success).toBe(true)
+			expect(result.data?.properties).toMatchObject({
+				taskId: "task-1",
+				kind: "rejection",
+				raceId: "R-9",
+				eventTag: "askStarted",
+				mismatched: ["abortReason"],
+				phase: "running",
+				streamTag: "none",
+				abort: true,
+			})
+		},
+	)
+
+	it("accepts an event without a race ID", () => {
+		const result = rooCodeTelemetryEventSchema.safeParse({
+			type: TelemetryEventName.RUN_STATE_DIVERGENCE,
+			properties: runStateProperties,
+		})
+
+		expect(result.success).toBe(true)
+	})
+
+	it("rejects an unknown kind", () => {
+		const result = rooCodeTelemetryEventSchema.safeParse({
+			type: TelemetryEventName.RUN_STATE_DIVERGENCE,
+			properties: { ...runStateProperties, kind: "other" },
+		})
+
+		expect(result.success).toBe(false)
+	})
+
+	it.each(["taskId", "eventTag", "mismatched", "phase", "streamTag", "abort"])(
+		"rejects an event without %s",
+		(property) => {
+			const { [property]: _omitted, ...properties } = runStateProperties as Record<string, unknown>
+
+			const result = rooCodeTelemetryEventSchema.safeParse({
+				type: TelemetryEventName.RUN_STATE_KNOWN_RACE,
+				properties,
+			})
+
+			expect(result.success).toBe(false)
+		},
+	)
+})

packages/types/src/telemetry.ts

@@ -86,6 +86,8 @@ export enum TelemetryEventName {
 	TELEMETRY_SETTINGS_CHANGED = "Telemetry Settings Changed",
 	MODEL_CACHE_EMPTY_RESPONSE = "Model Cache Empty Response",
 	READ_FILE_LEGACY_FORMAT_USED = "Read File Legacy Format Used",
+	RUN_STATE_DIVERGENCE = "Run State Divergence",
+	RUN_STATE_KNOWN_RACE = "Run State Known Race",
 }
 
 /**
@@ -221,6 +223,20 @@ export const rooCodeTelemetryEventSchema = z.discriminatedUnion("type", [
 		]),
 		properties: telemetryPropertiesSchema,
 	}),
+	z.object({
+		type: z.enum([TelemetryEventName.RUN_STATE_DIVERGENCE, TelemetryEventName.RUN_STATE_KNOWN_RACE]),
+		properties: z.object({
+			...telemetryPropertiesSchema.shape,
+			taskId: z.string(),
+			kind: z.enum(["mismatch", "rejection"]),
+			raceId: z.string().optional(),
+			eventTag: z.string(),
+			mismatched: z.array(z.string()),
+			phase: z.string(),
+			streamTag: z.string(),
+			abort: z.boolean(),
+		}),
+	}),
 	z.object({
 		type: z.literal(TelemetryEventName.TELEMETRY_SETTINGS_CHANGED),
 		properties: z.object({

src/__mocks__/vscode.js

@@ -107,6 +107,7 @@ export const env = {
 	openExternal: () => Promise.resolve(),
 }
 
+export const ExtensionMode = { Production: 1, Development: 2, Test: 3 }
 export const Uri = mockUri
 export const Range = mockRange
 export const Position = mockPosition

src/core/task/Task.ts

@@ -145,6 +145,12 @@ import { shouldAddUserMessageToHistory } from "./messageCounting"
 import { type TaskExecutionContext } from "./providerHandoff"
 import { initialRunState, type Ask, type RunEvent, type RunState } from "./run-state/runState"
 import { applyRunEvent, type RunRejection } from "./run-state/runStateShadow"
+import {
+	divergenceKey,
+	findDivergences,
+	isDivergenceCheckEnabled,
+	reportDivergence,
+} from "./run-state/runStateDivergence"
 
 const MAX_EXPONENTIAL_BACKOFF_SECONDS = 600 // 10 minutes
 const DEFAULT_USAGE_COLLECTION_TIMEOUT_MS = 5000 // 5 seconds
@@ -408,6 +414,8 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
 
 	runState: RunState = initialRunState
 	readonly runRejections: RunRejection[] = []
+	private readonly divergenceCheckEnabled: boolean
+	private readonly reportedDivergences = new Set<string>()
 
 	// API
 	apiConfiguration: ProviderSettings
@@ -753,6 +761,7 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
 		)
 		this.providerRef = new WeakRef(provider)
 		this.globalStoragePath = provider.context.globalStorageUri.fsPath
+		this.divergenceCheckEnabled = isDivergenceCheckEnabled(Package.releaseChannel, provider.context.extensionMode)
 		this.diffViewProvider = new DiffViewProvider(this.cwd, this)
 		this.enableCheckpoints = enableCheckpoints
 		this.checkpointTimeout = checkpointTimeout
@@ -3535,7 +3544,26 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
 	}
 
 	private driveRunEvent(event: RunEvent): void {
-		this.runState = applyRunEvent(this.runState, event, this.runRejections)
+		const { state, rejected } = applyRunEvent(this.runState, event, this.runRejections)
+		this.runState = state
+
+		if (this.divergenceCheckEnabled) {
+			const fields = {
+				abort: this.abort,
+				abandoned: this.abandoned,
+				abortReason: this.abortReason,
+				isInitialized: this.isInitialized,
+				isStreaming: this.isStreaming,
+				didFinishAbortingStream: this.didFinishAbortingStream,
+			}
+			for (const divergence of findDivergences({ event, state, fields, rejected })) {
+				if (this.reportedDivergences.has(divergenceKey(divergence))) {
+					continue
+				}
+				this.reportedDivergences.add(divergenceKey(divergence))
+				reportDivergence(this.taskId, divergence)
+			}
+		}
 	}
 
 	public abortTask(isAbandoned = false): Promise<void> {

src/core/task/__tests__/Task.run-shadow.spec.ts

@@ -1,6 +1,8 @@
 // npx vitest run core/task/__tests__/Task.run-shadow.spec.ts
 
+import * as vscode from "vscode"
 import { providerIdentifiers } from "@roo-code/types/provider-identifiers"
+import { TelemetryEventName } from "@roo-code/types"
 import { TelemetryService } from "@roo-code/telemetry"
 
 import { makeExtensionContext, makeUri } from "../../../test-utils/vscode"
@@ -11,10 +13,13 @@ import { initialRunState } from "../run-state/runState"
 vi.mock("../../../i18n", () => ({ t: (key: string) => key }))
 vi.mock("../../ignore/RooIgnoreController")
 
-function makeTask(): Task {
+function makeTask(extensionMode = vscode.ExtensionMode.Production): Task {
 	const provider = {
 		log: vi.fn(),
-		context: makeExtensionContext({ globalStorageUri: makeUri("/mock/global-storage") }),
+		context: makeExtensionContext({
+			globalStorageUri: makeUri("/mock/global-storage"),
+			extensionMode,
+		}),
 	}
 	// Double cast: the constructor reads only these provider members.
 	return new Task({
@@ -129,3 +134,71 @@ describe("Task run-state write helpers", () => {
 		expect(task.runRejections).toEqual([])
 	})
 })
+
+describe("Task run-state divergence report", () => {
+	let warn: ReturnType<typeof vi.spyOn>
+	let captureEvent: ReturnType<typeof vi.spyOn>
+
+	const divergenceMetrics = () =>
+		captureEvent.mock.calls.filter(
+			([name]: unknown[]) =>
+				name === TelemetryEventName.RUN_STATE_DIVERGENCE || name === TelemetryEventName.RUN_STATE_KNOWN_RACE,
+		)
+
+	const divergenceWarnings = () =>
+		warn.mock.calls.filter(([message]: unknown[]) => message === "[RunState] divergence")
+
+	beforeEach(() => {
+		if (!TelemetryService.hasInstance()) {
+			TelemetryService.createInstance([])
+		}
+		warn = vi.spyOn(console, "warn").mockImplementation(() => {})
+		captureEvent = vi.spyOn(TelemetryService.instance, "captureEvent").mockImplementation(() => {})
+	})
+
+	afterEach(() => {
+		vi.restoreAllMocks()
+	})
+
+	it("logs and counts a rejected event once when the check is enabled", () => {
+		const task = makeTask(vscode.ExtensionMode.Test)
+
+		task.requestAbandon()
+
+		expect(divergenceWarnings()).toHaveLength(1)
+		expect(divergenceWarnings()[0][1]).toMatchObject({
+			taskId: task.taskId,
+			kind: "rejection",
+			event: { tag: "abandonRequested" },
+			state: initialRunState,
+			mismatched: [],
+		})
+		expect(divergenceMetrics()).toEqual([
+			[
+				TelemetryEventName.RUN_STATE_DIVERGENCE,
+				expect.objectContaining({ taskId: task.taskId, kind: "rejection", eventTag: "abandonRequested" }),
+			],
+		])
+	})
+
+	it("reports a repeated rejection of the same event once per task", () => {
+		const task = makeTask(vscode.ExtensionMode.Test)
+
+		task.requestAbandon()
+		task.requestAbandon()
+
+		expect(task.runRejections).toHaveLength(2)
+		expect(divergenceWarnings()).toHaveLength(1)
+		expect(divergenceMetrics()).toHaveLength(1)
+	})
+
+	it("stays silent for a production build on the stable channel", () => {
+		const task = makeTask(vscode.ExtensionMode.Production)
+
+		task.requestAbandon()
+
+		expect(task.runRejections).toHaveLength(1)
+		expect(divergenceWarnings()).toHaveLength(0)
+		expect(divergenceMetrics()).toHaveLength(0)
+	})
+})