diff --git a/.changeset/instrumentation-provider-layout.md b/.changeset/instrumentation-provider-layout.md new file mode 100644 index 0000000000..1e40d0691a --- /dev/null +++ b/.changeset/instrumentation-provider-layout.md @@ -0,0 +1,5 @@ +--- +"eve": patch +--- + +Add the experimental `agent/instrumentation/` provider layout with durable lifecycle handlers, including user input boundaries and action settlement-time, outcome, error-code, and usage metadata, final setup context, reserved OpenTelemetry destinations, and coordinated flush and shutdown. OpenTelemetry singleton settings and destinations are exposed through `eve/instrumentation/otel`, and eve's AI SDK bridge composes with registered integrations. diff --git a/packages/eve/package.json b/packages/eve/package.json index c59c49f037..bfda4d0eb9 100644 --- a/packages/eve/package.json +++ b/packages/eve/package.json @@ -206,6 +206,11 @@ "import": "./dist/src/public/instrumentation/index.js", "default": "./dist/src/public/instrumentation/index.js" }, + "./instrumentation/otel": { + "types": "./dist/src/public/instrumentation/otel.d.ts", + "import": "./dist/src/public/instrumentation/otel.js", + "default": "./dist/src/public/instrumentation/otel.js" + }, "./schedules": { "types": "./dist/src/public/schedules/index.d.ts", "import": "./dist/src/public/schedules/index.js", diff --git a/packages/eve/src/compiler/manifest.ts b/packages/eve/src/compiler/manifest.ts index aec6a0b597..8adfe3cde9 100644 --- a/packages/eve/src/compiler/manifest.ts +++ b/packages/eve/src/compiler/manifest.ts @@ -428,6 +428,7 @@ const compiledAgentConfigSchema: z.ZodType = z dynamicModel: compiledDynamicModelDefinitionSchema.optional(), experimental: z .object({ + instrumentationProviders: z.boolean().optional(), subagentPersistentSessions: z.boolean().optional(), workflow: compiledAgentWorkflowDefinitionSchema.optional(), }) @@ -833,6 +834,7 @@ export function createCompiledAgentNodeManifest(input: { input.config.experimental === undefined ? undefined : { + instrumentationProviders: input.config.experimental.instrumentationProviders, subagentPersistentSessions: input.config.experimental.subagentPersistentSessions, workflow: input.config.experimental.workflow === undefined diff --git a/packages/eve/src/compiler/normalize-agent-config.ts b/packages/eve/src/compiler/normalize-agent-config.ts index 710f6fc68a..a8412bea81 100644 --- a/packages/eve/src/compiler/normalize-agent-config.ts +++ b/packages/eve/src/compiler/normalize-agent-config.ts @@ -176,6 +176,10 @@ function normalizeExperimentalDefinition( const compiledExperimental: Mutable> = {}; + if (experimental.instrumentationProviders !== undefined) { + compiledExperimental.instrumentationProviders = experimental.instrumentationProviders; + } + if (experimental.subagentPersistentSessions !== undefined) { compiledExperimental.subagentPersistentSessions = experimental.subagentPersistentSessions; } diff --git a/packages/eve/src/discover/agent.integration.test.ts b/packages/eve/src/discover/agent.integration.test.ts index fb726983cd..39b6daf100 100644 --- a/packages/eve/src/discover/agent.integration.test.ts +++ b/packages/eve/src/discover/agent.integration.test.ts @@ -511,6 +511,23 @@ describe("discoverAgent (memory)", () => { ]); }); + it("recognizes the instrumentation provider directory", async () => { + const project = buildMemoryAgentProject({ + agentDirectories: ["instrumentation"], + agentFiles: { + "instructions.md": "You are a precise assistant.", + }, + }); + + const result = await discoverAgent({ + agentRoot: project.agentRoot, + appRoot: project.appRoot, + source: project.source, + }); + + expect(result.diagnostics).toEqual([]); + }); + it("rejects authored tool filenames that violate the tool-name charset", async () => { const project = buildMemoryAgentProject({ agentFiles: { diff --git a/packages/eve/src/discover/filesystem.ts b/packages/eve/src/discover/filesystem.ts index 814233d3c8..2e96b3d04b 100644 --- a/packages/eve/src/discover/filesystem.ts +++ b/packages/eve/src/discover/filesystem.ts @@ -44,6 +44,7 @@ export type AgentRootEntryKind = | "extensions-directory" | "hooks-directory" | "ignored-directory" + | "instrumentation-directory" | "instructions-directory" | "instructions-markdown" | "instructions-module" @@ -179,6 +180,10 @@ export function classifyAgentRootEntry( return "instructions-directory"; } + if (name === "instrumentation") { + return "instrumentation-directory"; + } + if (name === "lib") { return "lib-directory"; } diff --git a/packages/eve/src/evals/cli/eval.ts b/packages/eve/src/evals/cli/eval.ts index 26a7090752..2ed4991ac0 100644 --- a/packages/eve/src/evals/cli/eval.ts +++ b/packages/eve/src/evals/cli/eval.ts @@ -1,8 +1,13 @@ +import { randomUUID } from "node:crypto"; import { readFile } from "node:fs/promises"; import { basename, join } from "node:path"; import { loadDevelopmentEnvironmentFiles } from "#cli/dev/environment.js"; import { shutdownActiveSandboxHandles } from "#execution/sandbox/active-handles.js"; +import { + EVE_EVALUATION_ENV_FLAG, + EVE_EVALUATION_RUN_ID_ENV, +} from "#internal/application/dev-environment.js"; import { resolveApplicationRoot } from "#internal/application/paths.js"; import { createDevelopmentServer, type DevelopmentServer } from "#internal/nitro/host.js"; import { createEvalClient } from "#evals/cli/eval-client.js"; @@ -141,6 +146,10 @@ export async function runEvalCommand( url: options.url, }); } else { + // Set before the server boots, because a provider's `setup` reads it + // once at startup and never again. + process.env[EVE_EVALUATION_ENV_FLAG] = "1"; + process.env[EVE_EVALUATION_RUN_ID_ENV] = randomUUID(); devServer = createDevelopmentServer(appRoot, { host: "127.0.0.1", port: 0 }); const started = await devServer.start(); client = await createEvalClient({ kind: "local", url: started.url }); diff --git a/packages/eve/src/harness/ai-sdk-hook-bridge.test.ts b/packages/eve/src/harness/ai-sdk-hook-bridge.test.ts index 0a51032d6e..f0fbea575f 100644 --- a/packages/eve/src/harness/ai-sdk-hook-bridge.test.ts +++ b/packages/eve/src/harness/ai-sdk-hook-bridge.test.ts @@ -94,7 +94,10 @@ describe("createAiSdkHookBridge", () => { it("passes the identity captured at model-call start to the context runner", async () => { const ids: string[] = []; const hooks = createInstrumentationHooks([ - { events: { "model.call.started": (event) => void ids.push(event.idempotencyKey) } }, + { + events: { "model.call.started": (event) => void ids.push(event.idempotencyKey) }, + name: "identity", + }, ]); const bridge = createAiSdkHookBridge(scope, hooks, (operation, execute) => { ids.push(operation.idempotencyKey); @@ -132,7 +135,10 @@ describe("createAiSdkHookBridge", () => { it("derives replay-stable model identity without the AI SDK call ID", async () => { const keys: string[] = []; const hooks = createInstrumentationHooks([ - { events: { "model.call.started": (event) => void keys.push(event.idempotencyKey) } }, + { + events: { "model.call.started": (event) => void keys.push(event.idempotencyKey) }, + name: "identity", + }, ]); for (const callId of ["sdk-random-1", "sdk-random-2"]) { @@ -155,6 +161,7 @@ describe("createAiSdkHookBridge", () => { events.push(event); }, }, + name: "metadata", }, ]); const bridge = createAiSdkHookBridge(scope, hooks); @@ -183,11 +190,13 @@ describe("createAiSdkHookBridge", () => { throw new Error("provider failed"); }, }, + name: "failing", }, { events: { "model.call.completed": after, }, + name: "observer", }, ]); const bridge = createAiSdkHookBridge(scope, hooks); @@ -211,7 +220,9 @@ describe("createAiSdkHookBridge", () => { it("terminalizes started operations when the attempt errors", async () => { const after = vi.fn(); - const hooks = createInstrumentationHooks([{ events: { "model.call.failed": after } }]); + const hooks = createInstrumentationHooks([ + { events: { "model.call.failed": after }, name: "terminal" }, + ]); const bridge = createAiSdkHookBridge(scope, hooks); await Reflect.apply(bridge.onLanguageModelCallStart!, bridge, [ @@ -252,8 +263,8 @@ describe("createAiSdkHookBridge", () => { }); const started = vi.fn(); const hooks = createInstrumentationHooks([ - { events: { "step.attempt.started": mutator } }, - { events: { "step.attempt.started": started } }, + { events: { "step.attempt.started": mutator }, name: "mutator" }, + { events: { "step.attempt.started": started }, name: "observer" }, ]); const bridge = createAiSdkHookBridge(scope, hooks); @@ -288,7 +299,10 @@ describe("createAiSdkHookBridge", () => { expect(Object.isFrozen(event.usage.inputTokenDetails)).toBe(true); }); const hooks = createInstrumentationHooks([ - { events: { "model.call.completed": after, "model.call.started": before } }, + { + events: { "model.call.completed": after, "model.call.started": before }, + name: "model", + }, ]); const bridge = createAiSdkHookBridge(scope, hooks); @@ -404,7 +418,10 @@ describe("createAiSdkHookBridge", () => { expect(Object.isFrozen(event.output)).toBe(true); }); const hooks = createInstrumentationHooks([ - { events: { "tool.call.completed": after, "tool.call.started": before } }, + { + events: { "tool.call.completed": after, "tool.call.started": before }, + name: "tool", + }, ]); const bridge = createAiSdkHookBridge(scope, hooks); const toolCall = { input: { q: "eve" }, toolCallId: "tool-1", toolName: "search" }; @@ -478,7 +495,9 @@ describe("createAiSdkHookBridge", () => { it("skips a terminal handler when the operation never started", async () => { const completed = vi.fn(); - const hooks = createInstrumentationHooks([{ events: { "model.call.completed": completed } }]); + const hooks = createInstrumentationHooks([ + { events: { "model.call.completed": completed }, name: "terminal" }, + ]); const bridge = createAiSdkHookBridge(scope, hooks); // No onLanguageModelCallStart, so the bridge holds no id and publishes @@ -516,6 +535,7 @@ describe("createAiSdkHookBridge", () => { terminalStates.set(event.idempotencyKey, started.get(event.idempotencyKey)); }, }, + name: "parallel", }, ]); const bridge = createAiSdkHookBridge(scope, hooks); diff --git a/packages/eve/src/harness/ai-sdk-telemetry.test.ts b/packages/eve/src/harness/ai-sdk-telemetry.test.ts new file mode 100644 index 0000000000..6049c90cae --- /dev/null +++ b/packages/eve/src/harness/ai-sdk-telemetry.test.ts @@ -0,0 +1,25 @@ +import type { Telemetry } from "ai"; +import { afterEach, describe, expect, it } from "vitest"; + +import { getRegisteredTelemetryIntegrations } from "#harness/ai-sdk-telemetry.js"; + +describe("getRegisteredTelemetryIntegrations", () => { + const original = globalThis.AI_SDK_TELEMETRY_INTEGRATIONS; + + afterEach(() => { + globalThis.AI_SDK_TELEMETRY_INTEGRATIONS = original; + }); + + it("is empty when nothing has registered", () => { + globalThis.AI_SDK_TELEMETRY_INTEGRATIONS = undefined; + expect(getRegisteredTelemetryIntegrations()).toEqual([]); + }); + + it("reports the integrations in registration order", () => { + const first: Telemetry = { onStart() {} }; + const second: Telemetry = { onStart() {} }; + globalThis.AI_SDK_TELEMETRY_INTEGRATIONS = [first, second]; + + expect(getRegisteredTelemetryIntegrations()).toEqual([first, second]); + }); +}); diff --git a/packages/eve/src/harness/otel-integration.ts b/packages/eve/src/harness/ai-sdk-telemetry.ts similarity index 50% rename from packages/eve/src/harness/otel-integration.ts rename to packages/eve/src/harness/ai-sdk-telemetry.ts index d34108a791..02954907f0 100644 --- a/packages/eve/src/harness/otel-integration.ts +++ b/packages/eve/src/harness/ai-sdk-telemetry.ts @@ -16,10 +16,17 @@ export function ensureOtelIntegration(): void { return; } registered = true; - registerTelemetry(createOtelIntegration()); + registerTelemetry(new OpenTelemetry({ runtimeContext: true })); } -/** Creates the existing OTel integration for explicit per-call composition. */ -export function createOtelIntegration(): Telemetry { - return new OpenTelemetry({ runtimeContext: true }); +/** + * Every integration currently registered with the AI SDK — eve's own, plus any + * an authored instrumentation module added with `registerTelemetry`. + * + * A per-call `integrations` list replaces the registered ones rather than + * adding to them, so anything that passes integrations per call has to carry + * these forward or they stop receiving events. + */ +export function getRegisteredTelemetryIntegrations(): readonly Telemetry[] { + return globalThis.AI_SDK_TELEMETRY_INTEGRATIONS ?? []; } diff --git a/packages/eve/src/harness/instrumentation-config.test.ts b/packages/eve/src/harness/instrumentation-config.test.ts index af254587d2..b142437bbb 100644 --- a/packages/eve/src/harness/instrumentation-config.test.ts +++ b/packages/eve/src/harness/instrumentation-config.test.ts @@ -1,5 +1,7 @@ import { describe, expect, it, vi } from "vitest"; +import type { InstrumentationSetupContext } from "#public/instrumentation/index.js"; + /** * Regression coverage for the instrumentation-config chunk-isolation failure. * @@ -25,7 +27,7 @@ describe("instrumentation-config chunk-isolation regression", () => { vi.resetModules(); const moduleA = await import("#harness/instrumentation-config.js"); const config = { functionId: "test.instrumentation.cross-module.alice" }; - moduleA.registerInstrumentationConfig(config, { agentName: "test-agent" }); + await moduleA.registerInstrumentationConfig(config, { agentName: "test-agent" }); vi.resetModules(); const moduleB = await import("#harness/instrumentation-config.js"); @@ -40,7 +42,7 @@ describe("instrumentation-config chunk-isolation regression", () => { const { registerInstrumentationConfig } = await import("#harness/instrumentation-config.js"); const canary = { functionId: "test.instrumentation.global-mount.canary" }; - registerInstrumentationConfig(canary, { agentName: "test-agent" }); + await registerInstrumentationConfig(canary, { agentName: "test-agent" }); expect((globalThis as Record)[globalKey]).toBe(canary); }); @@ -51,7 +53,7 @@ describe("instrumentation-config chunk-isolation regression", () => { vi.resetModules(); const moduleA = await import("#harness/instrumentation-config.js"); const config = { functionId: "test.instrumentation.reimport.canary" }; - moduleA.registerInstrumentationConfig(config, { agentName: "test-agent" }); + await moduleA.registerInstrumentationConfig(config, { agentName: "test-agent" }); const firstRef = (globalThis as Record)[globalKey]; vi.resetModules(); @@ -61,13 +63,23 @@ describe("instrumentation-config chunk-isolation regression", () => { expect(secondRef).toBe(firstRef); }); - it("invokes the setup callback with the supplied context", async () => { + it("awaits the setup callback with the resolved context", async () => { vi.resetModules(); const { registerInstrumentationConfig } = await import("#harness/instrumentation-config.js"); - const setup = vi.fn(); - registerInstrumentationConfig({ setup }, { agentName: "weather-agent" }); + const contexts: InstrumentationSetupContext[] = []; + await registerInstrumentationConfig( + { + setup: (context) => { + contexts.push(context); + }, + }, + { agentName: "weather-agent" }, + ); - expect(setup).toHaveBeenCalledExactlyOnceWith({ agentName: "weather-agent" }); + expect(contexts).toHaveLength(1); + expect(contexts[0]?.agentName).toBe("weather-agent"); + expect(contexts[0]?.environment).toMatch(/^(development|production)$/); + expect(contexts[0]?.frameworkVersion).toEqual(expect.any(String)); }); }); diff --git a/packages/eve/src/harness/instrumentation-config.ts b/packages/eve/src/harness/instrumentation-config.ts index 9e24c72244..52ba4c67b4 100644 --- a/packages/eve/src/harness/instrumentation-config.ts +++ b/packages/eve/src/harness/instrumentation-config.ts @@ -1,8 +1,6 @@ -import type { - InstrumentationDefinition, - InstrumentationSetupContext, -} from "#public/instrumentation/index.js"; +import { createInstrumentationSetupContext } from "#harness/instrumentation-setup-context.js"; import { activateOtelSettings } from "#harness/otel-settings.js"; +import type { InstrumentationDefinition } from "#public/instrumentation/index.js"; /** * Process-global store for the authored instrumentation config. @@ -29,26 +27,32 @@ interface InstrumentationConfigGlobal { const globalContainer = globalThis as typeof globalThis & InstrumentationConfigGlobal; /** - * Registers the authored instrumentation config and invokes its `setup` - * callback with the resolved agent name. + * Registers the authored instrumentation config and awaits its `setup` + * callback. * * Called once by the generated instrumentation Nitro plugin at server * startup. Subsequent calls overwrite the previous value. * + * The store write lands before `setup` runs so a synchronous caller sees the + * config without waiting on the returned promise. + * * @internal — not part of the public API. */ export async function registerInstrumentationConfig( config: InstrumentationDefinition, - context: InstrumentationSetupContext, + input: { readonly agentName: string }, ): Promise { globalContainer[INSTRUMENTATION_CONFIG_GLOBAL_KEY] = config; + // The presence of a config is what turns telemetry on in this layout, so the + // settings the harness reads are activated with it rather than by a + // registered pipeline — this layout leaves `registerOTel` to `setup`. activateOtelSettings({ functionId: config.functionId, recordInputs: config.recordInputs, recordOutputs: config.recordOutputs, traceChannelRequests: config.traceChannelRequests === true, }); - await config.setup?.(context); + await config.setup?.(createInstrumentationSetupContext(input.agentName)); } /** diff --git a/packages/eve/src/harness/instrumentation-dispatch.ts b/packages/eve/src/harness/instrumentation-dispatch.ts index 5125fd7383..fffbf3c2b9 100644 --- a/packages/eve/src/harness/instrumentation-dispatch.ts +++ b/packages/eve/src/harness/instrumentation-dispatch.ts @@ -155,17 +155,20 @@ async function dispatchToProvider( const startedBoundary = event.type.endsWith(".started") || event.type === "input.requested"; const owner = stateOwner(event); const providerName = provider.name; - if (isInstrumentationStateAbandoned(providerName, event.idempotencyKey)) return; + const stateNamespace = provider.stateNamespace ?? providerName; + if (isInstrumentationStateAbandoned(stateNamespace, event.idempotencyKey)) return; const handler = provider.events?.[event.type]; if (handler === undefined) return; - const state = instrumentationStateSlot(providerName, event.idempotencyKey, owner); + const state = instrumentationStateSlot(stateNamespace, event.idempotencyKey, owner); try { const settled = await withTimeout( () => (handler as InstrumentationEventHandler)(event, { state }), handlerTimeoutMs, () => { state.revoke(); - if (startedBoundary) abandonInstrumentationState(providerName, event.idempotencyKey, owner); + if (startedBoundary) { + abandonInstrumentationState(stateNamespace, event.idempotencyKey, owner); + } }, ); if (!settled) { diff --git a/packages/eve/src/harness/instrumentation-lifecycle.ts b/packages/eve/src/harness/instrumentation-lifecycle.ts index c8a7bd0226..f9cbce54ad 100644 --- a/packages/eve/src/harness/instrumentation-lifecycle.ts +++ b/packages/eve/src/harness/instrumentation-lifecycle.ts @@ -445,6 +445,8 @@ export type InstrumentationEventHandler = ( /** Internal provider shape mirrored by the future public hook contract. */ export interface InstrumentationProviderDefinition { readonly name: string; + /** Durable state identity, separate from the human-readable log name. */ + readonly stateNamespace?: string; readonly events?: { readonly "step.attempt.started"?: InstrumentationEventHandler; readonly "step.attempt.completed"?: InstrumentationEventHandler; diff --git a/packages/eve/src/harness/instrumentation-native-events.test.ts b/packages/eve/src/harness/instrumentation-native-events.test.ts index cc10078a9b..a8c813dba1 100644 --- a/packages/eve/src/harness/instrumentation-native-events.test.ts +++ b/packages/eve/src/harness/instrumentation-native-events.test.ts @@ -153,6 +153,7 @@ describe("createInstrumentationHandleEvent", () => { }, ]); }); + it("publishes every runtime action and settles it in a replacement worker", async () => { const events: unknown[] = []; const scope = { diff --git a/packages/eve/src/harness/instrumentation-native-events.ts b/packages/eve/src/harness/instrumentation-native-events.ts index bfe369b6ef..477d2469e4 100644 --- a/packages/eve/src/harness/instrumentation-native-events.ts +++ b/packages/eve/src/harness/instrumentation-native-events.ts @@ -50,9 +50,9 @@ export function createInstrumentationHandleEvent( const handleEvent = input.handleEvent; const hooks = input.hooks; - let activeTurnId = input.turnId; const publishedActions = new Set(); const publishedInputs = new Set(); + let activeTurnId = input.turnId; return async (event, messages) => { await handleEvent(event, messages); const lifecycleEvent = toLifecycleEvent(event, input, activeTurnId); diff --git a/packages/eve/src/harness/instrumentation-providers-local.scenario.test.ts b/packages/eve/src/harness/instrumentation-providers-local.scenario.test.ts new file mode 100644 index 0000000000..ca81922e43 --- /dev/null +++ b/packages/eve/src/harness/instrumentation-providers-local.scenario.test.ts @@ -0,0 +1,45 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { + finalizeInstrumentationProviders, + getInstrumentationProviders, + registerInstrumentationProvider, + seedInstrumentationProviders, +} from "#harness/instrumentation-providers.js"; +import { DEVELOPMENT_WORKER_APP_ROOT_ENV } from "#internal/workflow/development-world-protocol.js"; +import { otelIntegration } from "#public/instrumentation/otel.js"; + +const temporaryDirectories: string[] = []; + +afterEach(async () => { + vi.unstubAllEnvs(); + await Promise.all( + temporaryDirectories.splice(0).map((path) => rm(path, { force: true, recursive: true })), + ); +}); + +describe("instrumentation provider local default", () => { + it("registers default local traces and an authored destination in one pipeline", async () => { + const appRoot = await mkdtemp(join(tmpdir(), "eve-provider-local-")); + temporaryDirectories.push(appRoot); + vi.stubEnv(DEVELOPMENT_WORKER_APP_ROOT_ENV, appRoot); + vi.stubEnv("EVE_TRACES", "off"); + + seedInstrumentationProviders(); + await registerInstrumentationProvider({ + agentName: "weather", + slot: "backend", + value: otelIntegration(), + }); + + const runtime = finalizeInstrumentationProviders({ serviceName: "weather" }); + await runtime.forceFlush(); + await runtime.shutdown(); + + expect(getInstrumentationProviders().map(({ slot }) => slot)).toEqual(["backend", "local"]); + }); +}); diff --git a/packages/eve/src/harness/instrumentation-providers-production.scenario.test.ts b/packages/eve/src/harness/instrumentation-providers-production.scenario.test.ts new file mode 100644 index 0000000000..1afb442453 --- /dev/null +++ b/packages/eve/src/harness/instrumentation-providers-production.scenario.test.ts @@ -0,0 +1,33 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { + finalizeInstrumentationProviders, + getInstrumentationProviders, + registerInstrumentationProvider, + seedInstrumentationProviders, +} from "#harness/instrumentation-providers.js"; +import { localTraces } from "#public/instrumentation/otel.js"; + +afterEach(() => { + vi.unstubAllEnvs(); +}); + +describe("instrumentation provider production defaults", () => { + it("keeps authored local traces inert beside Agent Runs", async () => { + vi.stubEnv("EVE_DEV_WORKER_APP_ROOT", undefined); + vi.stubEnv("VERCEL_ENV", "production"); + + seedInstrumentationProviders(); + await registerInstrumentationProvider({ + agentName: "weather", + slot: "local", + value: localTraces(), + }); + + const runtime = finalizeInstrumentationProviders({ serviceName: "weather" }); + await runtime.forceFlush(); + await runtime.shutdown(); + + expect(getInstrumentationProviders().map(({ slot }) => slot)).toEqual(["agent-runs", "local"]); + }); +}); diff --git a/packages/eve/src/harness/instrumentation-providers.integration.test.ts b/packages/eve/src/harness/instrumentation-providers.integration.test.ts new file mode 100644 index 0000000000..dfb6cd0894 --- /dev/null +++ b/packages/eve/src/harness/instrumentation-providers.integration.test.ts @@ -0,0 +1,92 @@ +import { beforeEach, describe, expect, it } from "vitest"; + +import { turnIdempotencyKey } from "#harness/instrumentation-lifecycle.js"; +import { + finalizeInstrumentationProviders, + registerInstrumentationProvider, +} from "#harness/instrumentation-providers.js"; +import { defineInstrumentation } from "#public/instrumentation/index.js"; + +const REGISTRY_GLOBAL_KEY = Symbol.for("eve.harness-instrumentation-providers"); +const RUNTIME_GLOBAL_KEY = Symbol.for("eve.instrumentation-runtime"); + +function deferred(): { readonly promise: Promise; readonly resolve: () => void } { + let resolve!: () => void; + const promise = new Promise((complete) => { + resolve = complete; + }); + return { promise, resolve }; +} + +describe("authored instrumentation provider dispatch", () => { + beforeEach(() => { + delete (globalThis as Record)[REGISTRY_GLOBAL_KEY]; + delete (globalThis as Record)[RUNTIME_GLOBAL_KEY]; + }); + + it("awaits setup in slot order while event handlers overlap", async () => { + const firstSetup = deferred(); + const firstHandler = deferred(); + const secondHandler = deferred(); + const order: string[] = []; + + const firstRegistration = registerInstrumentationProvider({ + agentName: "weather", + slot: "first", + value: defineInstrumentation({ + events: { + "turn.started": async () => { + order.push("first:handler"); + await firstHandler.promise; + }, + }, + setup: async () => { + order.push("first:setup"); + await firstSetup.promise; + order.push("first:setup-complete"); + }, + }), + }); + + expect(order).toEqual(["first:setup"]); + firstSetup.resolve(); + await firstRegistration; + + await registerInstrumentationProvider({ + agentName: "weather", + slot: "second", + value: defineInstrumentation({ + events: { + "turn.started": async () => { + order.push("second:handler"); + await secondHandler.promise; + }, + }, + setup: () => void order.push("second:setup"), + }), + }); + + expect(order).toEqual(["first:setup", "first:setup-complete", "second:setup"]); + + const runtime = finalizeInstrumentationProviders({ serviceName: "weather" }); + const publication = runtime.hooks.publish({ + idempotencyKey: turnIdempotencyKey("session-1", "turn-1"), + rootSessionId: "session-1", + sequence: 0, + sessionId: "session-1", + turnId: "turn-1", + type: "turn.started", + }); + + expect(order).toEqual([ + "first:setup", + "first:setup-complete", + "second:setup", + "first:handler", + "second:handler", + ]); + firstHandler.resolve(); + secondHandler.resolve(); + await publication; + }); +}); diff --git a/packages/eve/src/harness/instrumentation-providers.test.ts b/packages/eve/src/harness/instrumentation-providers.test.ts new file mode 100644 index 0000000000..1b28428693 --- /dev/null +++ b/packages/eve/src/harness/instrumentation-providers.test.ts @@ -0,0 +1,255 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { turnIdempotencyKey } from "#harness/instrumentation-lifecycle.js"; +import { + EVE_EVALUATION_ENV_FLAG, + EVE_EVALUATION_RUN_ID_ENV, +} from "#internal/application/dev-environment.js"; +import { + finalizeInstrumentationProviders, + getInstrumentationProviders, + registerInstrumentationProvider, + seedInstrumentationProviders, + shutdownInstrumentationProviders, +} from "#harness/instrumentation-providers.js"; +import { DEVELOPMENT_WORKER_APP_ROOT_ENV } from "#internal/workflow/development-world-protocol.js"; +import { defineInstrumentation } from "#public/instrumentation/index.js"; +import { agentRuns, localTraces, otelIntegration } from "#public/instrumentation/otel.js"; +import { + disableInstrumentation, + type ProviderSetupContext, +} from "#public/instrumentation/provider.js"; + +const REGISTRY_GLOBAL_KEY = Symbol.for("eve.harness-instrumentation-providers"); +const RUNTIME_GLOBAL_KEY = Symbol.for("eve.instrumentation-runtime"); + +function register(slot: string, value: unknown): Promise { + return registerInstrumentationProvider({ agentName: "weather-agent", slot, value }); +} + +describe("registerInstrumentationProvider", () => { + beforeEach(() => { + vi.unstubAllEnvs(); + delete (globalThis as Record)[REGISTRY_GLOBAL_KEY]; + delete (globalThis as Record)[RUNTIME_GLOBAL_KEY]; + }); + + it("registers a provider under its slot", async () => { + const provider = defineInstrumentation({ events: {} }); + await register("otel", provider); + + expect(getInstrumentationProviders()).toEqual([{ provider, slot: "otel" }]); + }); + + it("preserves registration order across slots", async () => { + await register("agent-runs", defineInstrumentation({})); + await register("local", defineInstrumentation({})); + + expect(getInstrumentationProviders().map(({ slot }) => slot)).toEqual(["agent-runs", "local"]); + }); + + it("awaits setup with the resolved provider context", async () => { + const contexts: ProviderSetupContext[] = []; + await register( + "otel", + defineInstrumentation({ + setup: (context) => { + contexts.push(context); + }, + shutdown: () => {}, + }), + ); + + expect(contexts).toHaveLength(1); + expect(contexts[0]?.agentName).toBe("weather-agent"); + expect(contexts[0]?.environment).toMatch(/^(development|preview|production)$/); + expect(contexts[0]?.evaluation).toBeUndefined(); + expect(contexts[0]?.frameworkVersion).toEqual(expect.any(String)); + }); + + it("reports Vercel preview separately from production", async () => { + vi.stubEnv("VERCEL_ENV", "preview"); + const contexts: ProviderSetupContext[] = []; + + await register( + "otel", + defineInstrumentation({ setup: (context) => void contexts.push(context) }), + ); + + expect(contexts[0]?.environment).toBe("preview"); + }); + + it("reports an evaluation server to setup", async () => { + vi.stubEnv(EVE_EVALUATION_ENV_FLAG, "1"); + vi.stubEnv(EVE_EVALUATION_RUN_ID_ENV, "eval-run-1"); + + const contexts: ProviderSetupContext[] = []; + await register( + "otel", + defineInstrumentation({ + setup: (context) => { + contexts.push(context); + }, + }), + ); + + expect(contexts[0]?.evaluation).toEqual({ runId: "eval-run-1" }); + }); + + it("registers nothing for a disabled slot", async () => { + await register("local", disableInstrumentation()); + + expect(getInstrumentationProviders()).toEqual([]); + }); + + it("removes an already-registered provider when a later file disables the slot", async () => { + await register("local", defineInstrumentation({})); + await register("local", disableInstrumentation()); + + expect(getInstrumentationProviders()).toEqual([]); + }); + + // A slot that registers nothing is telemetry that silently does nothing — + // the failure this surface exists to prevent — so the shape check throws + // rather than skipping the file. + it.each([ + ["a bare object", { events: {} }], + ["a function", () => {}], + ["undefined", undefined], + ["null", null], + ])("throws when the default export is %s", async (_label, value) => { + await expect(register("otel", value)).rejects.toThrow( + /The default export of "instrumentation\/otel" is not an instrumentation provider/, + ); + }); +}); + +describe("seedInstrumentationProviders", () => { + beforeEach(() => { + vi.unstubAllEnvs(); + delete (globalThis as Record)[REGISTRY_GLOBAL_KEY]; + delete (globalThis as Record)[RUNTIME_GLOBAL_KEY]; + vi.stubEnv("EVE_TRACES", "off"); + vi.stubEnv("VERCEL_ENV", undefined); + vi.stubEnv(DEVELOPMENT_WORKER_APP_ROOT_ENV, "/tmp/eve-seed-test"); + }); + + it("sorts default local traces with authored destinations", async () => { + seedInstrumentationProviders(); + await register("backend", otelIntegration()); + + expect(getInstrumentationProviders().map(({ slot }) => slot)).toEqual(["backend", "local"]); + }); + + it("seeds Agent Runs only in Vercel production", () => { + vi.stubEnv(DEVELOPMENT_WORKER_APP_ROOT_ENV, undefined); + vi.stubEnv("VERCEL_ENV", "production"); + + seedInstrumentationProviders(); + + expect(getInstrumentationProviders().map(({ slot }) => slot)).toEqual(["agent-runs"]); + }); + + it("lets an authored reserved slot reconfigure or disable its default", async () => { + seedInstrumentationProviders(); + const authored = localTraces({ recordInputs: false }); + await register("local", authored); + expect(getInstrumentationProviders()).toEqual([{ provider: authored, slot: "local" }]); + + await register("local", disableInstrumentation()); + expect(getInstrumentationProviders()).toEqual([]); + }); + + it("lets an authored Agent Runs slot narrow the production default", async () => { + vi.stubEnv(DEVELOPMENT_WORKER_APP_ROOT_ENV, undefined); + vi.stubEnv("VERCEL_ENV", "production"); + seedInstrumentationProviders(); + const authored = agentRuns({ recordOutputs: false }); + + await register("agent-runs", authored); + + expect(getInstrumentationProviders()).toEqual([{ provider: authored, slot: "agent-runs" }]); + }); + + it("sorts built-ins, authored slots, and reserved-slot replacements together", async () => { + vi.stubEnv("VERCEL_ENV", "production"); + seedInstrumentationProviders(); + await register("zeta", defineInstrumentation({})); + await register("audit", defineInstrumentation({})); + await register("local", localTraces({ recordInputs: false })); + + expect(getInstrumentationProviders().map(({ slot }) => slot)).toEqual([ + "agent-runs", + "audit", + "local", + "zeta", + ]); + }); +}); + +describe("finalizeInstrumentationProviders", () => { + const turnStarted = { + idempotencyKey: turnIdempotencyKey("session-1", "turn-1"), + rootSessionId: "session-1", + sequence: 0, + sessionId: "session-1", + turnId: "turn-1", + type: "turn.started", + } as const; + + beforeEach(() => { + vi.unstubAllEnvs(); + delete (globalThis as Record)[REGISTRY_GLOBAL_KEY]; + delete (globalThis as Record)[RUNTIME_GLOBAL_KEY]; + }); + + it("publishes to an authored handler", async () => { + const started = vi.fn(); + await register("rows", defineInstrumentation({ events: { "turn.started": started } })); + + const runtime = finalizeInstrumentationProviders({ serviceName: "weather-agent" }); + await runtime.hooks.publish(turnStarted); + + expect(started).toHaveBeenCalledOnce(); + expect(started.mock.calls[0]?.[0]).toMatchObject({ turnId: "turn-1" }); + }); + + it("still runs execution when no destination was declared", async () => { + // A directory with no `otel()` has nothing to hang a span on, so + // `runInContext` degrades to running the work directly rather than + // going missing. + await register("rows", defineInstrumentation({})); + + const runtime = finalizeInstrumentationProviders({ serviceName: "weather-agent" }); + const result = await runtime.runInContext( + { + idempotencyKey: "tool:session-1:turn-1:0:0:call-1:0", + scope: { + attemptId: "session-1:turn-1:0:0", + attemptIndex: 0, + sessionId: "session-1", + stepIndex: 0, + turnId: "turn-1", + }, + type: "tool.call", + }, + () => Promise.resolve("ran"), + ); + + expect(result).toBe("ran"); + }); + + it("drains and releases every provider", async () => { + const flush = vi.fn(); + const shutdown = vi.fn(); + await register("rows", defineInstrumentation({ flush, shutdown })); + + const runtime = finalizeInstrumentationProviders({ serviceName: "weather-agent" }); + await runtime.forceFlush(); + await shutdownInstrumentationProviders(); + await shutdownInstrumentationProviders(); + + expect(flush).toHaveBeenCalledOnce(); + expect(shutdown).toHaveBeenCalledOnce(); + }); +}); diff --git a/packages/eve/src/harness/instrumentation-providers.ts b/packages/eve/src/harness/instrumentation-providers.ts new file mode 100644 index 0000000000..13fb2a3eb9 --- /dev/null +++ b/packages/eve/src/harness/instrumentation-providers.ts @@ -0,0 +1,153 @@ +import type { InstrumentationProviderDefinition } from "#harness/instrumentation-lifecycle.js"; +import { + getInstrumentationRuntime, + type InstrumentationRuntime, +} from "#harness/instrumentation-runtime.js"; +import { createInstrumentationSetupContext } from "#harness/instrumentation-setup-context.js"; +import { resolveInstalledPackageInfo } from "#internal/application/package.js"; +import { DEVELOPMENT_WORKER_APP_ROOT_ENV } from "#internal/workflow/development-world-protocol.js"; +import { agentRuns, localTraces } from "#public/instrumentation/otel.js"; +import { installInstrumentationRuntime } from "#tracing/install-instrumentation-runtime.js"; +import { collectOtelPipeline } from "#tracing/otel-declaration.js"; +import { + isInstrumentationDisabled, + isInstrumentationProvider, + type InstrumentationProvider, +} from "#public/instrumentation/provider.js"; + +/** + * Process-global registry of the providers authored under + * `agent/instrumentation/`. + * + * Rooted on `globalThis` for the same reason the single-config store is: the + * generated Nitro plugin stays external by `file://` URL while the harness + * chunk is inlined, so the two resolve to distinct ESM module instances and + * need one shared source of truth. + */ +const INSTRUMENTATION_PROVIDERS_GLOBAL_KEY = Symbol.for("eve.harness-instrumentation-providers"); + +/** One provider and the `instrumentation/.ts` file it came from. */ +export interface RegisteredInstrumentationProvider { + readonly provider: InstrumentationProvider; + readonly slot: string; +} + +interface InstrumentationProvidersGlobal { + [INSTRUMENTATION_PROVIDERS_GLOBAL_KEY]?: Map; +} + +const globalContainer = globalThis as typeof globalThis & InstrumentationProvidersGlobal; + +function providerRegistry(): Map { + const existing = globalContainer[INSTRUMENTATION_PROVIDERS_GLOBAL_KEY]; + if (existing !== undefined) { + return existing; + } + + const created = new Map(); + globalContainer[INSTRUMENTATION_PROVIDERS_GLOBAL_KEY] = created; + return created; +} + +/** Fills reserved slots before authored files may reconfigure or disable them. */ +export function seedInstrumentationProviders(): void { + const registry = providerRegistry(); + if (process.env.VERCEL_ENV === "production") registry.set("agent-runs", agentRuns()); + if (process.env[DEVELOPMENT_WORKER_APP_ROOT_ENV] !== undefined) { + registry.set("local", localTraces()); + } +} + +/** + * Registers one authored provider and awaits its `setup`. + * + * Called once per `instrumentation/.ts` by the generated Nitro plugin at + * server startup, before any event is published. A default export that is not + * a `defineInstrumentation` result throws rather than being skipped: a slot + * that registers nothing is telemetry that silently does nothing, which is the + * failure this surface exists to prevent. + * + * @internal — not part of the public API. + */ +export async function registerInstrumentationProvider(input: { + readonly agentName: string; + readonly slot: string; + readonly value: unknown; +}): Promise { + if (isInstrumentationDisabled(input.value)) { + providerRegistry().delete(input.slot); + return; + } + + if (!isInstrumentationProvider(input.value)) { + throw new Error( + `The default export of "instrumentation/${input.slot}" is not an instrumentation provider. Return the result of \`defineInstrumentation\` or \`disableInstrumentation\` from it.`, + ); + } + + providerRegistry().set(input.slot, input.value); + await input.value.setup?.(createInstrumentationSetupContext(input.agentName)); +} + +/** Registered providers in slot order. @internal */ +export function getInstrumentationProviders(): readonly RegisteredInstrumentationProvider[] { + return [...providerRegistry()] + .sort(([left], [right]) => left.localeCompare(right)) + .map(([slot, provider]) => ({ provider, slot })); +} + +/** + * Installs the process instrumentation runtime from the registered providers. + * + * Called once by the generated Nitro plugin after every slot has registered, + * which is also why it cannot happen inside `setup`: the OpenTelemetry pipeline + * is the union of every destination declared in the directory, so no single + * file knows enough to build it. A `setup` that reaches for a tracer therefore + * gets the no-op one; declare destinations as values and let this assemble + * them. + * + * A directory that declared no OpenTelemetry at all still gets a bus. Its + * providers see every event; they just have no spans to hang them on. + * + * @internal — not part of the public API. + */ +export function finalizeInstrumentationProviders(input: { + readonly serviceName: string; +}): InstrumentationRuntime { + const registered = getInstrumentationProviders(); + return installInstrumentationRuntime({ + collected: collectOtelPipeline(registered.map((entry) => entry.provider)), + frameworkVersion: resolveInstalledPackageInfo().version, + providers: registered.map(toProviderDefinition), + serviceName: input.serviceName, + }); +} + +/** + * Releases every registered provider and OTel processor from Nitro's close + * hook, the last point a buffered exporter can still reach the network. + */ +export async function shutdownInstrumentationProviders(): Promise { + await getInstrumentationRuntime()?.shutdown(); +} + +/** + * Adapts an authored provider onto the internal bus contract. + * + * The event maps are the same shape — the public one is derived from the + * internal union and both handlers take `(event, ctx)` — so only the name has + * to be supplied. + */ +function toProviderDefinition( + entry: RegisteredInstrumentationProvider, +): InstrumentationProviderDefinition { + return { + events: entry.provider.events as InstrumentationProviderDefinition["events"], + flush: entry.provider.flush, + // The file the provider came from, which is the only name an author can + // recognize in a log line about it. + name: entry.slot, + shutdown: entry.provider.shutdown, + stateNamespace: `authored:${entry.slot}`, + }; +} diff --git a/packages/eve/src/harness/instrumentation-setup-context.ts b/packages/eve/src/harness/instrumentation-setup-context.ts new file mode 100644 index 0000000000..b23dfe575a --- /dev/null +++ b/packages/eve/src/harness/instrumentation-setup-context.ts @@ -0,0 +1,31 @@ +import { + isEveDevEnvironment, + resolveEveEvaluationRunId, +} from "#internal/application/dev-environment.js"; +import { resolveInstalledPackageInfo } from "#internal/application/package.js"; +import type { ProviderSetupContext } from "#public/instrumentation/provider.js"; + +/** + * Builds the context handed to an authored `setup` at server startup. + * + * Shared by both layouts so the two cannot drift: a divergent context type + * would leave `defineInstrumentation`'s union without a contextual signature + * for `setup`, silently making every authored `setup(context)` parameter an + * implicit `any`. + * + * @internal — not part of the public API. + */ +export function createInstrumentationSetupContext(agentName: string): ProviderSetupContext { + const evaluationRunId = resolveEveEvaluationRunId(); + return { + agentName, + environment: resolveInstrumentationEnvironment(), + evaluation: evaluationRunId === undefined ? undefined : { runId: evaluationRunId }, + frameworkVersion: resolveInstalledPackageInfo().version, + }; +} + +function resolveInstrumentationEnvironment(): ProviderSetupContext["environment"] { + if (isEveDevEnvironment() || process.env.VERCEL_ENV === "development") return "development"; + return process.env.VERCEL_ENV === "preview" ? "preview" : "production"; +} diff --git a/packages/eve/src/harness/tool-loop.test.ts b/packages/eve/src/harness/tool-loop.test.ts index b7770f2246..ef9789943d 100644 --- a/packages/eve/src/harness/tool-loop.test.ts +++ b/packages/eve/src/harness/tool-loop.test.ts @@ -75,18 +75,25 @@ vi.mock("ai", () => ({ tool: vi.fn((t: unknown) => t), })); -const { existingOtelIntegration, mockCreateAiSdkHookBridge } = vi.hoisted(() => ({ - existingOtelIntegration: { onStart: vi.fn() }, +const { + mockCreateAiSdkHookBridge, + mockGetRegisteredTelemetryIntegrations, + registeredAuthorIntegration, + registeredOtelIntegration, +} = vi.hoisted(() => ({ mockCreateAiSdkHookBridge: vi.fn((..._args: unknown[]) => ({ onStart: vi.fn() })), + mockGetRegisteredTelemetryIntegrations: vi.fn((): unknown[] => []), + registeredAuthorIntegration: { onStart: vi.fn() }, + registeredOtelIntegration: { onStart: vi.fn() }, })); vi.mock("./ai-sdk-hook-bridge.js", () => ({ createAiSdkHookBridge: (...args: unknown[]) => mockCreateAiSdkHookBridge(...args), })); -vi.mock("./otel-integration.js", () => ({ - createOtelIntegration: vi.fn(() => existingOtelIntegration), +vi.mock("./ai-sdk-telemetry.js", () => ({ ensureOtelIntegration: vi.fn(), + getRegisteredTelemetryIntegrations: () => mockGetRegisteredTelemetryIntegrations(), })); const mockGetInstrumentationConfig = vi.fn().mockReturnValue(undefined); @@ -99,6 +106,10 @@ vi.mock("./otel-settings.js", () => ({ getOtelSettings: (...args: unknown[]) => mockGetOtelSettings(...args), })); +/** + * Registering an authored config writes both stores, so the tests toggle + * telemetry through one call rather than keeping two mocks in step by hand. + */ function declareTelemetry(config: Readonly> | undefined): void { mockGetInstrumentationConfig.mockReturnValue(config); mockGetOtelSettings.mockReturnValue( @@ -128,6 +139,7 @@ afterEach(() => { vi.clearAllMocks(); vi.unstubAllEnvs(); declareTelemetry(undefined); + mockGetRegisteredTelemetryIntegrations.mockReturnValue([]); }); function createTestSession(overrides?: Partial): HarnessSession { @@ -9588,7 +9600,7 @@ describe("createToolLoopHarness", () => { }); const attemptCompleted = vi.fn(); const hooks = createInstrumentationHooks([ - { events: { "step.attempt.completed": attemptCompleted } }, + { events: { "step.attempt.completed": attemptCompleted }, name: "attempt" }, ]); const runInContext: InstrumentationContextRunner = (_operation, execute) => execute(); const config = createTestConfig("conversation", undefined, { @@ -9662,7 +9674,7 @@ describe("createToolLoopHarness", () => { }); const started = vi.fn(); const hooks = createInstrumentationHooks([ - { events: { "action.started": started }, name: "test" }, + { events: { "action.started": started }, name: "action" }, ]); const { emit } = createEventCollector(); const runStep = createToolLoopHarness( @@ -9685,7 +9697,7 @@ describe("createToolLoopHarness", () => { ); }); - it("composes lifecycle hooks with existing authored OTel", async () => { + it("composes the bridge with every registered integration", async () => { setupMockAgent({ finishReason: "stop", response: { messages: [{ content: "Hello!", role: "assistant" }] }, @@ -9694,6 +9706,11 @@ describe("createToolLoopHarness", () => { toolResults: [], }); declareTelemetry({ recordInputs: true, recordOutputs: false }); + // An authored module can register its own integration alongside eve's. + mockGetRegisteredTelemetryIntegrations.mockReturnValue([ + registeredOtelIntegration, + registeredAuthorIntegration, + ]); const hooks = createInstrumentationHooks([]); const runStep = createToolLoopHarness( createTestConfig("conversation", undefined, { @@ -9715,7 +9732,7 @@ describe("createToolLoopHarness", () => { }; }; expect(agentCall.telemetry).toMatchObject({ - integrations: [bridge, existingOtelIntegration], + integrations: [bridge, registeredOtelIntegration, registeredAuthorIntegration], recordInputs: true, recordOutputs: false, }); diff --git a/packages/eve/src/harness/tool-loop.ts b/packages/eve/src/harness/tool-loop.ts index c8d4ae0321..101607099a 100644 --- a/packages/eve/src/harness/tool-loop.ts +++ b/packages/eve/src/harness/tool-loop.ts @@ -169,7 +169,10 @@ import { hasEmptyDeliverySentinel, } from "#shared/empty-delivery.js"; import { extractWorkflowStreamWriteErrorDetails } from "#harness/workflow-stream-error.js"; -import { createOtelIntegration, ensureOtelIntegration } from "#harness/otel-integration.js"; +import { + ensureOtelIntegration, + getRegisteredTelemetryIntegrations, +} from "#harness/ai-sdk-telemetry.js"; import { getAdvertisedTools } from "#harness/advertised-tools.js"; import { applyLastToolCacheBreakpoint, @@ -223,7 +226,7 @@ import type { /** * Builds the `telemetry` value for the AI SDK from authored settings. * - * Custom context (authored instrumentation events plus + * Custom context (authored `InstrumentationDefinition.events` plus * eve-specific identifiers such as `eve.session.id`) is flowed through * {@link buildTelemetryRuntimeContext} because AI SDK v7 surfaces * per-call attributes via `runtimeContext`, not a dedicated metadata field on @@ -288,12 +291,12 @@ function enrichTelemetry( return { functionId: settings?.functionId ?? agentName, includeRuntimeContext, + // Passing integrations replaces the registered ones for this call, so the + // bridge has to be composed with them rather than handed over on its own. integrations: bridgeIntegration === undefined ? undefined - : getInstrumentationConfig() === undefined - ? [bridgeIntegration] - : [bridgeIntegration, createOtelIntegration()], + : [bridgeIntegration, ...getRegisteredTelemetryIntegrations()], isEnabled: true, recordInputs: settings?.recordInputs ?? true, recordOutputs: settings?.recordOutputs ?? true, @@ -522,6 +525,9 @@ function buildHarnessToolsWithDynamicSubagents( export function createToolLoopHarness(config: ToolLoopHarnessConfig): StepFn { const baseEmit = config.handleEvent; const otelSettings = getOtelSettings(); + // The custom-context enrichment below still reads the authored config object + // directly. Its replacement on the provider surface is unresolved, so a + // provider directory contributes no custom context yet. const authoredConfig = getInstrumentationConfig(); if (otelSettings !== undefined) { ensureOtelIntegration(); diff --git a/packages/eve/src/internal/application/compiled-artifacts.ts b/packages/eve/src/internal/application/compiled-artifacts.ts index 561d60f45a..ad1487eece 100644 --- a/packages/eve/src/internal/application/compiled-artifacts.ts +++ b/packages/eve/src/internal/application/compiled-artifacts.ts @@ -1,4 +1,3 @@ -import { existsSync } from "node:fs"; import { copyFile, mkdir, writeFile } from "node:fs/promises"; import { join } from "node:path"; @@ -12,12 +11,23 @@ import { } from "#internal/application/package.js"; import { buildPackageUserAgent } from "#internal/user-agent.js"; import type { AgentWorkflowWorldDefinition } from "#shared/agent-definition.js"; -import { readMaterializedAuthoredModuleIndex } from "#internal/materialized-authored-modules.js"; +import { + readMaterializedAuthoredModuleIndex, + type MaterializedInstrumentation, +} from "#internal/materialized-authored-modules.js"; +import { + resolveInstrumentationLayout, + type InstrumentationLayout, +} from "#internal/instrumentation-layout.js"; import { usesParentDevelopmentWorkflowWorld } from "#internal/workflow/development-world-protocol.js"; import { resolveWorkflowWorldImport } from "#internal/workflow/world-target.js"; export type BuiltInWorkflowWorldTarget = "local" | "vercel"; +export type GeneratedInstrumentationLayout = + | { readonly kind: "config" } + | { readonly kind: "providers"; readonly slots: readonly string[] }; + /** * Paths to the generated compiled-artifacts files shared by Nitro and the * vendored workflow bundles for one application. @@ -31,15 +41,18 @@ export interface GeneratedCompiledArtifactsFiles { /** Nitro plugin that installs the selected vendored Workflow world. */ workflowWorldPluginPath: string; /** - * Optional Nitro plugin that imports the authored instrumentation module + * Optional Nitro plugin that imports the authored instrumentation modules * from the application when present. */ instrumentationPluginPath?: string; + /** Layout identity used by dev plugin selection and structural fingerprinting. */ + instrumentationLayout?: GeneratedInstrumentationLayout; /** - * Absolute path to the authored instrumentation module when present. - * Nitro uses this to preserve the module's side effects during bundling. + * Absolute paths to the authored instrumentation modules when present — one + * for the single-config layout, one per provider otherwise. Nitro uses these + * to preserve each module's side effects during bundling. */ - instrumentationSourcePath?: string; + instrumentationSourcePaths?: readonly string[]; } /** @@ -57,7 +70,11 @@ export async function writeCompiledArtifactsFiles(input: { const bootstrapPath = join(input.outDir, "compiled-artifacts-bootstrap.mjs"); const instrumentationPluginPath = join(input.outDir, "compiled-artifacts-instrumentation.mjs"); const workflowWorldPluginPath = join(input.outDir, "compiled-artifacts-workflow-world.mjs"); - const instrumentationPath = resolveInstrumentationModule(input.compileResult.manifest.agentRoot); + const layout = resolveInstrumentationLayout({ + agentRoot: input.compileResult.manifest.agentRoot, + providersEnabled: + input.compileResult.manifest.config.experimental?.instrumentationProviders ?? false, + }); await mkdir(input.outDir, { recursive: true }); await writeFile( @@ -78,30 +95,31 @@ export async function writeCompiledArtifactsFiles(input: { }), ); - if (instrumentationPath !== undefined) { + const generatedArtifacts: GeneratedCompiledArtifactsFiles = { + bootstrapPath, + workflowWorldPluginPath, + }; + + if (layout !== undefined) { await writeFile( instrumentationPluginPath, createInstrumentationPluginSource({ agentName: input.compileResult.manifest.config.name, - instrumentationPath, - registerConfigPath: resolvePackageSourceFilePath("src/harness/instrumentation-config.ts"), + layout, }), ); - } - - const generatedArtifacts: GeneratedCompiledArtifactsFiles = { - bootstrapPath, - workflowWorldPluginPath, - }; - - if (instrumentationPath !== undefined) { generatedArtifacts.instrumentationPluginPath = instrumentationPluginPath; - generatedArtifacts.instrumentationSourcePath = instrumentationPath; + generatedArtifacts.instrumentationLayout = generatedInstrumentationLayout(layout); + generatedArtifacts.instrumentationSourcePaths = instrumentationSourcePathsOf(layout); } return generatedArtifacts; } +function instrumentationSourcePathsOf(layout: InstrumentationLayout): readonly string[] { + return layout.kind === "config" ? [layout.modulePath] : Object.values(layout.modulePathsBySlot); +} + // The dev host's Nitro inputs outlive any single generation, so nothing // written here may point into authored source or a prunable snapshot: the // bootstrap references no authored module, and the instrumentation bundle is @@ -113,10 +131,6 @@ export async function writeDevelopmentCompiledArtifactsFiles(input: { }): Promise { const bootstrapPath = join(input.outDir, "compiled-artifacts-bootstrap.mjs"); const instrumentationPluginPath = join(input.outDir, "compiled-artifacts-instrumentation.mjs"); - const instrumentationSourcePath = join( - input.outDir, - "compiled-artifacts-instrumentation-source.mjs", - ); const workflowWorldPluginPath = join(input.outDir, "compiled-artifacts-workflow-world.mjs"); const materializedIndex = await readMaterializedAuthoredModuleIndex(input.runtimeAppRoot); @@ -143,25 +157,66 @@ export async function writeDevelopmentCompiledArtifactsFiles(input: { }; if (materializedIndex.instrumentation !== undefined) { - await copyFile( - join(input.runtimeAppRoot, ".eve", "compile", materializedIndex.instrumentation), - instrumentationSourcePath, - ); + const layout = await copyMaterializedInstrumentation({ + instrumentation: materializedIndex.instrumentation, + outDir: input.outDir, + runtimeAppRoot: input.runtimeAppRoot, + }); + await writeFile( instrumentationPluginPath, createInstrumentationPluginSource({ agentName: input.compileResult.manifest.config.name, - instrumentationPath: instrumentationSourcePath, - registerConfigPath: resolvePackageSourceFilePath("src/harness/instrumentation-config.ts"), + layout, }), ); generatedArtifacts.instrumentationPluginPath = instrumentationPluginPath; - generatedArtifacts.instrumentationSourcePath = instrumentationSourcePath; + generatedArtifacts.instrumentationLayout = generatedInstrumentationLayout(layout); + generatedArtifacts.instrumentationSourcePaths = instrumentationSourcePathsOf(layout); } return generatedArtifacts; } +function generatedInstrumentationLayout( + layout: InstrumentationLayout, +): GeneratedInstrumentationLayout { + return layout.kind === "config" + ? { kind: "config" } + : { kind: "providers", slots: Object.keys(layout.modulePathsBySlot) }; +} + +/** + * Copies each materialized instrumentation bundle out of the prunable + * generation and into the stable dev-host directory, returning the layout in + * terms of the copies. + */ +async function copyMaterializedInstrumentation(input: { + readonly instrumentation: MaterializedInstrumentation; + readonly outDir: string; + readonly runtimeAppRoot: string; +}): Promise { + const compileRoot = join(input.runtimeAppRoot, ".eve", "compile"); + const copyOne = async (sourceId: string, modulePath: string): Promise => { + const destination = join(input.outDir, `compiled-artifacts-instrumentation-${sourceId}.mjs`); + await copyFile(join(compileRoot, modulePath), destination); + return destination; + }; + + if (input.instrumentation.kind === "config") { + return { + kind: "config", + modulePath: await copyOne("source", input.instrumentation.modulePath), + }; + } + + const modulePathsBySlot: Record = {}; + for (const [slot, modulePath] of Object.entries(input.instrumentation.modulePathsBySlot)) { + modulePathsBySlot[slot] = await copyOne(slot, modulePath); + } + return { kind: "providers", modulePathsBySlot }; +} + function createDevelopmentCompiledArtifactsBootstrapSource(agentName: string): string { return [ "// Generated by eve. Do not edit by hand.", @@ -174,23 +229,6 @@ function createDevelopmentCompiledArtifactsBootstrapSource(agentName: string): s ].join("\n"); } -const INSTRUMENTATION_EXTENSIONS = [".ts", ".mts", ".js", ".mjs"]; - -/** - * Resolves the optional `agent/instrumentation` module from the agent root - * directory. Returns the absolute path if found, `undefined` otherwise. - */ -function resolveInstrumentationModule(agentRoot: string): string | undefined { - for (const ext of INSTRUMENTATION_EXTENSIONS) { - const candidate = join(agentRoot, `instrumentation${ext}`); - if (existsSync(candidate)) { - return candidate; - } - } - - return undefined; -} - function stripCompiledModuleMapExports(source: string): string { return source .replace(/^export const moduleMap = /m, "const moduleMap = ") @@ -380,23 +418,73 @@ export function createDevelopmentWorkflowWorldPluginSource(input: { ].join("\n"); } +/** + * Generates the Nitro plugin that registers the authored instrumentation. + * + * The single-config layout registers one default export; the provider layout + * registers one per file, in slot order, awaiting each `setup` so a provider + * cannot miss an event published while it is still starting up, then builds the + * one OpenTelemetry pipeline their declarations add up to. + */ function createInstrumentationPluginSource(input: { agentName: string; - instrumentationPath: string; - registerConfigPath: string; + layout: InstrumentationLayout; }): string { + const agentName = JSON.stringify(input.agentName); + + if (input.layout.kind === "config") { + const registerConfigPath = resolvePackageSourceFilePath( + "src/harness/instrumentation-config.ts", + ); + return [ + "// Generated by eve. Do not edit by hand.", + `import * as instrumentationModule from ${stringifyEsmImportSpecifier(input.layout.modulePath)};`, + `import { registerInstrumentationConfig } from ${stringifyEsmImportSpecifier(registerConfigPath)};`, + "", + "if (instrumentationModule.default != null) {", + ` await registerInstrumentationConfig(instrumentationModule.default, { agentName: ${agentName} });`, + "}", + "", + "// Default export satisfies the Nitro plugin contract so this file", + "// can be used directly as a Nitro plugin without a separate wrapper.", + "export default function installInstrumentationPlugin() {}", + "", + ].join("\n"); + } + + const registerProviderPath = resolvePackageSourceFilePath( + "src/harness/instrumentation-providers.ts", + ); + const slots = Object.entries(input.layout.modulePathsBySlot); + return [ "// Generated by eve. Do not edit by hand.", - `import * as instrumentationModule from ${stringifyEsmImportSpecifier(input.instrumentationPath)};`, - `import { registerInstrumentationConfig } from ${stringifyEsmImportSpecifier(input.registerConfigPath)};`, + ...slots.map( + ([slot, modulePath], index) => + `import * as provider${index} from ${stringifyEsmImportSpecifier(modulePath)}; // ${slot}`, + ), + `import { finalizeInstrumentationProviders, registerInstrumentationProvider, seedInstrumentationProviders, shutdownInstrumentationProviders } from ${stringifyEsmImportSpecifier(registerProviderPath)};`, "", - "if (instrumentationModule.default != null) {", - ` await registerInstrumentationConfig(instrumentationModule.default, { agentName: ${JSON.stringify(input.agentName)} });`, - "}", + "seedInstrumentationProviders();", + "", + ...slots.flatMap(([slot], index) => [ + "await registerInstrumentationProvider({", + ` agentName: ${agentName},`, + ` slot: ${JSON.stringify(slot)},`, + ` value: provider${index}.default,`, + "});", + ]), + "", + `finalizeInstrumentationProviders({ serviceName: ${agentName} });`, "", "// Default export satisfies the Nitro plugin contract so this file", "// can be used directly as a Nitro plugin without a separate wrapper.", - "export default function installInstrumentationPlugin() {}", + "export default function installInstrumentationPlugin(nitroApp) {", + " // The last point a buffered exporter can still reach the network.", + " nitroApp?.hooks?.hook('close', async () => {", + " await shutdownInstrumentationProviders();", + " });", + "}", "", ].join("\n"); } diff --git a/packages/eve/src/internal/application/dev-environment.ts b/packages/eve/src/internal/application/dev-environment.ts index acbd09cfce..daa2ff244f 100644 --- a/packages/eve/src/internal/application/dev-environment.ts +++ b/packages/eve/src/internal/application/dev-environment.ts @@ -5,3 +5,25 @@ export const EVE_DEV_ENV_FLAG = "EVE_DEV"; export function isEveDevEnvironment(): boolean { return process.env[EVE_DEV_ENV_FLAG] === "1"; } + +/** Environment flag set for a server `eve eval` started to run against. */ +export const EVE_EVALUATION_ENV_FLAG = "EVE_EVALUATION"; + +/** Stable identifier for the local eval run this server was started to serve. */ +export const EVE_EVALUATION_RUN_ID_ENV = "EVE_EVALUATION_RUN_ID"; + +/** + * Reports whether this process exists to serve an eval run. + * + * False for a server that `eve eval --url` merely points at: that process was + * started to serve ordinary traffic and cannot know an eval is among it. + */ +export function isEveEvaluationEnvironment(): boolean { + return process.env[EVE_EVALUATION_ENV_FLAG] === "1"; +} + +export function resolveEveEvaluationRunId(): string | undefined { + if (!isEveEvaluationEnvironment()) return undefined; + const runId = process.env[EVE_EVALUATION_RUN_ID_ENV]; + return runId === undefined || runId.length === 0 ? undefined : runId; +} diff --git a/packages/eve/src/internal/authored-definition/core.ts b/packages/eve/src/internal/authored-definition/core.ts index 348ed5a2a5..a3b4e5e3d1 100644 --- a/packages/eve/src/internal/authored-definition/core.ts +++ b/packages/eve/src/internal/authored-definition/core.ts @@ -257,9 +257,20 @@ function normalizeAgentExperimentalDefinition( message: string, ): NonNullable { const record = expectObjectRecord(value, message); - expectOnlyKnownKeys(record, ["subagentPersistentSessions", "workflow"], message); + expectOnlyKnownKeys( + record, + ["instrumentationProviders", "subagentPersistentSessions", "workflow"], + message, + ); const normalizedDefinition: Mutable> = {}; + if (record.instrumentationProviders !== undefined) { + if (typeof record.instrumentationProviders !== "boolean") { + throw new Error(`${message} "experimental.instrumentationProviders" must be a boolean.`); + } + normalizedDefinition.instrumentationProviders = record.instrumentationProviders; + } + if (record.subagentPersistentSessions !== undefined) { if (typeof record.subagentPersistentSessions !== "boolean") { throw new Error(`${message} "experimental.subagentPersistentSessions" must be a boolean.`); diff --git a/packages/eve/src/internal/instrumentation-layout.integration.test.ts b/packages/eve/src/internal/instrumentation-layout.integration.test.ts new file mode 100644 index 0000000000..0833ec49eb --- /dev/null +++ b/packages/eve/src/internal/instrumentation-layout.integration.test.ts @@ -0,0 +1,128 @@ +import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { beforeEach, describe, expect, it } from "vitest"; + +import { resolveInstrumentationLayout } from "#internal/instrumentation-layout.js"; + +let agentRoot: string; + +beforeEach(() => { + agentRoot = mkdtempSync(join(tmpdir(), "eve-instrumentation-layout-")); +}); + +function writeConfig(extension = ".ts"): string { + const path = join(agentRoot, `instrumentation${extension}`); + writeFileSync(path, "export default {};\n"); + return path; +} + +function writeProvider(fileName: string): string { + const directory = join(agentRoot, "instrumentation"); + mkdirSync(directory, { recursive: true }); + const path = join(directory, fileName); + writeFileSync(path, "export default {};\n"); + return path; +} + +describe("resolveInstrumentationLayout with providers off", () => { + it("returns nothing when the agent authored no instrumentation", () => { + expect(resolveInstrumentationLayout({ agentRoot, providersEnabled: false })).toBeUndefined(); + }); + + it("resolves the single config module", () => { + const modulePath = writeConfig(); + + expect(resolveInstrumentationLayout({ agentRoot, providersEnabled: false })).toEqual({ + kind: "config", + modulePath, + }); + }); + + it.each([[".mts"], [".js"], [".mjs"]])("resolves a %s config module", (extension) => { + const modulePath = writeConfig(extension); + + expect(resolveInstrumentationLayout({ agentRoot, providersEnabled: false })).toEqual({ + kind: "config", + modulePath, + }); + }); + + it("rejects a providers directory, naming the flag", () => { + writeProvider("otel.ts"); + + expect(() => resolveInstrumentationLayout({ agentRoot, providersEnabled: false })).toThrow( + /experimental\.instrumentationProviders/, + ); + }); +}); + +describe("resolveInstrumentationLayout with providers on", () => { + it("returns an empty provider layout for eve's built-in destinations", () => { + expect(resolveInstrumentationLayout({ agentRoot, providersEnabled: true })).toEqual({ + kind: "providers", + modulePathsBySlot: {}, + }); + }); + + it("keys each file by the slot its name derives", () => { + const otel = writeProvider("otel.ts"); + const local = writeProvider("local.mts"); + + expect(resolveInstrumentationLayout({ agentRoot, providersEnabled: true })).toEqual({ + kind: "providers", + modulePathsBySlot: { local, otel }, + }); + }); + + it("orders slots independently of directory enumeration", () => { + writeProvider("otel.ts"); + writeProvider("agent-runs.ts"); + writeProvider("local.ts"); + + const layout = resolveInstrumentationLayout({ agentRoot, providersEnabled: true }); + + expect(Object.keys(layout?.kind === "providers" ? layout.modulePathsBySlot : {})).toEqual([ + "agent-runs", + "local", + "otel", + ]); + }); + + it("ignores files that are not instrumentation modules", () => { + writeProvider("otel.ts"); + writeProvider("README.md"); + + const layout = resolveInstrumentationLayout({ agentRoot, providersEnabled: true }); + + expect(Object.keys(layout?.kind === "providers" ? layout.modulePathsBySlot : {})).toEqual([ + "otel", + ]); + }); + + it("rejects two files claiming one slot", () => { + writeProvider("otel.ts"); + writeProvider("otel.js"); + + expect(() => resolveInstrumentationLayout({ agentRoot, providersEnabled: true })).toThrow( + /Two files declare the "otel" instrumentation provider/, + ); + }); + + it("rejects a single config module, naming the flag", () => { + writeConfig(); + + expect(() => resolveInstrumentationLayout({ agentRoot, providersEnabled: true })).toThrow( + /experimental\.instrumentationProviders/, + ); + }); + + it("prefers the config error when both layouts are present", () => { + writeConfig(); + writeProvider("otel.ts"); + + expect(() => resolveInstrumentationLayout({ agentRoot, providersEnabled: true })).toThrow( + /Move it into the "instrumentation\/" directory/, + ); + }); +}); diff --git a/packages/eve/src/internal/instrumentation-layout.ts b/packages/eve/src/internal/instrumentation-layout.ts new file mode 100644 index 0000000000..44546d0ef1 --- /dev/null +++ b/packages/eve/src/internal/instrumentation-layout.ts @@ -0,0 +1,115 @@ +import { existsSync, readdirSync, statSync } from "node:fs"; +import { join } from "node:path"; + +const INSTRUMENTATION_EXTENSIONS = [".ts", ".mts", ".js", ".mjs"] as const; + +const INSTRUMENTATION_DIRECTORY = "instrumentation"; + +const PROVIDERS_FLAG = "experimental.instrumentationProviders"; + +/** + * How instrumentation is authored for one agent. + * + * `config` is the single `agent/instrumentation.ts` default export. `providers` + * is a directory of them, one provider per file, keyed by the slot name the + * file derives (`instrumentation/otel.ts` → `otel`). Which one an agent may use + * is decided by `experimental.instrumentationProviders`, never by what happens + * to be on disk. + */ +export type InstrumentationLayout = + | { readonly kind: "config"; readonly modulePath: string } + | { readonly kind: "providers"; readonly modulePathsBySlot: Readonly> }; + +/** + * Resolves the instrumentation layout for one agent root. + * + * With providers on, an empty provider layout still installs eve's built-in + * destinations. Throws when the layout on disk is not the one the flag selects: + * the wrong layout would otherwise be skipped silently, and telemetry that + * quietly does nothing is the failure this whole surface exists to prevent. + */ +export function resolveInstrumentationLayout(input: { + readonly agentRoot: string; + readonly providersEnabled: boolean; +}): InstrumentationLayout | undefined { + const configPath = resolveInstrumentationConfigModule(input.agentRoot); + const directoryPath = join(input.agentRoot, INSTRUMENTATION_DIRECTORY); + const hasDirectory = existsSync(directoryPath) && statSync(directoryPath).isDirectory(); + + if (!input.providersEnabled) { + if (hasDirectory) { + throw new Error( + `Found an "${INSTRUMENTATION_DIRECTORY}/" directory at "${input.agentRoot}", but instrumentation providers are off. Set \`${PROVIDERS_FLAG}: true\` in \`defineAgent\`, or move these files back into a single "${INSTRUMENTATION_DIRECTORY}.ts".`, + ); + } + + return configPath === undefined ? undefined : { kind: "config", modulePath: configPath }; + } + + if (configPath !== undefined) { + throw new Error( + `Found "${configPath}", but \`${PROVIDERS_FLAG}\` is on. Move it into the "${INSTRUMENTATION_DIRECTORY}/" directory as one file per provider.`, + ); + } + + if (!hasDirectory) { + return { kind: "providers", modulePathsBySlot: {} }; + } + + return { + kind: "providers", + modulePathsBySlot: collectInstrumentationProviderModules(directoryPath), + }; +} + +/** + * Maps each `instrumentation/.` file to its absolute path. + * + * Slots are sorted so the registration order a provider sees does not depend on + * how the filesystem happens to enumerate the directory. + */ +function collectInstrumentationProviderModules( + directoryPath: string, +): Readonly> { + const modulePathsBySlot = new Map(); + + for (const entry of readdirSync(directoryPath, { withFileTypes: true })) { + if (!entry.isFile()) continue; + + const extension = INSTRUMENTATION_EXTENSIONS.find((candidate) => + entry.name.endsWith(candidate), + ); + if (extension === undefined) continue; + + const slot = entry.name.slice(0, -extension.length); + if (slot === "") continue; + + const existing = modulePathsBySlot.get(slot); + if (existing !== undefined) { + throw new Error( + `Two files declare the "${slot}" instrumentation provider in "${directoryPath}". Keep one of them.`, + ); + } + + modulePathsBySlot.set(slot, join(directoryPath, entry.name)); + } + + return Object.fromEntries( + [...modulePathsBySlot].sort(([left], [right]) => left.localeCompare(right)), + ); +} + +/** + * Resolves the single `agent/instrumentation` module, ignoring any directory of + * the same name. + */ +function resolveInstrumentationConfigModule(agentRoot: string): string | undefined { + for (const extension of INSTRUMENTATION_EXTENSIONS) { + const candidate = join(agentRoot, `${INSTRUMENTATION_DIRECTORY}${extension}`); + if (existsSync(candidate)) { + return candidate; + } + } + + return undefined; +} diff --git a/packages/eve/src/internal/materialized-authored-modules.ts b/packages/eve/src/internal/materialized-authored-modules.ts index 7c13760806..a848f1883e 100644 --- a/packages/eve/src/internal/materialized-authored-modules.ts +++ b/packages/eve/src/internal/materialized-authored-modules.ts @@ -10,16 +10,24 @@ import { bundleAuthoredModuleMapForGeneration, } from "#internal/authored-module-loader.js"; import { serializeCompiledManifestForFingerprint } from "#internal/compiled-manifest-fingerprint.js"; +import { resolveInstrumentationLayout } from "#internal/instrumentation-layout.js"; const MATERIALIZED_MODULES_DIRECTORY = "authored-modules"; const MATERIALIZED_MODULES_INDEX = "authored-modules.json"; -const INSTRUMENTATION_EXTENSIONS = [".ts", ".mts", ".js", ".mjs"] as const; + +/** + * The materialized instrumentation modules, mirroring the layout they were + * authored in. Paths are relative to `.eve/compile`. + */ +export type MaterializedInstrumentation = + | { readonly kind: "config"; readonly modulePath: string } + | { readonly kind: "providers"; readonly modulePathsBySlot: Readonly> }; export interface MaterializedAuthoredModuleIndex { readonly fingerprint: string; - readonly instrumentation?: string; + readonly instrumentation?: MaterializedInstrumentation; readonly moduleMap: string; - readonly version: 2; + readonly version: 3; } export async function materializeAuthoredModules(input: { @@ -54,22 +62,40 @@ export async function materializeAuthoredModules(input: { await writeFile(join(modulesRoot, moduleMapFileName), moduleMapCode); fingerprint.update("module-map\0").update(moduleMapCode).update("\0"); - const instrumentation = resolveInstrumentationModule(manifest.agentRoot); - let instrumentationPath: string | undefined; - - if (instrumentation !== undefined) { - const code = await bundleAuthoredModuleForGeneration(instrumentation, { - externalDependencies: manifest.config.build?.externalDependencies ?? [], - }); + const layout = resolveInstrumentationLayout({ + agentRoot: manifest.agentRoot, + providersEnabled: manifest.config.experimental?.instrumentationProviders ?? false, + }); + const externalDependencies = manifest.config.build?.externalDependencies ?? []; + const materializeInstrumentationModule = async ( + sourceId: string, + sourcePath: string, + ): Promise => { + const code = await bundleAuthoredModuleForGeneration(sourcePath, { externalDependencies }); const fileName = createMaterializedModuleFileName( ROOT_COMPILED_AGENT_NODE_ID, - "instrumentation", + `instrumentation:${sourceId}`, code, ); await writeFile(join(modulesRoot, fileName), code); - instrumentationPath = join(MATERIALIZED_MODULES_DIRECTORY, fileName); - fingerprint.update("instrumentation\0").update(code).update("\0"); + fingerprint.update(`instrumentation:${sourceId}\0`).update(code).update("\0"); + return join(MATERIALIZED_MODULES_DIRECTORY, fileName); + }; + + let instrumentation: MaterializedInstrumentation | undefined; + + if (layout?.kind === "config") { + instrumentation = { + kind: "config", + modulePath: await materializeInstrumentationModule("config", layout.modulePath), + }; + } else if (layout?.kind === "providers") { + const modulePathsBySlot: Record = {}; + for (const [slot, sourcePath] of Object.entries(layout.modulePathsBySlot)) { + modulePathsBySlot[slot] = await materializeInstrumentationModule(slot, sourcePath); + } + instrumentation = { kind: "providers", modulePathsBySlot }; } await hashDirectoryIfPresent({ @@ -79,16 +105,16 @@ export async function materializeAuthoredModules(input: { }); const index: { fingerprint: string; - instrumentation?: string; + instrumentation?: MaterializedInstrumentation; moduleMap: string; - version: 2; + version: 3; } = { fingerprint: fingerprint.digest("hex"), moduleMap: moduleMapPath, - version: 2, + version: 3, }; - if (instrumentationPath !== undefined) { - index.instrumentation = instrumentationPath; + if (instrumentation !== undefined) { + index.instrumentation = instrumentation; } await writeFile(join(compileRoot, MATERIALIZED_MODULES_INDEX), `${JSON.stringify(index)}\n`); return index; @@ -106,12 +132,12 @@ export async function readMaterializedAuthoredModuleIndex( await readFile(indexPath, "utf8"), ) as Partial; if ( - parsed.version !== 2 || + parsed.version !== 3 || typeof parsed.fingerprint !== "string" || parsed.fingerprint.length === 0 || typeof parsed.moduleMap !== "string" || parsed.moduleMap.length === 0 || - (parsed.instrumentation !== undefined && typeof parsed.instrumentation !== "string") + !isMaterializedInstrumentation(parsed.instrumentation) ) { throw new Error(`Invalid materialized authored module index at "${indexPath}".`); } @@ -119,6 +145,26 @@ export async function readMaterializedAuthoredModuleIndex( return parsed as MaterializedAuthoredModuleIndex; } +function isMaterializedInstrumentation(value: unknown): boolean { + if (value === undefined) return true; + if (typeof value !== "object" || value === null) return false; + + const candidate = value as Partial; + if (candidate.kind === "config") { + return typeof (candidate as { modulePath?: unknown }).modulePath === "string"; + } + if (candidate.kind === "providers") { + const paths = (candidate as { modulePathsBySlot?: unknown }).modulePathsBySlot; + return ( + typeof paths === "object" && + paths !== null && + Object.values(paths).every((path) => typeof path === "string") + ); + } + + return false; +} + async function readCompiledManifest(path: string): Promise { const manifest = JSON.parse(await readFile(path, "utf8")) as CompiledAgentManifest; @@ -129,17 +175,6 @@ async function readCompiledManifest(path: string): Promise { @@ -454,9 +455,12 @@ describe("development generation artifacts", () => { join(first.runtimeAppRoot, ".eve", "compile", "authored-modules.json"), "utf8", ), - ) as { readonly instrumentation?: string }; + ) as { readonly instrumentation?: MaterializedInstrumentation }; + if (firstIndex.instrumentation?.kind !== "config") { + throw new Error("expected materialized config instrumentation"); + } const materializedInstrumentation = await readFile( - join(first.runtimeAppRoot, ".eve", "compile", firstIndex.instrumentation!), + join(first.runtimeAppRoot, ".eve", "compile", firstIndex.instrumentation.modulePath), "utf8", ); diff --git a/packages/eve/src/internal/nitro/host/create-application-nitro.scenario.test.ts b/packages/eve/src/internal/nitro/host/create-application-nitro.scenario.test.ts index 3af429a9b4..5a12a75adb 100644 --- a/packages/eve/src/internal/nitro/host/create-application-nitro.scenario.test.ts +++ b/packages/eve/src/internal/nitro/host/create-application-nitro.scenario.test.ts @@ -216,6 +216,7 @@ describe("application Nitro creation", () => { const { createDevelopmentApplicationNitro } = await import("#internal/nitro/host/create-application-nitro.js"); const preparedHost = createPreparedHost(); + preparedHost.compiledArtifacts.instrumentationLayout = { kind: "config" }; preparedHost.compiledArtifacts.instrumentationPluginPath = "/app/instrumentation.mjs"; await createDevelopmentApplicationNitro(preparedHost); @@ -227,6 +228,29 @@ describe("application Nitro creation", () => { ); }); + it("lets the provider pipeline own default local tracing", async () => { + const { createDevelopmentApplicationNitro } = + await import("#internal/nitro/host/create-application-nitro.js"); + + for (const slots of ["rows", "local"] as const) { + const nitroStub = createNitroStub(); + createNitroMock.mockResolvedValueOnce(nitroStub.nitro); + const preparedHost = createPreparedHost(); + preparedHost.compiledArtifacts.instrumentationLayout = { + kind: "providers", + slots: [slots], + }; + preparedHost.compiledArtifacts.instrumentationPluginPath = "/app/instrumentation.mjs"; + + await createDevelopmentApplicationNitro(preparedHost); + + const plugins = createNitroMock.mock.calls.at(-1)?.[0].plugins as string[]; + expect(plugins).not.toEqual( + expect.arrayContaining([expect.stringContaining("local-tracing-runtime-plugin.ts")]), + ); + } + }); + it("preserves workflow bundle side effects and skips workflow transform for cached bundles", async () => { const nitroStub = createNitroStub(); createNitroMock.mockResolvedValueOnce(nitroStub.nitro); diff --git a/packages/eve/src/internal/nitro/host/create-application-nitro.ts b/packages/eve/src/internal/nitro/host/create-application-nitro.ts index c3f2ccce5f..65483dee91 100644 --- a/packages/eve/src/internal/nitro/host/create-application-nitro.ts +++ b/packages/eve/src/internal/nitro/host/create-application-nitro.ts @@ -519,9 +519,11 @@ function addDynamicCapabilityTransformPlugin(nitro: Nitro): void { */ function addInstrumentationModuleSideEffectsPlugin( nitro: Nitro, - instrumentationModulePath: string, + instrumentationModulePaths: readonly string[], ): void { - const normalizedInstrumentationModulePath = normalizePath(instrumentationModulePath); + const normalizedInstrumentationModulePaths = new Set( + instrumentationModulePaths.map(normalizePath), + ); nitro.hooks.hook("rollup:before", (_nitro, config) => { if (!Array.isArray(config.plugins)) { @@ -531,7 +533,7 @@ function addInstrumentationModuleSideEffectsPlugin( config.plugins.unshift({ name: "eve:instrumentation-module-side-effects", resolveId(source: string) { - if (normalizePath(source) !== normalizedInstrumentationModulePath) { + if (!normalizedInstrumentationModulePaths.has(normalizePath(source))) { return null; } @@ -681,10 +683,10 @@ function configureSharedApplicationNitro( addDynamicCapabilityTransformPlugin(nitro); - if (preparedHost.compiledArtifacts.instrumentationSourcePath !== undefined) { + if (preparedHost.compiledArtifacts.instrumentationSourcePaths !== undefined) { addInstrumentationModuleSideEffectsPlugin( nitro, - preparedHost.compiledArtifacts.instrumentationSourcePath, + preparedHost.compiledArtifacts.instrumentationSourcePaths, ); } } diff --git a/packages/eve/src/internal/nitro/host/dev-host-fingerprint.integration.test.ts b/packages/eve/src/internal/nitro/host/dev-host-fingerprint.integration.test.ts index 62e5d36211..5f0d49be20 100644 --- a/packages/eve/src/internal/nitro/host/dev-host-fingerprint.integration.test.ts +++ b/packages/eve/src/internal/nitro/host/dev-host-fingerprint.integration.test.ts @@ -20,6 +20,7 @@ afterEach(async () => { interface HostVariant { readonly channels?: CompiledAgentManifest["channels"]; + readonly instrumentationSlot?: string; readonly instrumentationSource?: string; readonly schedules?: CompiledAgentManifest["schedules"]; readonly workflowWorld?: "local" | "vercel"; @@ -55,7 +56,15 @@ async function createHost(variant: HostVariant = {}): Promise { expect(changed).not.toBe(base); }); + it("treats provider slot names as structural", async () => { + const source = "export default {}\n"; + const first = await computeDevelopmentHostFingerprint( + await createHost({ instrumentationSlot: "a", instrumentationSource: source }), + ); + const renamed = await computeDevelopmentHostFingerprint( + await createHost({ instrumentationSlot: "b", instrumentationSource: source }), + ); + + expect(renamed).not.toBe(first); + }); + it("treats channel route topology as structural", async () => { const base = await computeDevelopmentHostFingerprint(await createHost()); const withRoute = await computeDevelopmentHostFingerprint( diff --git a/packages/eve/src/internal/nitro/host/dev-host-fingerprint.ts b/packages/eve/src/internal/nitro/host/dev-host-fingerprint.ts index 6ab2f04e0f..1e4ac273ba 100644 --- a/packages/eve/src/internal/nitro/host/dev-host-fingerprint.ts +++ b/packages/eve/src/internal/nitro/host/dev-host-fingerprint.ts @@ -43,12 +43,21 @@ export async function computeDevelopmentHostFingerprint( return createHash("sha256").update(JSON.stringify(payload)).digest("hex"); } -async function readInstrumentationSource( - host: PreparedDevelopmentApplicationHost, -): Promise { - const path = host.compiledArtifacts.instrumentationSourcePath; - if (path === undefined) { +async function readInstrumentationSource(host: PreparedDevelopmentApplicationHost): Promise<{ + readonly kind: "config" | "providers"; + readonly modules: readonly { readonly slot: string | null; readonly source: string }[]; +} | null> { + const paths = host.compiledArtifacts.instrumentationSourcePaths; + const layout = host.compiledArtifacts.instrumentationLayout; + if (paths === undefined || layout === undefined) { return null; } - return await readFile(path, "utf8"); + const sources = await Promise.all(paths.map(async (path) => await readFile(path, "utf8"))); + return { + kind: layout.kind, + modules: sources.map((source, index) => ({ + slot: layout.kind === "providers" ? (layout.slots[index] ?? null) : null, + source, + })), + }; } diff --git a/packages/eve/src/internal/nitro/host/prepare-application-host.scenario.test.ts b/packages/eve/src/internal/nitro/host/prepare-application-host.scenario.test.ts index 4f0612f9e9..e36d53b591 100644 --- a/packages/eve/src/internal/nitro/host/prepare-application-host.scenario.test.ts +++ b/packages/eve/src/internal/nitro/host/prepare-application-host.scenario.test.ts @@ -108,9 +108,9 @@ describe("application host preparation", () => { join(firstHostDirectory, "compiled-artifacts-workflow-world.mjs"), ); expect(firstHost.compiledArtifacts.bootstrapPath).not.toContain("/.eve/dev-runtime/snapshots/"); - expect(firstHost.compiledArtifacts.instrumentationSourcePath).toBe( + expect(firstHost.compiledArtifacts.instrumentationSourcePaths).toEqual([ join(firstHostDirectory, "compiled-artifacts-instrumentation-source.mjs"), - ); + ]); expect(await readFile(firstBootstrapPath, "utf8")).not.toContain( normalizeEsmImportSpecifier(agentModulePath), ); diff --git a/packages/eve/src/internal/nitro/routes/channel-request-instrumentation.ts b/packages/eve/src/internal/nitro/routes/channel-request-instrumentation.ts index fa1b0a3673..e0b5809585 100644 --- a/packages/eve/src/internal/nitro/routes/channel-request-instrumentation.ts +++ b/packages/eve/src/internal/nitro/routes/channel-request-instrumentation.ts @@ -58,9 +58,8 @@ export interface TraceChannelRequestInput { * waiting for `event.waitUntil()` work or streamed response bodies. * * Emitting these spans is opt-in: unless authored instrumentation enables it - * via `defineInstrumentation({ traceChannelRequests: true })`, the handler runs - * with no span (`undefined`) and no context extraction — a true bypass, not a - * non-recording span. + * via `traceChannelRequests: true`, the handler runs with no span (`undefined`) + * and no context extraction — a true bypass, not a non-recording span. * * This is observability-only: it never changes the response and performs no * synchronous span export in the request path, adding only minimal in-process diff --git a/packages/eve/src/public/definitions/exact.test.ts b/packages/eve/src/public/definitions/exact.test.ts index 066ae8ff27..f5ab5b8980 100644 --- a/packages/eve/src/public/definitions/exact.test.ts +++ b/packages/eve/src/public/definitions/exact.test.ts @@ -179,11 +179,18 @@ function typeOnlyFixtures(): void { // @ts-expect-error Instructions identity is path-derived. defineInstructions(instructionsWithName); + defineInstrumentation({ + isEnabled: true, + recordInputs: true, + }); + + // Unlike the helpers above, `defineInstrumentation` takes a generic union — a + // config and a provider overlap on `events` and `setup` — so it cannot use + // `ExactDefinition`. Excess keys reach `eve build` instead. const instrumentationWithEnabled = { isEnabled: true, recordInputs: true, }; - // @ts-expect-error Instrumentation has no separate enable toggle. defineInstrumentation(instrumentationWithEnabled); defineInstrumentation({ diff --git a/packages/eve/src/public/instrumentation/index.ts b/packages/eve/src/public/instrumentation/index.ts index a5d38f09d3..74555b0e8f 100644 --- a/packages/eve/src/public/instrumentation/index.ts +++ b/packages/eve/src/public/instrumentation/index.ts @@ -1,15 +1,21 @@ -import type { ExactDefinition } from "#public/definitions/exact.js"; - /** - * Instrumentation authoring helpers for `agent/instrumentation.ts`. + * Instrumentation authoring helpers for `agent/instrumentation.ts` and, with + * `experimental.instrumentationProviders` on, `agent/instrumentation/`. */ import type { ModelMessage, SystemModelMessage } from "ai"; import type { SessionAuthContext, SessionParent } from "#channel/types.js"; import type { InstrumentationChannel } from "#public/channels/index.js"; +import { + PROVIDER, + type ProviderDefinition, + type ProviderSetupContext, +} from "#public/instrumentation/provider.js"; import type { JsonObject } from "#shared/json.js"; +export * from "#public/instrumentation/provider.js"; + // Re-export channel metadata types so existing `eve/instrumentation` // imports continue to work. The canonical home is `eve/channels`. export { @@ -23,14 +29,13 @@ export { /** * Context passed to the {@link InstrumentationDefinition.setup} callback. + * + * The same context both layouts receive. Keeping one type is what gives + * {@link defineInstrumentation}'s union a contextual signature for `setup`; + * two divergent ones would leave every authored `setup(context)` parameter an + * implicit `any`. */ -export interface InstrumentationSetupContext { - /** - * The agent name declared by `defineAgent`. Use as the `serviceName` for - * `registerOTel` instead of a hard-coded string. - */ - readonly agentName: string; -} +export type InstrumentationSetupContext = ProviderSetupContext; /** * User-authored runtime context values attached to AI SDK telemetry spans. @@ -161,21 +166,39 @@ export interface InstrumentationDefinition { */ readonly traceChannelRequests?: boolean; /** - * Setup callback invoked at server startup with the resolved agent name. - * Use it to call `registerOTel` or other OTel provider setup; - * `context.agentName` comes from `defineAgent`. + * Setup callback invoked at server startup, before the first request. Use it + * to call `registerOTel` or other OTel provider setup; `context.agentName` + * comes from `defineAgent`. A returned promise is awaited. */ - readonly setup?: (context: InstrumentationSetupContext) => void; + readonly setup?: (context: InstrumentationSetupContext) => void | PromiseLike; } /** - * Export the result as the default export of `agent/instrumentation.ts`. eve - * reads these settings at server startup and applies them to every AI SDK - * model call. The `setup` callback runs later with the resolved agent name, - * not during `defineInstrumentation` itself. + * Declares instrumentation, in either of eve's two layouts. + * + * Export the result as the default export of `agent/instrumentation.ts`, or — + * with `experimental.instrumentationProviders` on — of one file under + * `agent/instrumentation/`. The layout decides how eve reads the value; the two + * are mutually exclusive builds, so only one can apply. `setup` runs at server + * startup, not during this call. + * + * The parameter is a union because a provider and a legacy config overlap on + * `events` and `setup`, so no value-level check separates them. One consequence + * is that excess-property checking is weaker here than it was against the + * config shape alone, and a misspelled key can reach `eve build` rather than + * failing at `tsc`. */ -export function defineInstrumentation( - definition: ExactDefinition, -): T { - return definition; +export function defineInstrumentation< + const TDefinition extends InstrumentationDefinition | ProviderDefinition, +>(definition: TDefinition): InstrumentationDeclaration { + return { ...definition, [PROVIDER]: true }; } + +/** The branded result of {@link defineInstrumentation}. */ +export type InstrumentationDeclaration< + TDefinition extends InstrumentationDefinition | ProviderDefinition = + | InstrumentationDefinition + | ProviderDefinition, +> = TDefinition & { + readonly [PROVIDER]: true; +}; diff --git a/packages/eve/src/public/instrumentation/otel.ts b/packages/eve/src/public/instrumentation/otel.ts new file mode 100644 index 0000000000..f5beb31edb --- /dev/null +++ b/packages/eve/src/public/instrumentation/otel.ts @@ -0,0 +1,59 @@ +/** + * The OpenTelemetry authoring surface for `agent/instrumentation/`. + * + * Two halves, because OpenTelemetry has two: `otel()` is the settings a process + * can only hold one of, and an integration is a destination, of which there may + * be as many as there are files. + * + * Reachable only with `experimental.instrumentationProviders` on. With the flag + * off nothing discovers that directory, so these compile but never run. + */ + +import { createLocalTracesProcessor, resolveLocalTracesContent } from "#tracing/local-traces.js"; +import { + agentRunsIntegration, + otelIntegration, + type ContentOptions, + type OtelIntegration, +} from "#tracing/otel-declaration.js"; + +export { + isOtelDeclaration, + isOtelIntegration, + otel, + otelIntegration, + type ContentOptions, + type OtelDeclaration, + type OtelIntegration, + type OtelIntegrationOptions, + type OtelOptions, +} from "#tracing/otel-declaration.js"; + +export type { SpanExporter, SpanProcessor } from "#compiled/@vercel/otel/index.js"; + +/** + * Vercel Agent Runs, enabled by default in production. + * + * Export it from `agent/instrumentation/agent-runs.ts` to narrow content, or + * export `disableInstrumentation()` from that file to turn it off. + */ +export function agentRuns(options: ContentOptions = {}): OtelIntegration { + return agentRunsIntegration(options); +} + +/** + * The local trace spool `eve dev` reads, as a destination. + * + * Export it from `agent/instrumentation/local.ts` to keep it alongside a hosted + * backend, or export `disableInstrumentation()` from that file to turn it off. + * Omitting the file leaves eve's default in place. + * + * `EVE_TRACES_CONTENT=off` narrows this destination and no other, so declining + * content locally leaves what a hosted backend receives alone. + */ +export function localTraces(options: ContentOptions = {}): OtelIntegration { + return otelIntegration({ + ...resolveLocalTracesContent(options), + spanProcessors: [createLocalTracesProcessor()], + }); +} diff --git a/packages/eve/src/public/instrumentation/provider.test.ts b/packages/eve/src/public/instrumentation/provider.test.ts new file mode 100644 index 0000000000..0d842b2d11 --- /dev/null +++ b/packages/eve/src/public/instrumentation/provider.test.ts @@ -0,0 +1,85 @@ +import { describe, expect, it } from "vitest"; + +import { + defineInstrumentation, + disableInstrumentation, + isInstrumentationDisabled, + isInstrumentationProvider, +} from "#public/instrumentation/index.js"; + +describe("defineInstrumentation", () => { + it("brands a provider-shaped declaration", () => { + const provider = defineInstrumentation({ + events: { + "session.started"(event) { + void event.sessionId; + }, + }, + }); + + expect(isInstrumentationProvider(provider)).toBe(true); + }); + + it("brands a legacy config-shaped declaration", () => { + const config = defineInstrumentation({ functionId: "support", recordInputs: false }); + + expect(isInstrumentationProvider(config)).toBe(true); + expect(config.functionId).toBe("support"); + expect(config).toMatchObject({ functionId: "support", recordInputs: false }); + }); + + it("infers terminal handler events from union-typed discriminants", () => { + const provider = defineInstrumentation({ + events: { + "action.completed": (event) => void [event.acceptedAtMs, event.outcome, event.usage], + "action.failed": (event) => void [event.acceptedAtMs, event.errorCode, event.outcome], + "session.completed": (event) => void event.sessionId, + "step.attempt.completed": (event) => void event.scope, + "turn.failed": (event) => void event.error, + }, + }); + + expect(isInstrumentationProvider(provider)).toBe(true); + }); + + it("exposes durable input request and resolution events", () => { + const provider = defineInstrumentation({ + events: { + "input.requested": (event) => void event.action.callId, + "input.resolved": (event) => void event.outcome, + }, + }); + + expect(isInstrumentationProvider(provider)).toBe(true); + }); + + it("preserves the authored fields", () => { + const setup = (): void => {}; + const declaration = defineInstrumentation({ setup }); + + expect(declaration).toMatchObject({ setup }); + }); +}); + +describe("isInstrumentationProvider", () => { + it("rejects a value that never went through defineInstrumentation", () => { + expect(isInstrumentationProvider({ events: {} })).toBe(false); + }); + + it.each([[null], [undefined], ["provider"], [42]])("rejects %p", (value) => { + expect(isInstrumentationProvider(value)).toBe(false); + }); +}); + +describe("disableInstrumentation", () => { + it("is recognizable as a disabled slot rather than a provider", () => { + const disabled = disableInstrumentation(); + + expect(isInstrumentationDisabled(disabled)).toBe(true); + expect(isInstrumentationProvider(disabled)).toBe(false); + }); + + it("does not treat a provider as a disabled slot", () => { + expect(isInstrumentationDisabled(defineInstrumentation({}))).toBe(false); + }); +}); diff --git a/packages/eve/src/public/instrumentation/provider.ts b/packages/eve/src/public/instrumentation/provider.ts new file mode 100644 index 0000000000..952f7b08b0 --- /dev/null +++ b/packages/eve/src/public/instrumentation/provider.ts @@ -0,0 +1,187 @@ +/** + * The provider contract authored under `agent/instrumentation/`. + * + * Reachable only with `experimental.instrumentationProviders` on. With the flag + * off nothing discovers that directory, so these types compile but never run. + */ + +// Type-only, so nothing couples the public entrypoint to the harness at +// runtime. The event shapes are eve's own vocabulary; deriving the handler map +// from the union below is what keeps the public contract from drifting away +// from the bus that feeds it. +import type { InstrumentationEvent } from "#harness/instrumentation-lifecycle.js"; +import type { JsonValue } from "#public/types/json.js"; + +export type { JsonValue } from "#public/types/json.js"; + +export type { + InstrumentationActionCompletedEvent, + InstrumentationActionFailedEvent, + InstrumentationActionKind, + InstrumentationActionOutcome, + InstrumentationActionOutput, + InstrumentationActionStartedEvent, + InstrumentationAttemptScope, + InstrumentationContentPart, + InstrumentationEvent, + InstrumentationInputKind, + InstrumentationInputOption, + InstrumentationInputOutcome, + InstrumentationInputRequest, + InstrumentationInputRequestedEvent, + InstrumentationInputResolvedEvent, + InstrumentationInputResponse, + InstrumentationModelCallCompletedEvent, + InstrumentationModelCallFailedEvent, + InstrumentationModelCallStartedEvent, + InstrumentationModelRef, + InstrumentationOperationRef, + InstrumentationParentLineage, + InstrumentationSessionFailedEvent, + InstrumentationSessionSettledEvent, + InstrumentationSessionStartedEvent, + InstrumentationSessionTransitionEvent, + InstrumentationStepAttemptMetadataEvent, + InstrumentationStepAttemptCompletedEvent, + InstrumentationStepAttemptFailedEvent, + InstrumentationStepAttemptStartedEvent, + InstrumentationStepAttemptTerminalEvent, + InstrumentationToolCallCompletedEvent, + InstrumentationToolCallFailedEvent, + InstrumentationToolCallStartedEvent, + InstrumentationToolOutput, + InstrumentationTraceContext, + InstrumentationTurnFailedEvent, + InstrumentationTurnSettledEvent, + InstrumentationTurnStartedEvent, + InstrumentationTurnTerminalEvent, + InstrumentationUsage, +} from "#harness/instrumentation-lifecycle.js"; + +/** + * Marks a value as having come from `defineInstrumentation` or a built-in + * factory. + * + * It does not say which layout the value belongs to: a provider and a legacy + * config both carry `events` and `setup`, so no value-level check separates + * them. The layout decides — `agent/instrumentation.ts` is read as a config and + * `agent/instrumentation/*.ts` as providers, and the two are mutually exclusive + * builds. The brand's job is only to catch a default export that never went + * through eve at all. + */ +export const PROVIDER = Symbol.for("eve.instrumentation.provider"); + +/** Marks a slot the author turned off rather than configured. */ +export const DISABLED = Symbol.for("eve.instrumentation.disabled"); + +/** Where the agent is running when `setup` fires. */ +export type InstrumentationEnvironment = "development" | "preview" | "production"; + +/** The local eval run this server was started to serve. */ +export interface EvaluationRef { + readonly runId: string; +} + +/** + * Passed to {@link InstrumentationProvider.setup} once at server startup, + * before any event is published. + */ +export interface ProviderSetupContext { + /** The agent name declared by `defineAgent`. */ + readonly agentName: string; + readonly environment: InstrumentationEnvironment; + /** Present only when this server was started for a local `eve eval` run. */ + readonly evaluation?: EvaluationRef; + /** The eve version running the agent. */ + readonly frameworkVersion: string; +} + +export interface ProviderState { + get(): JsonValue | undefined; + /** Stages a JSON value; `undefined` releases this operation's slot. */ + set(value: JsonValue | undefined): void; +} + +export interface ProviderContext { + readonly state: ProviderState; +} + +/** + * One event handler. + * + * A handler can carry durable JSON state from a start to its terminal through + * `ctx.state`. eve scopes and releases that state by provider and operation. + */ +export type Handler = (event: TEvent, ctx: ProviderContext) => void | PromiseLike; + +type EventForType = TEvent extends { readonly type: infer TEventType } + ? TType extends TEventType + ? TEvent & { readonly type: TType } + : never + : never; + +/** + * The events a provider may handle, one optional handler per event type. + * + * Derived from the event union rather than written out, so a new event reaches + * providers the moment the bus can publish it. + */ +export type ProviderEvents = { + readonly [TType in InstrumentationEvent["type"]]?: Handler< + EventForType + >; +}; + +/** + * What an author writes for one file under `agent/instrumentation/`. + * + * Setup runs in slot order, but event handlers across authored providers run + * concurrently and are failure-isolated. Do not coordinate providers through + * completion order. + */ +export interface ProviderDefinition { + readonly events?: ProviderEvents; + /** Runs once at server startup, before any event is published. */ + readonly setup?: (context: ProviderSetupContext) => void | PromiseLike; + /** Drains anything buffered. eve calls this before a session goes idle. */ + readonly flush?: () => void | PromiseLike; + /** Releases resources when the process is going away. */ + readonly shutdown?: () => void | PromiseLike; +} + +/** A {@link ProviderDefinition} that has been through `defineInstrumentation`. */ +export type InstrumentationProvider = ProviderDefinition & { + readonly [PROVIDER]: true; +}; + +/** A slot the author turned off. eve registers nothing for it. */ +export interface InstrumentationDisabled { + readonly [DISABLED]: true; +} + +/** + * Turns off the slot the file it is exported from names. + * + * Export it as the default of `agent/instrumentation/local.ts` to stop eve + * spooling local traces, for instance. Omitting the file entirely leaves eve's + * default in place, which is why turning one off takes a value. + */ +export function disableInstrumentation(): InstrumentationDisabled { + return { [DISABLED]: true }; +} + +export function isInstrumentationProvider(value: unknown): value is InstrumentationProvider { + return ( + typeof value === "object" && + value !== null && + (value as Partial)[PROVIDER] === true + ); +} + +export function isInstrumentationDisabled(value: unknown): value is InstrumentationDisabled { + return ( + typeof value === "object" && + value !== null && + (value as Partial)[DISABLED] === true + ); +} diff --git a/packages/eve/src/runtime/resolve-agent.ts b/packages/eve/src/runtime/resolve-agent.ts index 65ebf719ea..4d0da10934 100644 --- a/packages/eve/src/runtime/resolve-agent.ts +++ b/packages/eve/src/runtime/resolve-agent.ts @@ -227,6 +227,7 @@ function createResolvedAgentConfig(manifest: CompiledAgentNodeManifest): Resolve if (manifest.config.experimental !== undefined) { config.experimental = { + instrumentationProviders: manifest.config.experimental.instrumentationProviders, subagentPersistentSessions: manifest.config.experimental.subagentPersistentSessions, workflow: manifest.config.experimental.workflow === undefined diff --git a/packages/eve/src/shared/agent-definition.ts b/packages/eve/src/shared/agent-definition.ts index e79311fb7a..3f8e7d3410 100644 --- a/packages/eve/src/shared/agent-definition.ts +++ b/packages/eve/src/shared/agent-definition.ts @@ -196,6 +196,15 @@ export interface AgentLimitsDefinition { * These options are unstable and may change or be removed in any release. */ export interface AgentExperimentalDefinition { + /** + * Reads instrumentation from an `instrumentation/` directory of providers + * rather than a single `agent/instrumentation.ts` config object. + * + * The two layouts are mutually exclusive: with this on, an + * `agent/instrumentation.ts` is a build error, and with it off, an + * `instrumentation/` directory is. + */ + readonly instrumentationProviders?: boolean; /** * Keeps this agent's delegated subagent sessions alive after they answer. * The model can pass `agentId` to a subagent tool to continue a previous diff --git a/packages/eve/src/tracing/agent-otel-provider.test.ts b/packages/eve/src/tracing/agent-otel-provider.test.ts index e5611b1cce..2db1b4ad22 100644 --- a/packages/eve/src/tracing/agent-otel-provider.test.ts +++ b/packages/eve/src/tracing/agent-otel-provider.test.ts @@ -938,10 +938,10 @@ describe("createAgentOtelInstrumentation", () => { spanProcessors: [new SimpleSpanProcessor(exporter)], }); const agentOtel = createAgentOtelInstrumentation({ - recordInputs: false, - recordOutputs: false, frameworkVersion: "test", idGenerator, + recordInputs: false, + recordOutputs: false, stateStore: new InMemoryAgentTraceStateStore(), tracer: provider.getTracer("eve.agent"), }); diff --git a/packages/eve/src/tracing/agent-otel-provider.ts b/packages/eve/src/tracing/agent-otel-provider.ts index 9ff4b72809..b7347a35d2 100644 --- a/packages/eve/src/tracing/agent-otel-provider.ts +++ b/packages/eve/src/tracing/agent-otel-provider.ts @@ -58,9 +58,13 @@ interface AttemptSpanState { } export interface AgentOtelInstrumentationInput { - /** Whether any destination requested model prompts and tool inputs. */ + /** + * Whether to write model prompts and tool call inputs onto spans at all. + * This is the union across destinations, not one destination's policy: a + * destination that declined drops these on its way out instead. + */ readonly recordInputs?: boolean; - /** Whether any destination requested model responses and tool outputs. */ + /** The same, for model responses and tool call outputs. */ readonly recordOutputs?: boolean; readonly frameworkVersion: string; readonly idGenerator: AgentSpanIdGenerator; @@ -509,7 +513,6 @@ export function createAgentOtelInstrumentation( return { hook: { - name: "eve.otel", events: { "action.completed": actions.events["action.completed"], "action.failed": actions.events["action.failed"], @@ -535,6 +538,7 @@ export function createAgentOtelInstrumentation( "turn.failed": onTurnTerminal, "turn.started": onTurnStarted, }, + name: "eve.otel", }, runInContext(operation, execute) { const scope = attemptScopes.get(operation.scope.attemptId) ?? operation.scope; diff --git a/packages/eve/src/tracing/install-instrumentation-runtime.test.ts b/packages/eve/src/tracing/install-instrumentation-runtime.test.ts index 6c0b35bd99..7ff4ee4e51 100644 --- a/packages/eve/src/tracing/install-instrumentation-runtime.test.ts +++ b/packages/eve/src/tracing/install-instrumentation-runtime.test.ts @@ -1,10 +1,13 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; +import { ContextContainer, contextStorage } from "#context/container.js"; +import { turnIdempotencyKey } from "#harness/instrumentation-lifecycle.js"; import { installInstrumentationRuntime } from "#tracing/install-instrumentation-runtime.js"; import { otelIntegration, collectOtelPipeline } from "#tracing/otel-declaration.js"; -const { forceFlush, shutdown } = vi.hoisted(() => ({ +const { forceFlush, internalTerminalState, shutdown } = vi.hoisted(() => ({ forceFlush: vi.fn(async () => undefined), + internalTerminalState: vi.fn(), shutdown: vi.fn(async () => undefined), })); @@ -25,7 +28,17 @@ vi.mock("#tracing/otel-registration.js", async (importOriginal) => { vi.mock("#tracing/agent-otel-provider.js", () => ({ createAgentOtelInstrumentation: () => ({ - hook: {}, + hook: { + events: { + "turn.completed": (_event: unknown, ctx: { state: { get(): unknown } }) => { + internalTerminalState(ctx.state.get()); + }, + "turn.started": (_event: unknown, ctx: { state: { set(value: string): void } }) => { + ctx.state.set("framework"); + }, + }, + name: "eve.otel", + }, runInContext: (_operation: unknown, execute: () => PromiseLike) => execute(), }), })); @@ -35,6 +48,7 @@ const RUNTIME_GLOBAL_KEY = Symbol.for("eve.instrumentation-runtime"); describe("installInstrumentationRuntime", () => { beforeEach(() => { forceFlush.mockClear(); + internalTerminalState.mockClear(); shutdown.mockClear(); delete (globalThis as Record)[RUNTIME_GLOBAL_KEY]; }); @@ -58,4 +72,44 @@ describe("installInstrumentationRuntime", () => { expect(shutdown).toHaveBeenCalledOnce(); expect(providerShutdown).toHaveBeenCalledOnce(); }); + + it("isolates authored state from an internal provider with the same name", async () => { + const authoredTerminalState = vi.fn(); + const runtime = installInstrumentationRuntime({ + collected: collectOtelPipeline([otelIntegration()]), + frameworkVersion: "test", + providers: [ + { + events: { + "turn.completed": (_event, ctx) => authoredTerminalState(ctx.state.get()), + "turn.started": (_event, ctx) => ctx.state.set("authored"), + }, + name: "eve.otel", + stateNamespace: "authored:eve.otel", + }, + ], + serviceName: "weather", + }); + const idempotencyKey = turnIdempotencyKey("session-1", "turn-1"); + + await contextStorage.run(new ContextContainer(), async () => { + await runtime.hooks.publish({ + idempotencyKey, + rootSessionId: "session-1", + sequence: 0, + sessionId: "session-1", + turnId: "turn-1", + type: "turn.started", + }); + await runtime.hooks.publish({ + idempotencyKey, + sessionId: "session-1", + turnId: "turn-1", + type: "turn.completed", + }); + }); + + expect(internalTerminalState).toHaveBeenCalledExactlyOnceWith("framework"); + expect(authoredTerminalState).toHaveBeenCalledExactlyOnceWith("authored"); + }); }); diff --git a/packages/eve/src/tracing/install-instrumentation-runtime.ts b/packages/eve/src/tracing/install-instrumentation-runtime.ts index 9abc32eafb..e3e1319a8f 100644 --- a/packages/eve/src/tracing/install-instrumentation-runtime.ts +++ b/packages/eve/src/tracing/install-instrumentation-runtime.ts @@ -19,14 +19,24 @@ import { registerOtelPipeline, type RegisteredOtelPipeline } from "#tracing/otel const log = createLogger("tracing.install-instrumentation-runtime"); -/** Installs the bus and the one OpenTelemetry pipeline collected for this process. */ +/** + * Installs the process instrumentation runtime around a collected pipeline. + * + * Both layouts land here. `eve dev`'s zero-config default and an authored + * `agent/instrumentation/` directory differ only in where the declared values + * came from, so sharing the install keeps them on one runtime path. + * + * A directory that declared no OpenTelemetry still gets a bus: its providers + * see every event, they just have no spans to hang them on. + */ export function installInstrumentationRuntime(input: { readonly collected: CollectedOtel; readonly frameworkVersion: string; readonly providers: readonly InstrumentationProviderDefinition[]; readonly serviceName: string; }): InstrumentationRuntime { - const providers: InstrumentationProviderDefinition[] = [...input.providers]; + const serialBefore: InstrumentationProviderDefinition[] = []; + const serialAfter: InstrumentationProviderDefinition[] = []; let otelRuntime: RegisteredOtelPipeline | undefined; let runInContext: InstrumentationRuntime["runInContext"] = (_operation, execute) => execute(); @@ -44,28 +54,34 @@ export function installInstrumentationRuntime(input: { stateStore: new ContextAgentTraceStateStore(), tracer: trace.getTracer("eve.agent", input.frameworkVersion), }); - providers.unshift(agentOtel.hook); + // The span must exist before authored providers observe the lifecycle event. + serialBefore.push({ ...agentOtel.hook, stateNamespace: "internal:otel" }); runInContext = agentOtel.runInContext; const releasable = input.collected.pipeline.spanProcessors .filter(isSpanProcessor) .filter(hasSessionRelease); - if (releasable.length > 0) providers.push(sessionReleaseProvider(releasable)); + if (releasable.length > 0) serialAfter.push(sessionReleaseProvider(releasable)); } + const allProviders = [...serialBefore, ...input.providers, ...serialAfter]; let shutdown: Promise | undefined; return registerInstrumentationRuntime({ forceFlush: () => settleAll([ ...(otelRuntime === undefined ? [] : [otelRuntime.forceFlush]), - ...providers.map((provider) => () => provider.flush?.()), + ...allProviders.map((provider) => () => provider.flush?.()), ]), - hooks: createInstrumentationHooks(providers), + hooks: createInstrumentationHooks({ + parallel: input.providers, + serialAfter, + serialBefore, + }), runInContext, shutdown: () => { shutdown ??= settleAll([ ...(otelRuntime === undefined ? [] : [otelRuntime.shutdown]), - ...providers.map((provider) => () => provider.shutdown?.()), + ...allProviders.map((provider) => () => provider.shutdown?.()), ]); return shutdown; }, @@ -85,6 +101,7 @@ function sessionReleaseProvider( return { events: { "session.completed": release, "session.failed": release }, name: "eve.session-release", + stateNamespace: "internal:session-release", }; } diff --git a/packages/eve/src/tracing/local-traces.test.ts b/packages/eve/src/tracing/local-traces.test.ts index 2128084f44..fb8e1b50a4 100644 --- a/packages/eve/src/tracing/local-traces.test.ts +++ b/packages/eve/src/tracing/local-traces.test.ts @@ -1,6 +1,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; -import { createLocalTracesProcessor, resolveLocalTracesContent } from "#tracing/local-traces.js"; +import { createLocalTracesProcessor } from "#tracing/local-traces.js"; +import { localTraces } from "#public/instrumentation/otel.js"; vi.mock("#tracing/local-trace-span-processor.js", () => ({ LocalTraceSpanProcessor: class { @@ -54,23 +55,11 @@ describe("createLocalTracesProcessor", () => { it("is inert outside a development worker", async () => { vi.stubEnv("EVE_DEV_WORKER_APP_ROOT", undefined); - const processor = createLocalTracesProcessor(); + const [processor] = localTraces().spanProcessors; + if (processor === undefined || processor === "auto") throw new Error("Expected a processor."); expect(() => processor.onEnd(agentSpan("session-one", "a".repeat(32)))).not.toThrow(); await expect(processor.forceFlush()).resolves.toBeUndefined(); await expect(processor.shutdown()).resolves.toBeUndefined(); }); - - it("lets the environment override narrow only this destination", () => { - vi.stubEnv("EVE_TRACES_CONTENT", "off"); - expect(resolveLocalTracesContent()).toEqual({ recordInputs: false, recordOutputs: false }); - }); - - it("preserves explicit destination narrowing when content is enabled", () => { - vi.stubEnv("EVE_TRACES_CONTENT", "on"); - expect(resolveLocalTracesContent({ recordInputs: false })).toEqual({ - recordInputs: false, - recordOutputs: true, - }); - }); }); diff --git a/packages/eve/src/tracing/local-traces.ts b/packages/eve/src/tracing/local-traces.ts index 8500f7d1e9..83ca4591f2 100644 --- a/packages/eve/src/tracing/local-traces.ts +++ b/packages/eve/src/tracing/local-traces.ts @@ -23,7 +23,15 @@ export interface LocalTracesProcessor extends SpanProcessor { releaseSession(sessionId: string): Promise; } -/** Whether a processor still exposes the local spool's session lifecycle. */ +/** + * Reports whether a processor tracks which session owns which trace, so eve can + * tell it when that session is done. + * + * Anything standing between eve and the spool has to answer for the spool, so + * this is the check a wrapper uses to decide whether it must forward the call. + * + * @internal + */ export function hasSessionRelease(processor: SpanProcessor): processor is LocalTracesProcessor { return typeof (processor as Partial).releaseSession === "function"; } @@ -35,7 +43,8 @@ export function hasSessionRelease(processor: SpanProcessor): processor is LocalT * to observe spans to track which session owns which trace. * * Internal because of `releaseSession`, which eve's runtime drives off session - * lifecycle. + * lifecycle. The authored surface is `localTraces()`, which wraps this in an + * `OtelIntegration`. */ export function createLocalTracesProcessor( input: { readonly appRoot?: string } = {}, @@ -80,7 +89,16 @@ export function createLocalTracesProcessor( }; } -/** Intersects the local destination policy with `EVE_TRACES_CONTENT`. */ +/** + * The local spool's content policy: its options, intersected with + * `EVE_TRACES_CONTENT`. + * + * The variable used to be the process-wide switch. It now applies to this one + * destination, and only ever narrows — `off` still wins where it applies, but + * it no longer decides what a hosted backend beside it receives. + * + * @internal + */ export function resolveLocalTracesContent( options: { readonly recordInputs?: boolean; diff --git a/packages/eve/src/tracing/otel-declaration.test.ts b/packages/eve/src/tracing/otel-declaration.test.ts index 0487cbcdf7..09a8eed6d3 100644 --- a/packages/eve/src/tracing/otel-declaration.test.ts +++ b/packages/eve/src/tracing/otel-declaration.test.ts @@ -61,7 +61,9 @@ describe("otelIntegration", () => { expect(otelIntegration().content).toStrictEqual({ recordInputs: true, recordOutputs: true }); }); - it("puts a declined policy in front of every processor", () => { + // An author's own processor is part of this destination, and the point of + // declining is that nothing under it sees what was said. + it("puts a declined policy in front of every processor, an author's included", () => { const first = processor(); const integration = otelIntegration({ recordOutputs: false, spanProcessors: [first] }); @@ -135,12 +137,16 @@ describe("collectOtelPipeline", () => { }); expect(collected.settings).toStrictEqual({ functionId: "weather", + // Nothing declared a destination, so nothing asked for content. recordInputs: false, recordOutputs: false, traceChannelRequests: true, }); }); + // Content governs what is written onto the span, which is upstream of every + // destination — so one that wants it is enough, and the ones that declined + // drop it on their own way out. it("takes content capture as the union across destinations", () => { const collected = collectOtelPipeline([ otelIntegration({ recordInputs: false, recordOutputs: false }), diff --git a/packages/eve/src/tracing/otel-declaration.ts b/packages/eve/src/tracing/otel-declaration.ts index b603e64cd0..abfd256140 100644 --- a/packages/eve/src/tracing/otel-declaration.ts +++ b/packages/eve/src/tracing/otel-declaration.ts @@ -6,6 +6,7 @@ import type { SpanProcessorOrName, } from "#compiled/@vercel/otel/index.js"; +import { PROVIDER, type InstrumentationProvider } from "#public/instrumentation/provider.js"; import { batchSpanProcessor } from "#tracing/batch-span-processor.js"; import type { ResolvedContentOptions } from "#tracing/content-attributes.js"; import { contentFilteringProcessor } from "#tracing/content-span-processor.js"; @@ -50,7 +51,15 @@ export interface OtelOptions { readonly propagators?: readonly PropagatorOrName[]; } -/** What one destination records of the conversation itself. */ +/** + * What one destination records of the conversation itself. + * + * Declining is per destination, not per process: content is written onto the + * span if any destination wants it, and one that declined never exports it. So + * an agent whose every destination declines still never materializes a prompt — + * the union of nothing is nothing — but a local spool and a hosted backend no + * longer have to agree. + */ export interface ContentOptions { /** Record model prompts and tool call inputs. Defaults to `true`. */ readonly recordInputs?: boolean; @@ -73,14 +82,15 @@ const OTEL_INTEGRATION = Symbol.for("eve.instrumentation.otel-integration"); * The declared OpenTelemetry pipeline settings. eve collects this before * building the tracer provider, so it is a value rather than a side effect. */ -export interface OtelDeclaration { +export interface OtelDeclaration extends InstrumentationProvider { readonly [OTEL_DECLARATION]: true; readonly options: OtelOptions; } /** One declared destination. A process may have as many as it has files. */ -export interface OtelIntegration { +export interface OtelIntegration extends InstrumentationProvider { readonly [OTEL_INTEGRATION]: true; + /** Resolved from `ContentOptions`, so the union does not re-apply defaults. */ readonly content: ResolvedContentOptions; readonly spanProcessors: readonly SpanProcessorOrName[]; } @@ -88,10 +98,12 @@ export interface OtelIntegration { /** * Declares the process-wide OpenTelemetry settings. * - * This remains internal until the provider authoring API exposes it. + * Export it from `agent/instrumentation/otel.ts`. Omitting the file is the + * common case: eve registers the pipeline for whatever destinations are + * declared beside it, and this only names what those destinations share. */ export function otel(options: OtelOptions = {}): OtelDeclaration { - return { [OTEL_DECLARATION]: true, options }; + return { [OTEL_DECLARATION]: true, [PROVIDER]: true, options }; } /** @@ -100,16 +112,25 @@ export function otel(options: OtelOptions = {}): OtelDeclaration { * A `traceExporter` is wrapped in eve's batching processor, which is what makes * the one-line form of a hosted backend enough. Pass `spanProcessors` instead * when the destination needs its own batching, sampling, or filtering. + * + * Declining content wraps every processor here, an author's included: they are + * this destination, and the point of declining is that nothing under it sees + * what was said. */ export function otelIntegration(options: OtelIntegrationOptions = {}): OtelIntegration { - const content = resolveContentOptions(options); + const content: ResolvedContentOptions = { + recordInputs: options.recordInputs !== false, + recordOutputs: options.recordOutputs !== false, + }; const declared = options.spanProcessors ?? []; const spanProcessors = options.traceExporter === undefined ? declared : [...declared, batchSpanProcessor(options.traceExporter)]; + return { [OTEL_INTEGRATION]: true, + [PROVIDER]: true, content, spanProcessors: content.recordInputs && content.recordOutputs @@ -123,6 +144,7 @@ export function agentRunsIntegration(options: ContentOptions = {}): OtelIntegrat const content = resolveContentOptions(options); return { [OTEL_INTEGRATION]: true, + [PROVIDER]: true, content, spanProcessors: content.recordInputs && content.recordOutputs @@ -167,8 +189,9 @@ export interface OtelHarnessSettings { readonly functionId?: string; readonly traceChannelRequests: boolean; /** - * What to materialize on spans at all. Each destination independently strips - * anything it declined before export. + * What to write onto a span at all, as opposed to what any one destination + * exports. `agent/instrumentation.ts` sets this directly; a provider + * directory arrives at it as the union of its destinations. */ readonly recordInputs?: boolean; readonly recordOutputs?: boolean; @@ -193,6 +216,10 @@ export interface CollectedOtel { * happened to visit first. With one declaration per file that collision needs * two files both exporting `otel()`, which is the only way to reach it. * + * Content capture is the union of what the destinations asked for, because it + * governs what is written rather than what is exported. Each destination's own + * processors already drop what it declined. + * * @internal */ export function collectOtelPipeline(values: readonly unknown[]): CollectedOtel { diff --git a/packages/eve/src/tracing/otel-registration.scenario.test.ts b/packages/eve/src/tracing/otel-registration.scenario.test.ts index 85d5e3d6f1..c9b70a0897 100644 --- a/packages/eve/src/tracing/otel-registration.scenario.test.ts +++ b/packages/eve/src/tracing/otel-registration.scenario.test.ts @@ -82,7 +82,8 @@ describe("registerOtelPipeline", () => { it("does not export the private registration span", async () => { const exporter = new InMemorySpanExporter(); const processor = new SimpleSpanProcessor(exporter); - registerOtelPipeline({ + const shutdown = vi.spyOn(processor, "shutdown"); + const runtime = registerOtelPipeline({ pipeline: { spanProcessors: [processor] }, serviceName: "weather", }); @@ -91,7 +92,8 @@ describe("registerOtelPipeline", () => { await processor.forceFlush(); expect(exporter.getFinishedSpans().map((span) => span.name)).toEqual(["user.work"]); - await processor.shutdown(); + await runtime.shutdown(); + expect(shutdown).toHaveBeenCalledOnce(); }); }); diff --git a/packages/eve/test/scenarios/compiled-artifacts-bootstrap.scenario.test.ts b/packages/eve/test/scenarios/compiled-artifacts-bootstrap.scenario.test.ts index d4cc234f38..0659305653 100644 --- a/packages/eve/test/scenarios/compiled-artifacts-bootstrap.scenario.test.ts +++ b/packages/eve/test/scenarios/compiled-artifacts-bootstrap.scenario.test.ts @@ -20,6 +20,10 @@ const createAppRoot = useTemporaryAppRoots(); describe("writeCompiledArtifactsFiles", () => { afterEach(() => { delete (globalThis as Record).__eveInstrumentationLoaded; + delete (globalThis as Record).__eveProviderSetups; + delete (globalThis as Record)[ + Symbol.for("eve.harness-instrumentation-providers") + ]; }); it("installs compile metadata into bundled compiled artifacts", async () => { @@ -112,6 +116,136 @@ describe("writeCompiledArtifactsFiles", () => { expect(instrumentationPluginModule.default()).toBeUndefined(); }); + it("registers one provider per file when the instrumentationProviders flag is on", async () => { + const { agentRoot, appRoot } = await createAppRoot("eve-compiled-artifacts-providers-", { + packageName: "compiled-artifacts-providers-test-agent", + }); + const outDir = join(appRoot, ".workflow-build"); + const definePath = resolvePackageSourceFilePath( + "src/public/instrumentation/index.ts", + ).replaceAll("\\", "/"); + + await writeFile( + join(agentRoot, "agent.ts"), + [ + "export default {", + ' model: "openai/gpt-5.4",', + " experimental: { instrumentationProviders: true },", + "};", + "", + ].join("\n"), + ); + await writeFile(join(agentRoot, "instructions.md"), "You are a precise assistant.\n"); + await mkdir(join(agentRoot, "instrumentation"), { recursive: true }); + for (const slot of ["local", "otel"]) { + await writeFile( + join(agentRoot, "instrumentation", `${slot}.ts`), + [ + `import { defineInstrumentation } from ${JSON.stringify(definePath)};`, + "", + "const container = globalThis as Record;", + "", + "export default defineInstrumentation({", + " setup(context) {", + " container.__eveProviderSetups ??= [];", + ` (container.__eveProviderSetups as string[]).push(\`${slot}:\${context.agentName}\`);`, + " },", + "});", + "", + ].join("\n"), + ); + } + + const compileResult = await compileAgent({ startPath: appRoot }); + const generatedArtifacts = await writeCompiledArtifactsFiles({ + compileResult, + defaultWorkflowWorld: "local", + outDir, + }); + const instrumentationPluginPath = generatedArtifacts.instrumentationPluginPath; + + if (instrumentationPluginPath === undefined) { + throw new Error("Expected instrumentation plugin path to be generated."); + } + + expect(generatedArtifacts.instrumentationSourcePaths).toEqual([ + join(agentRoot, "instrumentation", "local.ts"), + join(agentRoot, "instrumentation", "otel.ts"), + ]); + + const instrumentationPluginSource = await readFile(instrumentationPluginPath, "utf8"); + + expect(instrumentationPluginSource).toContain('slot: "local"'); + expect(instrumentationPluginSource).toContain('slot: "otel"'); + expect(instrumentationPluginSource).toContain("seedInstrumentationProviders();"); + expect(instrumentationPluginSource).toContain("shutdownInstrumentationProviders"); + expect(instrumentationPluginSource).toContain("hooks?.hook('close'"); + expect(instrumentationPluginSource).not.toContain("registerInstrumentationConfig"); + + const instrumentationPlugin = (await import(pathToFileURL(instrumentationPluginPath).href)) as { + default: (nitroApp: { + hooks: { hook(name: "close", handler: () => Promise): void }; + }) => void; + }; + const closeHandlers: Array<() => Promise> = []; + instrumentationPlugin.default({ + hooks: { + hook: (_name, handler) => closeHandlers.push(handler), + }, + }); + + // The plugin resolves the registry by absolute path while the assertion + // resolves it by package alias, so this also proves the globalThis rooting + // survives two module instances. + const { getInstrumentationProviders } = + await import("../../src/harness/instrumentation-providers.js"); + + expect((globalThis as Record).__eveProviderSetups).toEqual([ + "local:compiled-artifacts-providers-test-agent", + "otel:compiled-artifacts-providers-test-agent", + ]); + expect(getInstrumentationProviders().map((entry) => entry.slot)).toEqual(["local", "otel"]); + expect(closeHandlers).toHaveLength(1); + await closeHandlers[0]?.(); + }); + + it("generates the provider plugin for built-in destinations without authored files", async () => { + const { agentRoot, appRoot } = await createAppRoot( + "eve-compiled-artifacts-default-providers-", + { + packageName: "compiled-artifacts-default-providers-test-agent", + }, + ); + const outDir = join(appRoot, ".workflow-build"); + await writeFile( + join(agentRoot, "agent.ts"), + [ + "export default {", + ' model: "openai/gpt-5.4",', + " experimental: { instrumentationProviders: true },", + "};", + "", + ].join("\n"), + ); + await writeFile(join(agentRoot, "instructions.md"), "You are a precise assistant.\n"); + + const compileResult = await compileAgent({ startPath: appRoot }); + const generatedArtifacts = await writeCompiledArtifactsFiles({ + compileResult, + defaultWorkflowWorld: "local", + outDir, + }); + const instrumentationPluginPath = generatedArtifacts.instrumentationPluginPath; + if (instrumentationPluginPath === undefined) { + throw new Error("Expected instrumentation plugin path to be generated."); + } + + expect(generatedArtifacts.instrumentationSourcePaths).toEqual([]); + expect(await readFile(instrumentationPluginPath, "utf8")).toContain( + "seedInstrumentationProviders();", + ); + }); + it("surfaces instrumentation import failures when the Nitro plugin module loads", async () => { const { agentRoot, appRoot } = await createAppRoot( "eve-compiled-artifacts-instrumentation-error-", diff --git a/packages/eve/test/scenarios/eval-command-environment.scenario.test.ts b/packages/eve/test/scenarios/eval-command-environment.scenario.test.ts index 875a3c4196..d6dcd70c05 100644 --- a/packages/eve/test/scenarios/eval-command-environment.scenario.test.ts +++ b/packages/eve/test/scenarios/eval-command-environment.scenario.test.ts @@ -44,6 +44,8 @@ const DEVELOPMENT_ENV_KEYS = [ "EVE_DEV_LOCAL_ONLY", "EVE_DEV_SHARED", "EVE_DEV_SHELL_ONLY", + "EVE_EVALUATION", + "EVE_EVALUATION_RUN_ID", ] as const; async function createEnvironmentFixture(): Promise { @@ -192,6 +194,8 @@ describe("eve eval environment loading", () => { expect(close).toHaveBeenCalledTimes(1); expect(handle.shutdown).toHaveBeenCalledTimes(1); + expect(process.env.EVE_EVALUATION).toBe("1"); + expect(process.env.EVE_EVALUATION_RUN_ID).toMatch(/^[0-9a-f-]{36}$/u); expect(exit).toHaveBeenCalledWith(0); });