feat(eve): add the instrumentation provider layout

.changeset/instrumentation-provider-layout.md

@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+Add the experimental `agent/instrumentation/` provider layout with durable lifecycle handlers, including user input boundaries and action settlement-time, outcome, error-code, and usage metadata, final setup context, reserved OpenTelemetry destinations, and coordinated flush and shutdown. OpenTelemetry singleton settings and destinations are exposed through `eve/instrumentation/otel`, and eve's AI SDK bridge composes with registered integrations.

packages/eve/package.json

@@ -206,6 +206,11 @@
       "import": "./dist/src/public/instrumentation/index.js",
       "default": "./dist/src/public/instrumentation/index.js"
     },
+    "./instrumentation/otel": {
+      "types": "./dist/src/public/instrumentation/otel.d.ts",
+      "import": "./dist/src/public/instrumentation/otel.js",
+      "default": "./dist/src/public/instrumentation/otel.js"
+    },
     "./schedules": {
       "types": "./dist/src/public/schedules/index.d.ts",
       "import": "./dist/src/public/schedules/index.js",

packages/eve/src/compiler/manifest.ts

@@ -428,6 +428,7 @@ const compiledAgentConfigSchema: z.ZodType<CompiledAgentDefinition> = z
     dynamicModel: compiledDynamicModelDefinitionSchema.optional(),
     experimental: z
       .object({
+        instrumentationProviders: z.boolean().optional(),
         subagentPersistentSessions: z.boolean().optional(),
         workflow: compiledAgentWorkflowDefinitionSchema.optional(),
       })
@@ -833,6 +834,7 @@ export function createCompiledAgentNodeManifest(input: {
         input.config.experimental === undefined
           ? undefined
           : {
+              instrumentationProviders: input.config.experimental.instrumentationProviders,
               subagentPersistentSessions: input.config.experimental.subagentPersistentSessions,
               workflow:
                 input.config.experimental.workflow === undefined

packages/eve/src/compiler/normalize-agent-config.ts

@@ -176,6 +176,10 @@ function normalizeExperimentalDefinition(
 
   const compiledExperimental: Mutable<NonNullable<CompiledAgentDefinition["experimental"]>> = {};
 
+  if (experimental.instrumentationProviders !== undefined) {
+    compiledExperimental.instrumentationProviders = experimental.instrumentationProviders;
+  }
+
   if (experimental.subagentPersistentSessions !== undefined) {
     compiledExperimental.subagentPersistentSessions = experimental.subagentPersistentSessions;
   }

packages/eve/src/discover/agent.integration.test.ts

@@ -511,6 +511,23 @@ describe("discoverAgent (memory)", () => {
     ]);
   });
 
+  it("recognizes the instrumentation provider directory", async () => {
+    const project = buildMemoryAgentProject({
+      agentDirectories: ["instrumentation"],
+      agentFiles: {
+        "instructions.md": "You are a precise assistant.",
+      },
+    });
+
+    const result = await discoverAgent({
+      agentRoot: project.agentRoot,
+      appRoot: project.appRoot,
+      source: project.source,
+    });
+
+    expect(result.diagnostics).toEqual([]);
+  });
+
   it("rejects authored tool filenames that violate the tool-name charset", async () => {
     const project = buildMemoryAgentProject({
       agentFiles: {

packages/eve/src/discover/filesystem.ts

@@ -44,6 +44,7 @@ export type AgentRootEntryKind =
   | "extensions-directory"
   | "hooks-directory"
   | "ignored-directory"
+  | "instrumentation-directory"
   | "instructions-directory"
   | "instructions-markdown"
   | "instructions-module"
@@ -179,6 +180,10 @@ export function classifyAgentRootEntry(
       return "instructions-directory";
     }
 
+    if (name === "instrumentation") {
+      return "instrumentation-directory";
+    }
+
     if (name === "lib") {
       return "lib-directory";
     }

packages/eve/src/evals/cli/eval.ts

@@ -1,8 +1,13 @@
+import { randomUUID } from "node:crypto";
 import { readFile } from "node:fs/promises";
 import { basename, join } from "node:path";
 
 import { loadDevelopmentEnvironmentFiles } from "#cli/dev/environment.js";
 import { shutdownActiveSandboxHandles } from "#execution/sandbox/active-handles.js";
+import {
+  EVE_EVALUATION_ENV_FLAG,
+  EVE_EVALUATION_RUN_ID_ENV,
+} from "#internal/application/dev-environment.js";
 import { resolveApplicationRoot } from "#internal/application/paths.js";
 import { createDevelopmentServer, type DevelopmentServer } from "#internal/nitro/host.js";
 import { createEvalClient } from "#evals/cli/eval-client.js";
@@ -141,6 +146,10 @@ export async function runEvalCommand(
         url: options.url,
       });
     } else {
+      // Set before the server boots, because a provider's `setup` reads it
+      // once at startup and never again.
+      process.env[EVE_EVALUATION_ENV_FLAG] = "1";
+      process.env[EVE_EVALUATION_RUN_ID_ENV] = randomUUID();
       devServer = createDevelopmentServer(appRoot, { host: "127.0.0.1", port: 0 });
       const started = await devServer.start();
       client = await createEvalClient({ kind: "local", url: started.url });

packages/eve/src/harness/ai-sdk-hook-bridge.test.ts

@@ -94,7 +94,10 @@ describe("createAiSdkHookBridge", () => {
   it("passes the identity captured at model-call start to the context runner", async () => {
     const ids: string[] = [];
     const hooks = createInstrumentationHooks([
-      { events: { "model.call.started": (event) => void ids.push(event.idempotencyKey) } },
+      {
+        events: { "model.call.started": (event) => void ids.push(event.idempotencyKey) },
+        name: "identity",
+      },
     ]);
     const bridge = createAiSdkHookBridge(scope, hooks, (operation, execute) => {
       ids.push(operation.idempotencyKey);
@@ -132,7 +135,10 @@ describe("createAiSdkHookBridge", () => {
   it("derives replay-stable model identity without the AI SDK call ID", async () => {
     const keys: string[] = [];
     const hooks = createInstrumentationHooks([
-      { events: { "model.call.started": (event) => void keys.push(event.idempotencyKey) } },
+      {
+        events: { "model.call.started": (event) => void keys.push(event.idempotencyKey) },
+        name: "identity",
+      },
     ]);
 
     for (const callId of ["sdk-random-1", "sdk-random-2"]) {
@@ -155,6 +161,7 @@ describe("createAiSdkHookBridge", () => {
             events.push(event);
           },
         },
+        name: "metadata",
       },
     ]);
     const bridge = createAiSdkHookBridge(scope, hooks);
@@ -183,11 +190,13 @@ describe("createAiSdkHookBridge", () => {
             throw new Error("provider failed");
           },
         },
+        name: "failing",
       },
       {
         events: {
           "model.call.completed": after,
         },
+        name: "observer",
       },
     ]);
     const bridge = createAiSdkHookBridge(scope, hooks);
@@ -211,7 +220,9 @@ describe("createAiSdkHookBridge", () => {
 
   it("terminalizes started operations when the attempt errors", async () => {
     const after = vi.fn();
-    const hooks = createInstrumentationHooks([{ events: { "model.call.failed": after } }]);
+    const hooks = createInstrumentationHooks([
+      { events: { "model.call.failed": after }, name: "terminal" },
+    ]);
     const bridge = createAiSdkHookBridge(scope, hooks);
 
     await Reflect.apply(bridge.onLanguageModelCallStart!, bridge, [
@@ -236,8 +247,8 @@ describe("createAiSdkHookBridge", () => {
     });
     const started = vi.fn();
     const hooks = createInstrumentationHooks([
-      { events: { "step.attempt.started": mutator } },
-      { events: { "step.attempt.started": started } },
+      { events: { "step.attempt.started": mutator }, name: "mutator" },
+      { events: { "step.attempt.started": started }, name: "observer" },
     ]);
     const bridge = createAiSdkHookBridge(scope, hooks);
 
@@ -272,7 +283,10 @@ describe("createAiSdkHookBridge", () => {
       expect(Object.isFrozen(event.usage.inputTokenDetails)).toBe(true);
     });
     const hooks = createInstrumentationHooks([
-      { events: { "model.call.completed": after, "model.call.started": before } },
+      {
+        events: { "model.call.completed": after, "model.call.started": before },
+        name: "model",
+      },
     ]);
     const bridge = createAiSdkHookBridge(scope, hooks);
 
@@ -364,7 +378,10 @@ describe("createAiSdkHookBridge", () => {
         expect(Object.isFrozen(event.output)).toBe(true);
       });
       const hooks = createInstrumentationHooks([
-        { events: { "tool.call.completed": after, "tool.call.started": before } },
+        {
+          events: { "tool.call.completed": after, "tool.call.started": before },
+          name: "tool",
+        },
       ]);
       const bridge = createAiSdkHookBridge(scope, hooks);
       const toolCall = { input: { q: "eve" }, toolCallId: "tool-1", toolName: "search" };
@@ -438,7 +455,9 @@ describe("createAiSdkHookBridge", () => {
 
   it("skips a terminal handler when the operation never started", async () => {
     const completed = vi.fn();
-    const hooks = createInstrumentationHooks([{ events: { "model.call.completed": completed } }]);
+    const hooks = createInstrumentationHooks([
+      { events: { "model.call.completed": completed }, name: "terminal" },
+    ]);
     const bridge = createAiSdkHookBridge(scope, hooks);
 
     // No onLanguageModelCallStart, so the bridge holds no id and publishes
@@ -476,6 +495,7 @@ describe("createAiSdkHookBridge", () => {
             terminalStates.set(event.idempotencyKey, started.get(event.idempotencyKey));
           },
         },
+        name: "parallel",
       },
     ]);
     const bridge = createAiSdkHookBridge(scope, hooks);

packages/eve/src/harness/ai-sdk-telemetry.test.ts

@@ -0,0 +1,25 @@
+import type { Telemetry } from "ai";
+import { afterEach, describe, expect, it } from "vitest";
+
+import { getRegisteredTelemetryIntegrations } from "#harness/ai-sdk-telemetry.js";
+
+describe("getRegisteredTelemetryIntegrations", () => {
+  const original = globalThis.AI_SDK_TELEMETRY_INTEGRATIONS;
+
+  afterEach(() => {
+    globalThis.AI_SDK_TELEMETRY_INTEGRATIONS = original;
+  });
+
+  it("is empty when nothing has registered", () => {
+    globalThis.AI_SDK_TELEMETRY_INTEGRATIONS = undefined;
+    expect(getRegisteredTelemetryIntegrations()).toEqual([]);
+  });
+
+  it("reports the integrations in registration order", () => {
+    const first: Telemetry = { onStart() {} };
+    const second: Telemetry = { onStart() {} };
+    globalThis.AI_SDK_TELEMETRY_INTEGRATIONS = [first, second];
+
+    expect(getRegisteredTelemetryIntegrations()).toEqual([first, second]);
+  });
+});

packages/eve/src/harness/otel-integration.ts → packages/eve/src/harness/ai-sdk-telemetry.ts

@@ -16,10 +16,17 @@ export function ensureOtelIntegration(): void {
     return;
   }
   registered = true;
-  registerTelemetry(createOtelIntegration());
+  registerTelemetry(new OpenTelemetry({ runtimeContext: true }));
 }
 
-/** Creates the existing OTel integration for explicit per-call composition. */
-export function createOtelIntegration(): Telemetry {
-  return new OpenTelemetry({ runtimeContext: true });
+/**
+ * Every integration currently registered with the AI SDK — eve's own, plus any
+ * an authored instrumentation module added with `registerTelemetry`.
+ *
+ * A per-call `integrations` list replaces the registered ones rather than
+ * adding to them, so anything that passes integrations per call has to carry
+ * these forward or they stop receiving events.
+ */
+export function getRegisteredTelemetryIntegrations(): readonly Telemetry[] {
+  return globalThis.AI_SDK_TELEMETRY_INTEGRATIONS ?? [];
 }

packages/eve/src/harness/instrumentation-config.test.ts

@@ -1,5 +1,7 @@
 import { describe, expect, it, vi } from "vitest";
 
+import type { InstrumentationSetupContext } from "#public/instrumentation/index.js";
+
 /**
  * Regression coverage for the instrumentation-config chunk-isolation failure.
  *
@@ -25,7 +27,7 @@ describe("instrumentation-config chunk-isolation regression", () => {
     vi.resetModules();
     const moduleA = await import("#harness/instrumentation-config.js");
     const config = { functionId: "test.instrumentation.cross-module.alice" };
-    moduleA.registerInstrumentationConfig(config, { agentName: "test-agent" });
+    await moduleA.registerInstrumentationConfig(config, { agentName: "test-agent" });
 
     vi.resetModules();
     const moduleB = await import("#harness/instrumentation-config.js");
@@ -40,7 +42,7 @@ describe("instrumentation-config chunk-isolation regression", () => {
     const { registerInstrumentationConfig } = await import("#harness/instrumentation-config.js");
 
     const canary = { functionId: "test.instrumentation.global-mount.canary" };
-    registerInstrumentationConfig(canary, { agentName: "test-agent" });
+    await registerInstrumentationConfig(canary, { agentName: "test-agent" });
 
     expect((globalThis as Record<symbol, unknown>)[globalKey]).toBe(canary);
   });
@@ -51,7 +53,7 @@ describe("instrumentation-config chunk-isolation regression", () => {
     vi.resetModules();
     const moduleA = await import("#harness/instrumentation-config.js");
     const config = { functionId: "test.instrumentation.reimport.canary" };
-    moduleA.registerInstrumentationConfig(config, { agentName: "test-agent" });
+    await moduleA.registerInstrumentationConfig(config, { agentName: "test-agent" });
     const firstRef = (globalThis as Record<symbol, unknown>)[globalKey];
 
     vi.resetModules();
@@ -61,13 +63,23 @@ describe("instrumentation-config chunk-isolation regression", () => {
     expect(secondRef).toBe(firstRef);
   });
 
-  it("invokes the setup callback with the supplied context", async () => {
+  it("awaits the setup callback with the resolved context", async () => {
     vi.resetModules();
     const { registerInstrumentationConfig } = await import("#harness/instrumentation-config.js");
 
-    const setup = vi.fn();
-    registerInstrumentationConfig({ setup }, { agentName: "weather-agent" });
+    const contexts: InstrumentationSetupContext[] = [];
+    await registerInstrumentationConfig(
+      {
+        setup: (context) => {
+          contexts.push(context);
+        },
+      },
+      { agentName: "weather-agent" },
+    );
 
-    expect(setup).toHaveBeenCalledExactlyOnceWith({ agentName: "weather-agent" });
+    expect(contexts).toHaveLength(1);
+    expect(contexts[0]?.agentName).toBe("weather-agent");
+    expect(contexts[0]?.environment).toMatch(/^(development|production)$/);
+    expect(contexts[0]?.frameworkVersion).toEqual(expect.any(String));
   });
 });

packages/eve/src/harness/instrumentation-config.ts

@@ -1,8 +1,6 @@
-import type {
-  InstrumentationDefinition,
-  InstrumentationSetupContext,
-} from "#public/instrumentation/index.js";
+import { createInstrumentationSetupContext } from "#harness/instrumentation-setup-context.js";
 import { activateOtelSettings } from "#harness/otel-settings.js";
+import type { InstrumentationDefinition } from "#public/instrumentation/index.js";
 
 /**
  * Process-global store for the authored instrumentation config.
@@ -29,26 +27,32 @@ interface InstrumentationConfigGlobal {
 const globalContainer = globalThis as typeof globalThis & InstrumentationConfigGlobal;
 
 /**
- * Registers the authored instrumentation config and invokes its `setup`
- * callback with the resolved agent name.
+ * Registers the authored instrumentation config and awaits its `setup`
+ * callback.
  *
  * Called once by the generated instrumentation Nitro plugin at server
  * startup. Subsequent calls overwrite the previous value.
  *
+ * The store write lands before `setup` runs so a synchronous caller sees the
+ * config without waiting on the returned promise.
+ *
  * @internal — not part of the public API.
  */
 export async function registerInstrumentationConfig(
   config: InstrumentationDefinition,
-  context: InstrumentationSetupContext,
+  input: { readonly agentName: string },
 ): Promise<void> {
   globalContainer[INSTRUMENTATION_CONFIG_GLOBAL_KEY] = config;
+  // The presence of a config is what turns telemetry on in this layout, so the
+  // settings the harness reads are activated with it rather than by a
+  // registered pipeline — this layout leaves `registerOTel` to `setup`.
   activateOtelSettings({
     functionId: config.functionId,
     recordInputs: config.recordInputs,
     recordOutputs: config.recordOutputs,
     traceChannelRequests: config.traceChannelRequests === true,
   });
-  await config.setup?.(context);
+  await config.setup?.(createInstrumentationSetupContext(input.agentName));
 }
 
 /**