diff --git a/.changeset/atomic-start-hooks.md b/.changeset/atomic-start-hooks.md new file mode 100644 index 0000000000..08deadd0d5 --- /dev/null +++ b/.changeset/atomic-start-hooks.md @@ -0,0 +1,9 @@ +--- +"@workflow/core": minor +"@workflow/world": minor +"@workflow/world-local": minor +"@workflow/world-postgres": minor +"@workflow/world-vercel": patch +--- + +Add experimental atomic start-hook idempotency for worlds that opt into start-time hook admission. diff --git a/.changeset/idempotent-start-hooks.md b/.changeset/idempotent-start-hooks.md new file mode 100644 index 0000000000..acafc4dea8 --- /dev/null +++ b/.changeset/idempotent-start-hooks.md @@ -0,0 +1,13 @@ +--- +"@workflow/core": minor +"workflow": minor +"@workflow/errors": minor +"@workflow/world": minor +"@workflow/world-local": minor +"@workflow/world-postgres": minor +"@workflow/world-vercel": minor +"@workflow/cli": patch +"@workflow/web-shared": patch +--- + +Add queue-first experimental start-hook idempotency for Vercel worlds. diff --git a/docs/content/docs/v5/api-reference/workflow-api/start.mdx b/docs/content/docs/v5/api-reference/workflow-api/start.mdx index 2291fc16a3..a3aded8da8 100644 --- a/docs/content/docs/v5/api-reference/workflow-api/start.mdx +++ b/docs/content/docs/v5/api-reference/workflow-api/start.mdx @@ -56,10 +56,11 @@ Learn more about [`WorkflowReadableStreamOptions`](/docs/api-reference/workflow- * In v5, `start()` can also be called directly from a workflow function to spawn a child run or continue work in a new run. See [Workflow Composition](/cookbook/common-patterns/workflow-composition) and [Versioning](/docs/foundations/versioning). * This is different from calling workflow functions directly, which is the typical pattern in Next.js applications. * The function returns immediately after enqueuing the workflow - it doesn't wait for the workflow to complete. -* Each call to `start()` creates a new workflow run. If retried requests must route to one active workflow, have the workflow create a deterministic hook token and use [`getHookByToken()`](/docs/api-reference/workflow-api/get-hook-by-token) to reuse an already-registered active hook. The lookup is not atomic with `start()`, so concurrent callers can still create extra runs before the hook is registered; handle that race inside the workflow by checking `await hook.getConflict()` before duplicate-sensitive work — on a conflict it resolves with the run that owns the token, so the duplicate can return the active owner to the caller. If duplicates must be rejected before a workflow body runs, keep a durable request record until native atomic start-and-hook registration exists. See [Idempotency](/docs/foundations/idempotency#run-idempotency). +* Each call to `start()` creates a new workflow run unless you opt into an idempotency mechanism. The stable pattern is to have the workflow create a deterministic hook token and use [`getHookByToken()`](/docs/api-reference/workflow-api/get-hook-by-token) to reuse an already-registered active hook. The lookup is not atomic with `start()`, so concurrent callers can still create extra runs before the hook is registered; handle that race inside the workflow by checking `await hook.getConflict()` before duplicate-sensitive work. If duplicates must be rejected before duplicate workflow code can run, use `experimentalStartHook` in a supporting World. See [Idempotency](/docs/foundations/idempotency#run-idempotency). * All arguments must be [serializable](/docs/foundations/serialization). * When `deploymentId` is provided, the argument types and return type become `unknown` since there is no guarantee the workflow function's types will be consistent across different deployments. * `attributes` seeds plaintext run metadata as part of creation and requires a World implementing spec version 4 or later. Keys that start with `$` are reserved for framework and library code; framework-level callers can pass `allowReservedAttributes: true` to seed reserved keys, with the same semantics as the [`experimental_setAttributes`](/docs/api-reference/workflow/experimental-set-attributes) option of the same name. +* `experimentalStartHook` reserves a deterministic hook token as part of start admission in Worlds that support it. On Vercel, this uses queue-first admission, requires the target deployment to support queued start-hook handling, disables turbo mode for the first workflow invocation, rejects TTLs longer than 30 days or tokens larger than 255 bytes, and may throw [`HookConflictError`](/docs/api-reference/workflow-errors/hook-conflict-error) or [`WorkflowStartError`](/docs/api-reference/workflow-errors/workflow-start-error). See [Run idempotency](/docs/foundations/idempotency#atomic-start-hook-idempotency). If `start()` throws `'start' received an invalid workflow function. Ensure the Workflow Development Kit is configured correctly and the function includes a 'use workflow' directive.`, the passed function was not transformed as a workflow. The two most common causes are a missing `"use workflow"` directive or missing framework integration. See [start-invalid-workflow-function](/docs/errors/start-invalid-workflow-function). @@ -88,6 +89,29 @@ const run = await start(myWorkflow, ["arg1", "arg2"], { // [!code highlight] }); // [!code highlight] ``` +### With `experimentalStartHook` + +Use `experimentalStartHook` when duplicate starts must be rejected before duplicate workflow code can run. The workflow should still create the same hook token and check `hook.getConflict()` so the queued invocation can materialize the reserved hook and so non-supporting Worlds have the same in-workflow guard. + +```typescript +import { start } from "workflow/api"; +import { processOrder } from "./workflows/process-order"; + +try { + const run = await start(processOrder, ["order_123"], { + experimentalStartHook: { + token: "order:order_123", + experimental_ttl: "30 days", + }, + }); + console.log("Started run", run.runId); +} catch (error) { + // HookConflictError means another active or retained run owns the token. + // WorkflowStartError means the run may already be queued; inspect error.runId. + throw error; +} +``` + ### Using `deploymentId: "latest"` Set `deploymentId` to `"latest"` to automatically resolve the most recent deployment for the current environment. This is useful when you want to ensure a workflow run targets the latest deployed version of your application rather than the deployment that initiated the call. For when to use this and how it fits with default run pinning, see [Versioning](/docs/foundations/versioning). diff --git a/docs/content/docs/v5/api-reference/workflow-errors/index.mdx b/docs/content/docs/v5/api-reference/workflow-errors/index.mdx index eee43c82cf..ea3b45585d 100644 --- a/docs/content/docs/v5/api-reference/workflow-errors/index.mdx +++ b/docs/content/docs/v5/api-reference/workflow-errors/index.mdx @@ -51,6 +51,9 @@ All errors extend [`WorkflowError`](/docs/api-reference/workflow-errors/workflow Thrown when the workflow runtime encounters an execution error, such as serialization failures or timeouts. + + Thrown when start() cannot synchronously confirm queueing or start-hook admission. + Thrown when a workflow run has expired and can no longer be operated on. diff --git a/docs/content/docs/v5/api-reference/workflow-errors/workflow-start-error.mdx b/docs/content/docs/v5/api-reference/workflow-errors/workflow-start-error.mdx new file mode 100644 index 0000000000..873d222a53 --- /dev/null +++ b/docs/content/docs/v5/api-reference/workflow-errors/workflow-start-error.mdx @@ -0,0 +1,74 @@ +--- +title: WorkflowStartError +description: Thrown when start() cannot synchronously confirm queueing or start-hook admission. +type: reference +summary: Catch WorkflowStartError when a start call may have queued a run but could not confirm admission. +related: + - /docs/api-reference/workflow-api/start + - /docs/foundations/idempotency +--- + +`WorkflowStartError` is thrown by [`start()`](/docs/api-reference/workflow-api/start) when the SDK cannot synchronously confirm a run started successfully. It carries the generated `runId` so callers can inspect the run, poll later, or retry the start call. + +This is most important for `experimentalStartHook` on Vercel, where the queue is contacted before start-hook admission. If the queue accepts the message but admission cannot be confirmed, the original run may still start later. Retrying the same request can then return a [`HookConflictError`](/docs/api-reference/workflow-errors/hook-conflict-error) if the original run wins the token claim. + +```typescript lineNumbers +import { start } from "workflow/api"; +import { WorkflowStartError } from "workflow/errors"; +import { processOrder } from "./workflows/process-order"; + +try { + await start(processOrder, ["order_123"], { + experimentalStartHook: { + token: "order:order_123", + experimental_ttl: "30 days", + }, + }); +} catch (error) { + if (WorkflowStartError.is(error)) { // [!code highlight] + console.log("Run may already be queued:", error.runId); + } +} +``` + +## API Signature + +### Properties + + + +### Static Methods + +#### `WorkflowStartError.is(value)` + +Type-safe check for `WorkflowStartError` instances. + +```typescript +import { WorkflowStartError } from "workflow/errors" +declare const error: unknown; // @setup + +if (WorkflowStartError.is(error)) { + // error is typed as WorkflowStartError +} +``` diff --git a/docs/content/docs/v5/foundations/idempotency.mdx b/docs/content/docs/v5/foundations/idempotency.mdx index 190b5c9f81..8ab4bf7001 100644 --- a/docs/content/docs/v5/foundations/idempotency.mdx +++ b/docs/content/docs/v5/foundations/idempotency.mdx @@ -146,11 +146,80 @@ export async function POST(request: Request) { ``` -This avoids creating a new run only after the first run has registered its hook. Because `start()` returns before the run body executes and calls `createHook()`, two concurrent requests can both observe "no hook yet" and each call `start()`. The race is resolved inside the workflow body, where the losing run observes `getConflict()` resolving with the active owner and returns without doing duplicate-sensitive work — and the route detects it by comparing the resumed hook's `runId` against the run it just started, without waiting for either run to finish. A native API for atomically starting a run and registering a hook is in the works. Until then, model recovery inside the workflow by checking `hook.getConflict()`. +This avoids creating a new run only after the first run has registered its hook. Because `start()` returns before the run body executes and calls `createHook()`, two concurrent requests can both observe "no hook yet" and each call `start()`. The race is resolved inside the workflow body, where the losing run observes `getConflict()` resolving with the active owner and returns without doing duplicate-sensitive work — and the route detects it by comparing the resumed hook's `runId` against the run it just started, without waiting for either run to finish. If duplicates must be rejected before duplicate workflow code can run, use experimental start-hook admission in a supporting World. This is active-run coordination. When the workflow completes and disposes the hook, the token can be used again. If a duplicate request after completion must return the original result instead of starting fresh work, persist that completed result under the same domain key. +### Atomic start-hook idempotency + +`start()` can reserve the same hook token at admission time with the experimental `experimentalStartHook` option. In supporting Worlds, the run and hook-token claim are admitted atomically. On Vercel, the queue message is accepted first, turbo mode is disabled for that start, and the workflow-server claim decides whether this run owns the token. + +```typescript lineNumbers +import { start } from "workflow/api"; +import { HookConflictError, WorkflowStartError } from "workflow/errors"; +import { processOrder } from "./workflows/process-order"; + +export async function POST(request: Request) { + const { orderId } = await request.json(); + const token = `order:${orderId}`; + + try { + const run = await start(processOrder, [orderId], { + experimentalStartHook: { + token, + experimental_ttl: "30 days", + }, + }); + + return Response.json({ runId: run.runId, reused: false }); + } catch (error) { + if (HookConflictError.is(error)) { + return Response.json({ + runId: error.conflictingRunId, + reused: true, + }); + } + + if (WorkflowStartError.is(error)) { + // The queue may already have accepted this run. Keep error.runId so + // callers can poll it or retry the start call. + return Response.json( + { runId: error.runId, queued: error.queued }, + { status: 202 } + ); + } + + throw error; + } +} +``` + +The workflow should still create the same hook token and check `hook.getConflict()`. That materializes the reserved hook for the owning run and keeps the workflow body correct in Worlds that do not support atomic admission. + +```typescript lineNumbers +import { createHook } from "workflow"; + +type OrderRequest = { confirmed: boolean }; + +export async function processOrder(orderId: string) { + "use workflow"; + + using request = createHook({ + token: `order:${orderId}`, + }); + + const conflict = await request.getConflict(); + if (conflict) { + return { status: "duplicate" as const, runId: conflict.runId }; + } + + // Continue with the owning run. +} +``` + +On Vercel, `experimental_ttl` is capped at 30 days and hook tokens are capped at 255 bytes; values above either cap fail with a world contract error before queueing. Cross-deployment starts also fail before queueing unless the target deployment reports support for queued start-hook handling. The token claim is retained until at least the later of hook creation plus TTL and run completion, so duplicates can be rejected while the run is active and for the configured retention window. Cancelling a run before it has created its hook releases the claim immediately, so an explicit cancel-then-retry can reuse the token without waiting out the TTL. + ### Conflict-handling strategies Some workflow systems resolve duplicate IDs with a fixed, pre-declared policy — typically a static choice between rejecting the new execution, deferring to the existing one, or terminating it. Workflow has no policy enum. `hook.getConflict()` hands the duplicate run the conflicting `Run` itself, and the policy is ordinary code — including policies that inspect state before deciding, which static configuration can't express. diff --git a/packages/cli/src/lib/inspect/hydration.ts b/packages/cli/src/lib/inspect/hydration.ts index 499edaaac7..8a0162c2d4 100644 --- a/packages/cli/src/lib/inspect/hydration.ts +++ b/packages/cli/src/lib/inspect/hydration.ts @@ -159,6 +159,7 @@ const ERROR_REVIVER_KEYS = [ 'SyntaxError', 'TypeError', 'URIError', + 'WorkflowStartError', ] as const; /** diff --git a/packages/core/e2e/e2e.test.ts b/packages/core/e2e/e2e.test.ts index e463784277..c46f73961e 100644 --- a/packages/core/e2e/e2e.test.ts +++ b/packages/core/e2e/e2e.test.ts @@ -1880,6 +1880,38 @@ describe('e2e', () => { } ); + test.skipIf(!!process.env.WORKFLOW_VERCEL_ENV)( + 'experimentalStartHook rejects duplicate starts before hook materialization', + { timeout: 60_000 }, + async () => { + const token = Math.random().toString(36).slice(2); + const workflow = await e2e('experimentalStartHookWorkflow'); + const startOptions = { + experimentalStartHook: { + token, + experimental_ttl: 60_000, + }, + }; + + const run = await start(workflow, [token], startOptions); + + const duplicateError = await start(workflow, [token], startOptions).catch( + (error: unknown) => error + ); + expect(HookConflictError.is(duplicateError)).toBe(true); + assert(HookConflictError.is(duplicateError)); + expect(duplicateError.conflictingRunId).toBe(run.runId); + + const hook = await waitForHook(token, { runId: run.runId }); + await resumeHook(hook, { message: 'accepted' }); + + await expect(run.returnValue).resolves.toEqual({ + role: 'owner', + message: 'accepted', + }); + } + ); + test('hookGetConflictWorkflow - awaiting hook.getConflict() registers hook without payload', async () => { const token = Math.random().toString(36).slice(2); const customData = Math.random().toString(36).slice(2); diff --git a/packages/core/src/capabilities.test.ts b/packages/core/src/capabilities.test.ts index ad62b61331..c87b2b42ea 100644 --- a/packages/core/src/capabilities.test.ts +++ b/packages/core/src/capabilities.test.ts @@ -103,4 +103,31 @@ describe('getRunCapabilities', () => { expect(getRunCapabilities(version).framedByteStreams).toBe(true); }); }); + + describe('experimentalStartHookLoserAck', () => { + it('is false when version is missing or invalid', () => { + expect(getRunCapabilities(undefined).experimentalStartHookLoserAck).toBe( + false + ); + expect( + getRunCapabilities('not-a-version').experimentalStartHookLoserAck + ).toBe(false); + }); + + it('is false before the loser-ack cutoff', () => { + // beta.26 was published from main without loser-ack handling. + expect( + getRunCapabilities('5.0.0-beta.26').experimentalStartHookLoserAck + ).toBe(false); + }); + + it('is true at and after the loser-ack cutoff', () => { + expect( + getRunCapabilities('5.0.0-beta.27').experimentalStartHookLoserAck + ).toBe(true); + expect(getRunCapabilities('5.0.0').experimentalStartHookLoserAck).toBe( + true + ); + }); + }); }); diff --git a/packages/core/src/capabilities.ts b/packages/core/src/capabilities.ts index 4ef6b5a78e..394b79478a 100644 --- a/packages/core/src/capabilities.ts +++ b/packages/core/src/capabilities.ts @@ -31,6 +31,7 @@ * - `gzip` (gzip payload compression): added in `5.0.0-beta.18` * - `zstd` (zstd payload compression, preferred codec): added in `5.0.0-beta.18` * alongside gzip — they co-ship, so any run that can read one can read both. + * - `experimentalStartHookLoserAck` (queued start-hook loser handling): added in `5.0.0-beta.27` */ import semver from 'semver'; @@ -59,6 +60,13 @@ export interface RunCapabilities { * raw bytes (the legacy format) for compatibility with older runs. */ framedByteStreams: boolean; + + /** + * Whether the target workflow handler safely handles a queued start-hook + * run that LOST its token claim: turbo is disabled and `HookConflictError` + * during `run_started` is acknowledged without running user workflow code. + */ + experimentalStartHookLoserAck: boolean; } /** @@ -96,7 +104,18 @@ const CAPABILITY_VERSION_TABLE: ReadonlyArray<{ // to the next beta. A too-low cutoff makes new producers write framed bytes to // consumers that cannot unframe them (silent corruption); too-high merely // delays the optimization (safe). -}> = [{ capability: 'framedByteStreams', minVersion: '5.0.0-beta.15' }]; +}> = [ + { capability: 'framedByteStreams', minVersion: '5.0.0-beta.15' }, + // beta.26 was published from main WITHOUT this change, so the cutoff must + // point past it. TODO(release): bump again if another "Version Packages + // (beta)" PR merges before this change ships. A too-low cutoff runs user + // workflow code on a lost claim (unsafe); too-high merely rejects + // cross-deployment start-hook starts (safe). + { + capability: 'experimentalStartHookLoserAck', + minVersion: '5.0.0-beta.27', + }, +]; /** * The set of formats supported by all specVersion 2 runs, regardless of @@ -123,6 +142,7 @@ export function getRunCapabilities( return { supportedFormats: BASELINE_FORMATS, framedByteStreams: false, + experimentalStartHookLoserAck: false, }; } @@ -137,6 +157,7 @@ export function getRunCapabilities( const result: RunCapabilities = { supportedFormats: formats, framedByteStreams: false, + experimentalStartHookLoserAck: false, }; for (const { capability, minVersion } of CAPABILITY_VERSION_TABLE) { diff --git a/packages/core/src/runtime.test.ts b/packages/core/src/runtime.test.ts index 1cf2e3d0a0..fd5c954068 100644 --- a/packages/core/src/runtime.test.ts +++ b/packages/core/src/runtime.test.ts @@ -1,4 +1,5 @@ import { + HookConflictError, RUN_ERROR_CODES, ThrottleError, WorkflowWorldError, @@ -1479,13 +1480,14 @@ describe('workflowEntrypoint turbo mode', () => { return r; }${xform('workflow')}`; - async function makeRunInput(runId: string) { + async function makeRunInput(runId: string, extra?: Record) { return { input: await dehydrateWorkflowArguments([], runId, undefined, []), deploymentId: 'test-deployment', workflowName: 'workflow', specVersion: SPEC_VERSION_CURRENT, executionContext: {}, + ...extra, }; } @@ -1499,7 +1501,9 @@ describe('workflowEntrypoint turbo mode', () => { runId: string; attempt: number; source: string; + runInput?: Awaited>; runStartedGate?: Promise; + runStartedError?: unknown; }) { const { runId, attempt, source } = opts; const order = turboOrder; @@ -1530,6 +1534,7 @@ describe('workflowEntrypoint turbo mode', () => { const eventsCreate = vi.fn(async (_runId: string, data: any) => { if (data.eventType === 'run_started') { if (opts.runStartedGate) await opts.runStartedGate; + if (opts.runStartedError) throw opts.runStartedError; order.push('run_started_resolved'); return { run: runEntity, events: [] as Event[] }; } @@ -1573,7 +1578,7 @@ describe('workflowEntrypoint turbo mode', () => { { runId, requestedAt: new Date('2024-01-01T00:00:00.000Z'), - runInput: await makeRunInput(runId), + runInput: opts.runInput ?? (await makeRunInput(runId)), }, { requestId: 'req_turbo', @@ -1674,6 +1679,69 @@ describe('workflowEntrypoint turbo mode', () => { ); }); + it('does not turbo when runInput carries an experimental start hook', async () => { + let release!: () => void; + const gate = new Promise((r) => { + release = r; + }); + const runId = 'wrun_start_hook_no_turbo'; + const { handlerPromise, order, eventsCreate } = await driveTurbo({ + runId, + attempt: 1, + source: oneStepWorkflow, + runInput: await makeRunInput(runId, { + experimentalStartHook: { token: 'order:123', ttlSeconds: 60 }, + }), + runStartedGate: gate, + }); + + await vi.waitFor( + () => + expect( + eventsCreate.mock.calls.some( + (c) => (c[1] as any).eventType === 'run_started' + ) + ).toBe(true), + { timeout: 15_000 } + ); + expect(order).not.toContain('body'); + expect(order).not.toContain('run_started_resolved'); + + release(); + const res = await handlerPromise; + expect(res.status).toBe(204); + expect(order.indexOf('run_started_resolved')).toBeLessThan( + order.indexOf('body') + ); + const runStarted = eventsCreate.mock.calls.find( + (c) => (c[1] as any).eventType === 'run_started' + ); + expect((runStarted?.[2] as any)?.skipPreload).toBeUndefined(); + }); + + it('acknowledges queued start-hook losers without running workflow code', async () => { + const runId = 'wrun_start_hook_loser'; + const { handlerPromise, order, eventsCreate } = await driveTurbo({ + runId, + attempt: 1, + source: oneStepWorkflow, + runInput: await makeRunInput(runId, { + experimentalStartHook: { token: 'order:123', ttlSeconds: 60 }, + }), + runStartedError: new HookConflictError('order:123', 'wrun_winner'), + }); + + const res = await handlerPromise; + expect(res.status).toBe(204); + expect(order).not.toContain('body'); + expect(order).not.toContain('step_started_called'); + expect( + eventsCreate.mock.calls.some( + (c) => (c[1] as any).eventType === 'run_failed' + ) + ).toBe(false); + }); + it('asks the World to skip the run_started preload only under turbo', async () => { // The backgrounded run_started is used purely as a write barrier and its // preloaded events are never read (preloadedEvents is forced to []), so diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 7c533c242c..9381642aed 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -3,6 +3,7 @@ import { CorruptedEventLogError, EntityConflictError, FatalError, + HookConflictError, ReplayDivergenceError, RUN_ERROR_CODES, type RunErrorCode, @@ -539,6 +540,7 @@ export function workflowEntrypoint( const turbo = isTurboEnabled() && runInput !== undefined && + runInput.experimentalStartHook === undefined && metadata.attempt === 1 && incomingStepId === undefined && !replayDivergence; @@ -767,6 +769,8 @@ export function workflowEntrypoint( workflowName: runInput.workflowName, executionContext: runInput.executionContext, attributes: runInput.attributes, + experimentalStartHook: + runInput.experimentalStartHook, allowReservedAttributes: runInput.allowReservedAttributes, }, @@ -880,6 +884,19 @@ export function workflowEntrypoint( ); } } catch (err) { + if ( + runInput?.experimentalStartHook && + HookConflictError.is(err) + ) { + runtimeLogger.info( + 'Start-hook claim lost during setup, skipping queued run', + { + workflowRunId: runId, + message: err.message, + } + ); + return; + } // Run was concurrently completed/failed/cancelled if ( EntityConflictError.is(err) || diff --git a/packages/core/src/runtime/start.test.ts b/packages/core/src/runtime/start.test.ts index 82cdaa101f..c53c1cc547 100644 --- a/packages/core/src/runtime/start.test.ts +++ b/packages/core/src/runtime/start.test.ts @@ -1,9 +1,18 @@ -import { WorkflowRuntimeError, WorkflowWorldError } from '@workflow/errors'; +import { waitUntil } from '@vercel/functions'; +import { + EntityConflictError, + HookConflictError, + RUN_ERROR_CODES, + WorkflowRuntimeError, + WorkflowStartError, + WorkflowWorldError, +} from '@workflow/errors'; import { SPEC_VERSION_CURRENT, SPEC_VERSION_LEGACY, SPEC_VERSION_SUPPORTS_ATTRIBUTES, SPEC_VERSION_SUPPORTS_CBOR_QUEUE_TRANSPORT, + SPEC_VERSION_SUPPORTS_EVENT_SOURCING, } from '@workflow/world'; import { afterEach, @@ -88,6 +97,11 @@ describe('start', () => { describe('specVersion', () => { let mockEventsCreate: ReturnType; let mockQueue: ReturnType; + const queueFirstStartHookAdmission = { + mode: 'queue-first' as const, + maxTtlSeconds: 30 * 24 * 60 * 60, + maxTokenBytes: 255, + }; beforeEach(() => { mockEventsCreate = vi.fn().mockImplementation((runId) => { @@ -106,10 +120,21 @@ describe('start', () => { }); afterEach(() => { + vi.useRealTimers(); setWorld(undefined); vi.clearAllMocks(); }); + function setQueueFirstWorld(specVersion = SPEC_VERSION_CURRENT) { + setWorld({ + specVersion, + experimentalStartHookAdmission: queueFirstStartHookAdmission, + getDeploymentId: vi.fn().mockResolvedValue('deploy_123'), + events: { create: mockEventsCreate }, + queue: mockQueue, + } as any); + } + it('rejects worlds that do not declare a specVersion', async () => { const validWorkflow = Object.assign(() => Promise.resolve('result'), { workflowId: 'test-workflow', @@ -261,6 +286,492 @@ describe('start', () => { ); }); + it('seeds experimental start hook data and queues only after run creation succeeds', async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-01-01T00:00:00.000Z')); + + const validWorkflow = Object.assign(() => Promise.resolve('result'), { + workflowId: 'test-workflow', + }); + setWorld({ + specVersion: SPEC_VERSION_CURRENT, + experimentalStartHookAdmission: { mode: 'event-first' }, + getDeploymentId: vi.fn().mockResolvedValue('deploy_123'), + events: { create: mockEventsCreate }, + queue: mockQueue, + } as any); + + await start(validWorkflow, [], { + experimentalStartHook: { + token: 'order:123', + experimental_ttl: '30 days', + }, + }); + + expect(mockEventsCreate).toHaveBeenCalledWith( + expect.any(String), + expect.objectContaining({ + eventData: expect.objectContaining({ + experimentalStartHook: { + token: 'order:123', + ttlSeconds: 30 * 24 * 60 * 60, + }, + }), + }), + expect.anything() + ); + expect( + mockQueue.mock.calls[0]?.[1].runInput.experimentalStartHook + ).toEqual({ + token: 'order:123', + ttlSeconds: 30 * 24 * 60 * 60, + }); + expect(mockEventsCreate.mock.invocationCallOrder[0]).toBeLessThan( + mockQueue.mock.invocationCallOrder[0] + ); + + vi.useRealTimers(); + }); + + it('does not queue when experimental start hook admission conflicts', async () => { + const validWorkflow = Object.assign(() => Promise.resolve('result'), { + workflowId: 'test-workflow', + }); + mockEventsCreate.mockRejectedValueOnce( + new HookConflictError('order:123', 'wrun_conflicting') + ); + setWorld({ + specVersion: SPEC_VERSION_CURRENT, + experimentalStartHookAdmission: { mode: 'event-first' }, + getDeploymentId: vi.fn().mockResolvedValue('deploy_123'), + events: { create: mockEventsCreate }, + queue: mockQueue, + } as any); + + await expect( + start(validWorkflow, [], { + experimentalStartHook: { + token: 'order:123', + experimental_ttl: '30 days', + }, + }) + ).rejects.toThrow(HookConflictError); + expect(mockQueue).not.toHaveBeenCalled(); + }); + + it('rejects non-string experimental start hook tokens', async () => { + const validWorkflow = Object.assign(() => Promise.resolve('result'), { + workflowId: 'test-workflow', + }); + + await expect( + start(validWorkflow, [], { + experimentalStartHook: { + token: 123, + experimental_ttl: '30 days', + } as any, + }) + ).rejects.toThrow(/token must be a non-empty string/); + expect(mockEventsCreate).not.toHaveBeenCalled(); + expect(mockQueue).not.toHaveBeenCalled(); + }); + + it('cancels the admitted run when experimental start hook queueing fails', async () => { + const validWorkflow = Object.assign(() => Promise.resolve('result'), { + workflowId: 'test-workflow', + }); + const queueError = new Error('queue unavailable'); + mockQueue.mockRejectedValueOnce(queueError); + setWorld({ + specVersion: SPEC_VERSION_CURRENT, + experimentalStartHookAdmission: { mode: 'event-first' }, + getDeploymentId: vi.fn().mockResolvedValue('deploy_123'), + events: { create: mockEventsCreate }, + queue: mockQueue, + } as any); + + await expect( + start(validWorkflow, [], { + experimentalStartHook: { + token: 'order:123', + experimental_ttl: '30 days', + }, + }) + ).rejects.toThrow(queueError); + + expect(mockEventsCreate).toHaveBeenCalledTimes(2); + expect(mockEventsCreate.mock.calls[0]?.[1].eventType).toBe('run_created'); + expect(mockEventsCreate.mock.calls[1]?.[1]).toMatchObject({ + eventType: 'run_cancelled', + specVersion: SPEC_VERSION_CURRENT, + }); + }); + + it('does not queue when experimental start hook admission fails', async () => { + const validWorkflow = Object.assign(() => Promise.resolve('result'), { + workflowId: 'test-workflow', + }); + const admissionError = new WorkflowWorldError('response lost', { + status: 500, + }); + mockEventsCreate.mockRejectedValueOnce(admissionError); + setWorld({ + specVersion: SPEC_VERSION_CURRENT, + experimentalStartHookAdmission: { mode: 'event-first' }, + getDeploymentId: vi.fn().mockResolvedValue('deploy_123'), + events: { create: mockEventsCreate }, + queue: mockQueue, + } as any); + + await expect( + start(validWorkflow, [], { + experimentalStartHook: { + token: 'order:123', + experimental_ttl: '30 days', + }, + }) + ).rejects.toThrow(admissionError); + + expect(mockEventsCreate).toHaveBeenCalledTimes(1); + expect(mockQueue).not.toHaveBeenCalled(); + }); + + it('queues before admitting experimental start hooks in queue-first worlds', async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-01-01T00:00:00.000Z')); + + const validWorkflow = Object.assign(() => Promise.resolve('result'), { + workflowId: 'test-workflow', + }); + setQueueFirstWorld(); + + const run = await start(validWorkflow, [], { + experimentalStartHook: { + token: 'order:123', + experimental_ttl: '30 days', + }, + }); + + expect(run.runId).toBe(mockEventsCreate.mock.calls[0]?.[0]); + expect(mockQueue).toHaveBeenCalledTimes(1); + expect(mockEventsCreate).toHaveBeenCalledTimes(1); + expect( + mockQueue.mock.calls[0]?.[1].runInput.experimentalStartHook + ).toEqual({ + token: 'order:123', + ttlSeconds: 30 * 24 * 60 * 60, + }); + expect(mockEventsCreate.mock.calls[0]?.[1].eventData).toEqual( + expect.objectContaining({ + experimentalStartHook: { + token: 'order:123', + ttlSeconds: 30 * 24 * 60 * 60, + }, + }) + ); + expect(mockQueue.mock.calls[0]?.[2]).not.toHaveProperty('idempotencyKey'); + expect(mockQueue.mock.invocationCallOrder[0]).toBeLessThan( + mockEventsCreate.mock.invocationCallOrder[0] + ); + }); + + it('throws WorkflowStartError without admission when queue-first enqueue fails', async () => { + const validWorkflow = Object.assign(() => Promise.resolve('result'), { + workflowId: 'test-workflow', + }); + const queueError = new WorkflowWorldError('queue unavailable', { + status: 503, + }); + mockQueue.mockRejectedValueOnce(queueError); + setQueueFirstWorld(); + + const startPromise = start(validWorkflow, [], { + experimentalStartHook: { + token: 'order:123', + experimental_ttl: '30 days', + }, + }); + + await expect(startPromise).rejects.toBeInstanceOf(WorkflowStartError); + await expect(startPromise).rejects.toMatchObject({ + name: 'WorkflowStartError', + stage: 'queue', + queued: 'unknown', + retryable: true, + status: 503, + cause: queueError, + }); + expect(mockEventsCreate).not.toHaveBeenCalled(); + }); + + it('throws WorkflowStartError when queue-first admission cannot be confirmed', async () => { + const validWorkflow = Object.assign(() => Promise.resolve('result'), { + workflowId: 'test-workflow', + }); + const admissionError = new WorkflowWorldError('response lost', { + status: 500, + }); + mockEventsCreate.mockRejectedValueOnce(admissionError); + setQueueFirstWorld(); + + await expect( + start(validWorkflow, [], { + experimentalStartHook: { + token: 'order:123', + experimental_ttl: '30 days', + }, + }) + ).rejects.toMatchObject({ + name: 'WorkflowStartError', + stage: 'admission', + queued: true, + retryable: true, + status: 500, + cause: admissionError, + }); + expect(mockQueue).toHaveBeenCalledTimes(1); + }); + + it('wraps transient queue-first admission world errors after queueing', async () => { + const validWorkflow = Object.assign(() => Promise.resolve('result'), { + workflowId: 'test-workflow', + }); + const admissionError = new WorkflowWorldError('service unavailable', { + status: 503, + }); + mockEventsCreate.mockRejectedValueOnce(admissionError); + setQueueFirstWorld(); + + await expect( + start(validWorkflow, [], { + experimentalStartHook: { + token: 'order:123', + experimental_ttl: '30 days', + }, + }) + ).rejects.toMatchObject({ + name: 'WorkflowStartError', + stage: 'admission', + queued: true, + retryable: true, + status: 503, + cause: admissionError, + }); + expect(mockQueue).toHaveBeenCalledTimes(1); + }); + + it('wraps non-retryable queue-first admission world errors after queueing', async () => { + const validWorkflow = Object.assign(() => Promise.resolve('result'), { + workflowId: 'test-workflow', + }); + const admissionError = new WorkflowWorldError('forbidden', { + status: 403, + }); + mockEventsCreate.mockRejectedValueOnce(admissionError); + setQueueFirstWorld(); + + await expect( + start(validWorkflow, [], { + experimentalStartHook: { + token: 'order:123', + experimental_ttl: '30 days', + }, + }) + ).rejects.toMatchObject({ + name: 'WorkflowStartError', + stage: 'admission', + queued: true, + retryable: false, + status: 403, + cause: admissionError, + }); + expect(mockQueue).toHaveBeenCalledTimes(1); + }); + + it('wraps unknown queue-first admission failures after queueing', async () => { + const validWorkflow = Object.assign(() => Promise.resolve('result'), { + workflowId: 'test-workflow', + }); + const admissionError = new TypeError('socket closed'); + mockEventsCreate.mockRejectedValueOnce(admissionError); + setQueueFirstWorld(); + const waitUntilCalls = vi.mocked(waitUntil).mock.calls.length; + + await expect( + start(validWorkflow, [], { + experimentalStartHook: { + token: 'order:123', + experimental_ttl: '30 days', + }, + }) + ).rejects.toMatchObject({ + name: 'WorkflowStartError', + stage: 'admission', + queued: true, + retryable: false, + cause: admissionError, + }); + await vi.waitFor(() => + expect(vi.mocked(waitUntil).mock.calls.length).toBeGreaterThan( + waitUntilCalls + ) + ); + }); + + it('still surfaces HookConflictError after queue-first enqueue', async () => { + const validWorkflow = Object.assign(() => Promise.resolve('result'), { + workflowId: 'test-workflow', + }); + mockEventsCreate.mockRejectedValueOnce( + new HookConflictError('order:123', 'wrun_conflicting') + ); + setQueueFirstWorld(); + + await expect( + start(validWorkflow, [], { + experimentalStartHook: { + token: 'order:123', + experimental_ttl: '30 days', + }, + }) + ).rejects.toThrow(HookConflictError); + expect(mockQueue).toHaveBeenCalledTimes(1); + expect(mockQueue.mock.invocationCallOrder[0]).toBeLessThan( + mockEventsCreate.mock.invocationCallOrder[0] + ); + }); + + it('rejects experimental start hook TTLs above the world cap before queueing', async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-01-01T00:00:00.000Z')); + + const validWorkflow = Object.assign(() => Promise.resolve('result'), { + workflowId: 'test-workflow', + }); + setQueueFirstWorld(); + + await expect( + start(validWorkflow, [], { + experimentalStartHook: { + token: 'order:123', + experimental_ttl: '31 days', + }, + }) + ).rejects.toMatchObject({ + name: 'WorkflowWorldError', + code: RUN_ERROR_CODES.WORLD_CONTRACT_ERROR, + }); + expect(mockEventsCreate).not.toHaveBeenCalled(); + expect(mockQueue).not.toHaveBeenCalled(); + }); + + it('rejects queue-first start hooks with oversized tokens before queueing', async () => { + const validWorkflow = Object.assign(() => Promise.resolve('result'), { + workflowId: 'test-workflow', + }); + setQueueFirstWorld(); + + await expect( + start(validWorkflow, [], { + experimentalStartHook: { + token: 'x'.repeat(256), + experimental_ttl: '30 days', + }, + }) + ).rejects.toMatchObject({ + name: 'WorkflowWorldError', + code: RUN_ERROR_CODES.WORLD_CONTRACT_ERROR, + }); + expect(mockEventsCreate).not.toHaveBeenCalled(); + expect(mockQueue).not.toHaveBeenCalled(); + }); + + it('rejects queue-first start hooks without runInput queue transport before queueing', async () => { + const validWorkflow = Object.assign(() => Promise.resolve('result'), { + workflowId: 'test-workflow', + }); + setQueueFirstWorld(); + + // Worlds must match the runtime spec version, so a pre-CBOR-transport + // run can only come from an explicit opts.specVersion override. + await expect( + start(validWorkflow, [], { + specVersion: SPEC_VERSION_SUPPORTS_EVENT_SOURCING, + experimentalStartHook: { + token: 'order:123', + experimental_ttl: '30 days', + }, + }) + ).rejects.toMatchObject({ + name: 'WorkflowWorldError', + code: RUN_ERROR_CODES.WORLD_CONTRACT_ERROR, + }); + expect(mockEventsCreate).not.toHaveBeenCalled(); + expect(mockQueue).not.toHaveBeenCalled(); + }); + + it('rejects queue-first start hooks when the target deployment lacks runtime support', async () => { + const validWorkflow = Object.assign(() => Promise.resolve('result'), { + workflowId: 'test-workflow', + }); + setWorld({ + specVersion: SPEC_VERSION_CURRENT, + experimentalStartHookAdmission: queueFirstStartHookAdmission, + getDeploymentId: vi.fn().mockResolvedValue('deploy_current'), + events: { create: mockEventsCreate }, + queue: mockQueue, + streams: { + get: vi.fn().mockResolvedValue( + new Response( + JSON.stringify({ + healthy: true, + workflowCoreVersion: '5.0.0-beta.25', + }) + ).body + ), + }, + } as any); + + await expect( + start(validWorkflow, [], { + deploymentId: 'deploy_old', + experimentalStartHook: { + token: 'order:123', + experimental_ttl: '30 days', + }, + }) + ).rejects.toMatchObject({ + name: 'WorkflowWorldError', + code: RUN_ERROR_CODES.WORLD_CONTRACT_ERROR, + }); + expect(mockEventsCreate).not.toHaveBeenCalled(); + expect(mockQueue).toHaveBeenCalledTimes(1); + expect(mockQueue.mock.calls[0]?.[0]).toContain('health_check'); + }); + + it('rejects experimental start hooks when the world has not opted in', async () => { + const validWorkflow = Object.assign(() => Promise.resolve('result'), { + workflowId: 'test-workflow', + }); + setWorld({ + specVersion: SPEC_VERSION_CURRENT, + getDeploymentId: vi.fn().mockResolvedValue('deploy_123'), + events: { create: mockEventsCreate }, + queue: mockQueue, + } as any); + + await expect( + start(validWorkflow, [], { + experimentalStartHook: { + token: 'order:123', + experimental_ttl: '30 days', + }, + }) + ).rejects.toThrow(/supports experimental start-hook admission/); + expect(mockEventsCreate).not.toHaveBeenCalled(); + expect(mockQueue).not.toHaveBeenCalled(); + }); + it('rejects initial attributes for pre-v4 runs', async () => { const validWorkflow = Object.assign(() => Promise.resolve('result'), { workflowId: 'test-workflow', diff --git a/packages/core/src/runtime/start.ts b/packages/core/src/runtime/start.ts index 3eed21d386..236ab3cd87 100644 --- a/packages/core/src/runtime/start.ts +++ b/packages/core/src/runtime/start.ts @@ -1,4 +1,12 @@ -import { EntityConflictError, WorkflowRuntimeError } from '@workflow/errors'; +import { + EntityConflictError, + HookConflictError, + RUN_ERROR_CODES, + WorkflowRuntimeError, + WorkflowStartError, + WorkflowWorldError, +} from '@workflow/errors'; +import { parseDurationToDate } from '@workflow/utils'; import { workflowDisplayName } from '@workflow/utils/parse-name'; import type { WorkflowInvokePayload, World } from '@workflow/world'; import { @@ -7,6 +15,7 @@ import { SPEC_VERSION_SUPPORTS_CBOR_QUEUE_TRANSPORT, SPEC_VERSION_SUPPORTS_COMPRESSION, } from '@workflow/world'; +import type { StringValue } from 'ms'; import { monotonicFactory } from 'ulid'; import { normalizeAttributeChanges } from '../attribute-changes.js'; import { getRunCapabilities } from '../capabilities.js'; @@ -37,6 +46,7 @@ import { assertWorldSupportsRuntimeProtocol } from './world-compatibility.js'; * deployments don't recognize the health check at all. */ const CROSS_DEPLOYMENT_CAPABILITY_PROBE_TIMEOUT_MS = 2_000; +const textEncoder = new TextEncoder(); /** ULID generator for client-side runId generation */ const ulid = monotonicFactory(); @@ -57,6 +67,85 @@ export function _resetLatestNoOpWarnForTests(): void { hasWarnedLatestNoOp = false; } +interface ExperimentalStartHookOptions { + token: string; + experimental_ttl: StringValue | number; +} + +function normalizeExperimentalStartHook(hook: ExperimentalStartHookOptions): { + token: string; + ttlSeconds: number; +} { + if (typeof hook.token !== 'string' || hook.token.length === 0) { + throw new WorkflowRuntimeError( + 'experimentalStartHook.token must be a non-empty string.' + ); + } + + if (hook.experimental_ttl === undefined) { + throw new WorkflowRuntimeError( + 'experimentalStartHook.experimental_ttl must be provided.' + ); + } + + const ttlMilliseconds = + parseDurationToDate(hook.experimental_ttl).getTime() - Date.now(); + if (!Number.isFinite(ttlMilliseconds) || ttlMilliseconds <= 0) { + throw new WorkflowRuntimeError( + 'experimentalStartHook.experimental_ttl must be a positive duration.' + ); + } + + const ttlSeconds = Math.max(1, Math.floor(ttlMilliseconds / 1000)); + return { token: hook.token, ttlSeconds }; +} + +async function hasDurableRunCreatedEvent( + world: World, + runId: string +): Promise { + const page = await world.events.list({ + runId, + pagination: { limit: 1, sortOrder: 'asc' }, + resolveData: 'none', + }); + return page.data.some((event) => event.eventType === 'run_created'); +} + +function getRunCreatedResult( + result: Awaited>, + expectedRunId?: string +) { + if (!result.run) { + throw new WorkflowRuntimeError( + "Missing 'run' in server response for 'run_created' event" + ); + } + if (expectedRunId !== undefined && result.run.runId !== expectedRunId) { + throw new WorkflowRuntimeError( + `Server returned different runId than requested: expected ${expectedRunId}, got ${result.run.runId}` + ); + } + return result.run; +} + +function isWorkflowWorldError(error: unknown): error is WorkflowWorldError { + // `.is()` alone misses subclasses (their `name` differs, e.g. + // ThrottleError); `instanceof` alone misses cross-realm base instances. + // World code runs in the host realm, so together they cover both. + return error instanceof WorkflowWorldError || WorkflowWorldError.is(error); +} + +function getWorkflowWorldErrorDetails(error: unknown) { + if (!isWorkflowWorldError(error)) return {}; + return { + status: error.status, + url: error.url, + code: error.code, + retryAfter: error.retryAfter, + }; +} + export interface StartOptionsBase { /** * The world to use for the workflow run creation, @@ -89,6 +178,16 @@ export interface StartOptionsBase { * the `experimental_setAttributes` option of the same name. */ allowReservedAttributes?: boolean; + + /** + * EXPERIMENTAL: atomically reserve a hook token as part of run admission. + * + * If another active or retained run already owns the token, `start()` throws + * `HookConflictError`. On Vercel the queue may already have accepted the + * losing run before the conflict is observed; that queued invocation exits + * without running user workflow code. + */ + experimentalStartHook?: ExperimentalStartHookOptions; } export interface StartOptionsWithDeploymentId extends StartOptionsBase { @@ -250,12 +349,15 @@ export async function start( // mocks) can't service health checks, so we skip the probe for them. let framedByteStreams: boolean; let targetSupportsCompression: boolean; + let targetSupportsStartHookAdmission: boolean; if (deploymentId === currentDeploymentId) { framedByteStreams = true; targetSupportsCompression = true; + targetSupportsStartHookAdmission = true; } else if (typeof world.streams?.get !== 'function') { framedByteStreams = false; targetSupportsCompression = false; + targetSupportsStartHookAdmission = false; } else { const probe = await healthCheck(world, 'workflow', { deploymentId, @@ -266,9 +368,29 @@ export async function start( targetSupportsCompression = capabilities.supportedFormats.has( SerializationFormat.GZIP ); + targetSupportsStartHookAdmission = + capabilities.experimentalStartHookLoserAck; } const ops: Promise[] = []; + let opsFlushScheduled = false; + const scheduleOpsFlush = () => { + if (opsFlushScheduled) return; + opsFlushScheduled = true; + // These argument-stream ops are flushed in the background; the promise + // handed to waitUntil must never reject (an unconsumed waitUntil + // rejection crashes the process as unhandledRejection), so unexpected + // failures are logged instead. + safeWaitUntil(Promise.all(ops), (err) => { + runtimeLogger.warn( + 'Background flush of workflow argument streams failed', + { + workflowRunId: runId, + error: err instanceof Error ? err.message : String(err), + } + ); + }); + }; // Generate runId client-side so we have it before serialization // (required for future E2E encryption where runId is part of the encryption context) @@ -281,6 +403,53 @@ export async function start( // itself has already been checked against this runtime's spec version. const specVersion = opts.specVersion ?? world.specVersion; const v1Compat = isLegacySpecVersion(specVersion); + const experimentalStartHook = opts.experimentalStartHook + ? normalizeExperimentalStartHook(opts.experimentalStartHook) + : undefined; + const startHookAdmission = world.experimentalStartHookAdmission; + if (experimentalStartHook) { + if (v1Compat) { + throw new WorkflowRuntimeError( + 'experimentalStartHook requires an event-sourced World.' + ); + } + if (startHookAdmission === undefined) { + throw new WorkflowRuntimeError( + 'experimentalStartHook requires a World that supports experimental start-hook admission.' + ); + } + if (startHookAdmission.mode === 'queue-first') { + const contractError = (message: string) => + new WorkflowWorldError(message, { + code: RUN_ERROR_CODES.WORLD_CONTRACT_ERROR, + }); + if (!targetSupportsStartHookAdmission) { + throw contractError( + 'experimentalStartHook requires a target deployment that supports queue-first start-hook admission.' + ); + } + if ( + experimentalStartHook.ttlSeconds > startHookAdmission.maxTtlSeconds + ) { + throw contractError( + `experimentalStartHook.experimental_ttl exceeds this World's maximum of ${startHookAdmission.maxTtlSeconds} seconds.` + ); + } + if ( + textEncoder.encode(experimentalStartHook.token).byteLength > + startHookAdmission.maxTokenBytes + ) { + throw contractError( + `experimentalStartHook.token exceeds this World's maximum of ${startHookAdmission.maxTokenBytes} bytes.` + ); + } + if (specVersion < SPEC_VERSION_SUPPORTS_CBOR_QUEUE_TRANSPORT) { + throw contractError( + 'experimentalStartHook requires a spec version that supports runInput queue transport in queue-first Worlds.' + ); + } + } + } const allowReservedAttributes = opts.allowReservedAttributes === true; let attributes: Record | undefined; if (opts.attributes && Object.keys(opts.attributes).length > 0) { @@ -318,6 +487,9 @@ export async function start( : {}), } : {}; + const startHookSeed = experimentalStartHook + ? { experimentalStartHook } + : {}; // Resolve encryption key for the new run. The runId has already been // generated above (client-generated ULID) and will be used for both @@ -358,115 +530,185 @@ export async function start( features: { encryption: !!encryptionKey }, }; - // Call events.create (run_created) and queue in parallel. - // If events.create fails with 429/5xx, the run was still accepted - // via the queue and creation will be re-tried async by the runtime. - const [runCreatedResult, queueResult] = await Promise.allSettled([ - world.events.create( - runId, - { - eventType: 'run_created', - specVersion, - eventData: { - deploymentId: deploymentId, - workflowName: workflowName, - input: workflowArguments, - executionContext, - ...attributeSeed, - }, - }, - { v1Compat } - ), - world.queue( - getWorkflowQueueName(workflowName), - { - runId, - traceCarrier, - ...(specVersion >= SPEC_VERSION_SUPPORTS_CBOR_QUEUE_TRANSPORT - ? { - runInput: { - input: workflowArguments, - deploymentId, - workflowName, - specVersion, - executionContext, - ...attributeSeed, - }, - } - : {}), - } satisfies WorkflowInvokePayload, - { - deploymentId, - specVersion, - } - ), - ]); + const runCreatedEvent = { + eventType: 'run_created' as const, + specVersion, + eventData: { + deploymentId: deploymentId, + workflowName: workflowName, + input: workflowArguments, + executionContext, + ...attributeSeed, + ...startHookSeed, + }, + }; + const queuePayload = { + runId, + traceCarrier, + ...(specVersion >= SPEC_VERSION_SUPPORTS_CBOR_QUEUE_TRANSPORT + ? { + runInput: { + input: workflowArguments, + deploymentId, + workflowName, + specVersion, + executionContext, + ...attributeSeed, + ...startHookSeed, + }, + } + : {}), + } satisfies WorkflowInvokePayload; - // Queue failure is always fatal — the run was not enqueued - if (queueResult.status === 'rejected') { - throw queueResult.reason; - } + const createRunEvent = () => + world.events.create(runId, runCreatedEvent, { v1Compat }); + const enqueueRun = () => + world.queue(getWorkflowQueueName(workflowName), queuePayload, { + deploymentId, + specVersion, + }); - // Handle events.create result let resilientStart = false; - if (runCreatedResult.status === 'rejected') { - const err = runCreatedResult.reason; - if (EntityConflictError.is(err)) { - // 409: The run already exists. This can happen in extreme cases where - // the run creation call gets a cold start or other slowdown, and the queue - // + run_started call completes faster. We expect this to be <=1% of cases. - // In this case, we can safely return. - } else if (isRetryableWorldError(err)) { - // 429 (ThrottleError), 5xx, and transient transport failures - // (TRANSPORT/TIMEOUT) are retryable — the run was accepted via the - // queue and creation will be re-tried by the runtime when it calls - // run_started. - resilientStart = true; - runtimeLogger.warn( - 'Run creation event failed, but the run was accepted via the queue. ' + - 'The run_created event will be re-tried async by the runtime.', - { workflowRunId: runId, error: err.message } - ); + let createdRunForSpan: + | NonNullable>['run']> + | undefined; + if (experimentalStartHook) { + if (startHookAdmission?.mode === 'queue-first') { + try { + await enqueueRun(); + } catch (error) { + scheduleOpsFlush(); + throw new WorkflowStartError( + `Workflow start failed while enqueueing run "${runId}". The queue may have accepted the message; inspect this run ID or retry the start call.`, + { + runId, + stage: 'queue', + retryable: isRetryableWorldError(error), + cause: error, + ...getWorkflowWorldErrorDetails(error), + } + ); + } + + try { + createdRunForSpan = getRunCreatedResult( + await createRunEvent(), + runId + ); + } catch (error) { + scheduleOpsFlush(); + if (EntityConflictError.is(error)) { + if (!(await hasDurableRunCreatedEvent(world, runId))) { + throw error; + } + resilientStart = true; + runtimeLogger.warn( + 'Run creation conflicted after start-hook queue acceptance, but the run_created event is durable.', + { workflowRunId: runId, error: error.message } + ); + } else if (!HookConflictError.is(error)) { + throw new WorkflowStartError( + `Workflow run "${runId}" was queued, but start-hook admission could not be confirmed. Inspect this run ID or retry the start call; a retry may conflict if the queued run starts successfully.`, + { + runId, + stage: 'admission', + retryable: isRetryableWorldError(error), + cause: error, + ...getWorkflowWorldErrorDetails(error), + } + ); + } else { + throw error; + } + } } else { - throw err; + // Event-first: admission (run creation + token claim) must succeed + // before the run is queued, so a losing start never enqueues work. + // Any creation failure — including HookConflictError — fails the + // start; nothing was queued, so the caller can simply retry. + createdRunForSpan = getRunCreatedResult( + await createRunEvent(), + runId + ); + + try { + await enqueueRun(); + } catch (error) { + // Undo admission so the token is not fenced by a run that will + // never execute. Cancellation releases unmaterialized start-hook + // claims in event-first Worlds. + try { + await world.events.create( + runId, + { + eventType: 'run_cancelled', + specVersion, + }, + { v1Compat } + ); + } catch (cleanupError) { + runtimeLogger.warn( + 'Run queueing failed after start-hook admission, and cleanup also failed.', + { + workflowRunId: runId, + error: (cleanupError as Error).message, + } + ); + } + throw error; + } } } else { - const result = runCreatedResult.value; - // Assert that the run was created - if (!result.run) { - throw new WorkflowRuntimeError( - "Missing 'run' in server response for 'run_created' event" - ); + // Call events.create (run_created) and queue in parallel. + // If events.create fails with 429/5xx, the run was still accepted + // via the queue and creation will be re-tried async by the runtime. + const [runCreatedResult, queueResult] = await Promise.allSettled([ + createRunEvent(), + enqueueRun(), + ]); + + // Queue failure is always fatal — the run was not enqueued + if (queueResult.status === 'rejected') { + throw queueResult.reason; } - // Verify server accepted our runId - if (!v1Compat && result.run.runId !== runId) { - throw new WorkflowRuntimeError( - `Server returned different runId than requested: expected ${runId}, got ${result.run.runId}` + // Handle events.create result + if (runCreatedResult.status === 'rejected') { + const err = runCreatedResult.reason; + if (EntityConflictError.is(err)) { + // 409: The run already exists. This can happen in extreme cases where + // the run creation call gets a cold start or other slowdown, and the queue + // + run_started call completes faster. We expect this to be <=1% of cases. + // In this case, we can safely return. + } else if (isRetryableWorldError(err)) { + // 429 (ThrottleError), 5xx, and transient transport failures + // (TRANSPORT/TIMEOUT) are retryable — the run was accepted via + // the queue and creation will be re-tried by the runtime when it + // calls run_started. + resilientStart = true; + runtimeLogger.warn( + 'Run creation event failed, but the run was accepted via the queue. ' + + 'The run_created event will be re-tried async by the runtime.', + { workflowRunId: runId, error: err.message } + ); + } else { + throw err; + } + } else { + createdRunForSpan = getRunCreatedResult( + runCreatedResult.value, + v1Compat ? undefined : runId ); } } - // These argument-stream ops are flushed in the background; the promise - // handed to waitUntil must never reject (an unconsumed waitUntil - // rejection crashes the process as unhandledRejection), so unexpected - // failures are logged instead. - safeWaitUntil(Promise.all(ops), (err) => { - runtimeLogger.warn( - 'Background flush of workflow argument streams failed', - { - workflowRunId: runId, - error: err instanceof Error ? err.message : String(err), - } - ); - }); + scheduleOpsFlush(); span?.setAttributes({ ...Attribute.WorkflowRunId(runId), ...Attribute.DeploymentId(deploymentId), - ...(runCreatedResult.status === 'fulfilled' && - runCreatedResult.value.run - ? Attribute.WorkflowRunStatus(runCreatedResult.value.run.status) + ...(createdRunForSpan + ? Attribute.WorkflowRunStatus(createdRunForSpan.status) : {}), }); diff --git a/packages/core/src/serialization.test.ts b/packages/core/src/serialization.test.ts index 5c21c55c3c..aa78fd7629 100644 --- a/packages/core/src/serialization.test.ts +++ b/packages/core/src/serialization.test.ts @@ -5,6 +5,7 @@ import { HookConflictError, RetryableError, RuntimeDecryptionError, + WorkflowStartError, } from '@workflow/errors'; import { WORKFLOW_DESERIALIZE, WORKFLOW_SERIALIZE } from '@workflow/serde'; import { beforeAll, describe, expect, it, vi } from 'vitest'; @@ -2667,7 +2668,7 @@ describe('step function serialization', () => { const stepId = 'step//workflows/test.ts//addNumbers'; // Create a VM context like the workflow runner does - const { context, globalThis: vmGlobalThis } = createContext({ + const { globalThis: vmGlobalThis } = createContext({ seed: 'test', fixedTimestamp: 1714857600000, }); @@ -2729,7 +2730,7 @@ describe('step function serialization', () => { const stepId = 'step//workflows/test.ts//missingUseStep'; // Create a VM context WITHOUT setting up WORKFLOW_USE_STEP - const { context, globalThis: vmGlobalThis } = createContext({ + const { globalThis: vmGlobalThis } = createContext({ seed: 'test', fixedTimestamp: 1714857600000, }); @@ -3984,6 +3985,46 @@ describe('Workflow error serialization', () => { expect(str).not.toContain('Instance'); }); + it('should round-trip WorkflowStartError preserving recovery fields', async () => { + const error = new WorkflowStartError('queued but not admitted', { + runId: 'wrun_queued', + stage: 'admission', + retryable: true, + status: 408, + retryAfter: 2, + }); + const hydrated = (await roundTrip(error)) as WorkflowStartError; + + expect(hydrated).toBeInstanceOf(WorkflowStartError); + expect(WorkflowStartError.is(hydrated)).toBe(true); + expect(hydrated.runId).toBe('wrun_queued'); + expect(hydrated.stage).toBe('admission'); + expect(hydrated.queued).toBe(true); + expect(hydrated.retryable).toBe(true); + expect(hydrated.status).toBe(408); + expect(hydrated.retryAfter).toBe(2); + }); + + it('should serialize WorkflowStartError using its dedicated reducer key', async () => { + const error = new WorkflowStartError('queue response lost', { + runId: 'wrun_unknown', + stage: 'queue', + retryable: true, + }); + const serialized = await dehydrateStepReturnValue( + error, + mockRunId, + noEncryptionKey + ); + const str = new TextDecoder().decode( + (serialized as Uint8Array).subarray(4) + ); + + expect(str).toContain('["WorkflowStartError",'); + expect(str).not.toContain('["Error",'); + expect(str).not.toContain('Instance'); + }); + it('should preserve cause on FatalError when present', async () => { const cause = new Error('underlying issue'); const error = new FatalError('fatal with cause'); @@ -5561,7 +5602,7 @@ describe('isEncrypted', () => { // ============================================================================ describe('AbortController serialization', () => { - const { context, globalThis: vmGlobalThis } = createContext({ + const { globalThis: vmGlobalThis } = createContext({ seed: 'test-abort-serde', fixedTimestamp: 1714857600000, }); diff --git a/packages/core/src/serialization/reducers/common.ts b/packages/core/src/serialization/reducers/common.ts index 6ad8b773ff..e46dec7679 100644 --- a/packages/core/src/serialization/reducers/common.ts +++ b/packages/core/src/serialization/reducers/common.ts @@ -15,6 +15,7 @@ import { HookConflictError, RetryableError, RuntimeDecryptionError, + WorkflowStartError, } from '@workflow/errors'; import type { Reducers, Revivers, SerializableSpecial } from '../types.js'; @@ -228,6 +229,24 @@ export function getCommonReducers( } return reduced; }, + WorkflowStartError: (value) => { + const base = reduceNamedErrorSubclassBase('WorkflowStartError', value); + if (!base) return false; + const error = value as WorkflowStartError; + const reduced: SerializableSpecial['WorkflowStartError'] = { + ...base, + runId: error.runId, + stage: error.stage, + retryable: error.retryable, + }; + if (error.status !== undefined) reduced.status = error.status; + if (error.url !== undefined) reduced.url = error.url; + if (error.code !== undefined) reduced.code = error.code; + if (error.retryAfter !== undefined) { + reduced.retryAfter = error.retryAfter; + } + return reduced; + }, RangeError: makeErrorSubclassReducer('RangeError'), ReferenceError: makeErrorSubclassReducer('ReferenceError'), // RetryableError carries an extra `retryAfter` Date that we serialize as @@ -409,6 +428,24 @@ export function getCommonRevivers( } return error; }, + WorkflowStartError: (value) => { + const Ctor = + ((global as Record)[ + Symbol.for('@workflow/errors//WorkflowStartError') + ] as typeof WorkflowStartError | undefined) ?? WorkflowStartError; + const error = new Ctor(value.message, { + runId: value.runId, + stage: value.stage, + retryable: value.retryable, + status: value.status, + url: value.url, + code: value.code, + retryAfter: value.retryAfter, + ...('cause' in value ? { cause: value.cause } : {}), + }); + if (value.stack !== undefined) error.stack = value.stack; + return error; + }, RangeError: makeErrorSubclassReviver(global, 'RangeError'), ReferenceError: makeErrorSubclassReviver(global, 'ReferenceError'), RetryableError: (value) => { diff --git a/packages/core/src/serialization/types.ts b/packages/core/src/serialization/types.ts index 3668c05148..f4f6bd96f2 100644 --- a/packages/core/src/serialization/types.ts +++ b/packages/core/src/serialization/types.ts @@ -83,6 +83,20 @@ export interface SerializableSpecial { // TODO: Make this required when HookConflictError.conflictingRunId is required. conflictingRunId?: string; }; + WorkflowStartError: { + message: string; + stack?: string; + cause?: unknown; + runId: string; + // `queued` is derived from `stage` by the constructor, so only `stage` + // is serialized. + stage: 'queue' | 'admission'; + retryable: boolean; + status?: number; + url?: string; + code?: string; + retryAfter?: number; + }; Int8Array: string; // base64 string Int16Array: string; // base64 string Int32Array: string; // base64 string diff --git a/packages/errors/src/index.ts b/packages/errors/src/index.ts index 190d5efcbe..097c24deb4 100644 --- a/packages/errors/src/index.ts +++ b/packages/errors/src/index.ts @@ -54,9 +54,9 @@ function appendFramedDetails( const head = isLast ? '╰▶ ' : '├▶ '; const cont = isLast ? ' ' : '│ '; const text = `${detail.label}: ${detail.value}`; - text - .split('\n') - .forEach((line, i) => lines.push(`${i === 0 ? head : cont}${line}`)); + text.split('\n').forEach((line, i) => { + lines.push(`${i === 0 ? head : cont}${line}`); + }); }); return lines.join('\n'); } @@ -187,6 +187,48 @@ export class WorkflowWorldError extends WorkflowError { } } +interface WorkflowStartErrorOptions { + runId: string; + /** + * The phase that failed. `queue` means the enqueue call itself failed (the + * queue may or may not have accepted the message); `admission` means the + * message was queued but admission could not be confirmed. + */ + stage: 'queue' | 'admission'; + retryable: boolean; + status?: number; + url?: string; + code?: string; + retryAfter?: number; + cause?: unknown; +} + +/** + * Thrown when `start()` cannot synchronously confirm whether a workflow run was + * fully admitted. The error carries the client-generated run ID so callers can + * inspect or retry the run when the queue may already have accepted it. + */ +export class WorkflowStartError extends WorkflowWorldError { + readonly runId: string; + readonly stage: 'queue' | 'admission'; + /** Whether the queue accepted the message — derived from `stage`. */ + readonly queued: true | 'unknown'; + readonly retryable: boolean; + + constructor(message: string, options: WorkflowStartErrorOptions) { + super(message, options); + this.name = 'WorkflowStartError'; + this.runId = options.runId; + this.stage = options.stage; + this.queued = options.stage === 'queue' ? 'unknown' : true; + this.retryable = options.retryable; + } + + static is(value: unknown): value is WorkflowStartError { + return isError(value) && value.name === 'WorkflowStartError'; + } +} + /** * Thrown when a workflow run fails during execution. * @@ -914,7 +956,8 @@ export { RUN_ERROR_CODES, type RunErrorCode } from './error-codes.js'; // Cross-realm class registration // --------------------------------------------------------------------------- // -// `FatalError`, `RetryableError`, and `HookConflictError` are not built-ins, so different realms +// `FatalError`, `RetryableError`, `HookConflictError`, and +// `WorkflowStartError` are not built-ins, so different realms // (e.g. the workflow VM context vs. the host context that runs the queue // handler) bundle and load their own copies of this module — meaning each // realm has its own distinct class identity. Cross-realm `instanceof` fails @@ -933,6 +976,9 @@ const RETRYABLE_ERROR_KEY = Symbol.for('@workflow/errors//RetryableError'); const HOOK_CONFLICT_ERROR_KEY = Symbol.for( '@workflow/errors//HookConflictError' ); +const WORKFLOW_START_ERROR_KEY = Symbol.for( + '@workflow/errors//WorkflowStartError' +); const RUNTIME_DECRYPTION_ERROR_KEY = Symbol.for( '@workflow/errors//RuntimeDecryptionError' ); @@ -962,6 +1008,14 @@ if (typeof globalThis !== 'undefined') { configurable: false, }); } + if (!Object.hasOwn(globalThis, WORKFLOW_START_ERROR_KEY)) { + Object.defineProperty(globalThis, WORKFLOW_START_ERROR_KEY, { + value: WorkflowStartError, + writable: false, + enumerable: false, + configurable: false, + }); + } if (!Object.hasOwn(globalThis, RUNTIME_DECRYPTION_ERROR_KEY)) { Object.defineProperty(globalThis, RUNTIME_DECRYPTION_ERROR_KEY, { value: RuntimeDecryptionError, diff --git a/packages/web-shared/src/lib/hydration.ts b/packages/web-shared/src/lib/hydration.ts index edefa0caea..94565cf1b6 100644 --- a/packages/web-shared/src/lib/hydration.ts +++ b/packages/web-shared/src/lib/hydration.ts @@ -150,7 +150,7 @@ export function getWebRevivers(): Revivers { // `packages/core/src/serialization/reducers/common.ts`) emits a tagged // entry for each built-in Error subclass plus the workflow-specific // `FatalError` / `RetryableError` / `HookConflictError` / - // `RuntimeDecryptionError` and `AggregateError`. Without + // `WorkflowStartError` / `RuntimeDecryptionError` and `AggregateError`. Without // matching revivers here, `devalue.unflatten` throws "Unknown type X" // — which surfaces in the web o11y UI as "Failed to load resource // details: Unknown type FatalError". @@ -212,6 +212,30 @@ export function getWebRevivers(): Revivers { if (value.stack !== undefined) error.stack = value.stack; return error; }, + WorkflowStartError: (value) => { + const opts = 'cause' in value ? { cause: value.cause } : undefined; + const error = new Error(value.message, opts) as Error & { + runId?: string; + stage?: string; + queued?: true | 'unknown'; + retryable?: boolean; + status?: number; + url?: string; + code?: string; + retryAfter?: number; + }; + error.name = 'WorkflowStartError'; + error.runId = value.runId; + error.stage = value.stage; + error.queued = value.stage === 'queue' ? 'unknown' : true; + error.retryable = value.retryable; + if (value.status !== undefined) error.status = value.status; + if (value.url !== undefined) error.url = value.url; + if (value.code !== undefined) error.code = value.code; + if (value.retryAfter !== undefined) error.retryAfter = value.retryAfter; + if (value.stack !== undefined) error.stack = value.stack; + return error; + }, RetryableError: (value) => { const opts = 'cause' in value ? { cause: value.cause } : undefined; const error = new Error(value.message, opts) as Error & { diff --git a/packages/web-shared/test/hydration.test.ts b/packages/web-shared/test/hydration.test.ts index 87e0d72534..baa93bd100 100644 --- a/packages/web-shared/test/hydration.test.ts +++ b/packages/web-shared/test/hydration.test.ts @@ -4,7 +4,11 @@ import { dehydrateStepReturnValue, } from '@workflow/core/serialization'; import { hydrateData } from '@workflow/core/serialization-format'; -import { FatalError, RetryableError } from '@workflow/errors'; +import { + FatalError, + RetryableError, + WorkflowStartError, +} from '@workflow/errors'; import { describe, expect, it } from 'vitest'; import { getWebRevivers, @@ -123,6 +127,34 @@ describe('getWebRevivers — error family', () => { expect(revived.conflictingRunId).toBe('wrun_conflicting'); }); + it('hydrates a WorkflowStartError with recovery details preserved', async () => { + const revived = await roundTrip< + Error & { + runId?: string; + stage?: string; + queued?: true | 'unknown'; + retryable?: boolean; + status?: number; + } + >( + new WorkflowStartError('admission uncertain', { + runId: 'wrun_queued', + stage: 'admission', + retryable: false, + status: 403, + }) + ); + + expect(revived).toBeInstanceOf(Error); + expect(revived.name).toBe('WorkflowStartError'); + expect(revived.message).toBe('admission uncertain'); + expect(revived.runId).toBe('wrun_queued'); + expect(revived.stage).toBe('admission'); + expect(revived.queued).toBe(true); + expect(revived.retryable).toBe(false); + expect(revived.status).toBe(403); + }); + it('hydrates a RetryableError with retryAfter as a Date', async () => { const retryAt = new Date('2025-01-01T00:00:00.000Z'); const revived = await roundTrip( diff --git a/packages/web-shared/test/serializable-revivers.test.ts b/packages/web-shared/test/serializable-revivers.test.ts index c5163bea7a..3d480a085f 100644 --- a/packages/web-shared/test/serializable-revivers.test.ts +++ b/packages/web-shared/test/serializable-revivers.test.ts @@ -1,10 +1,10 @@ import { readFileSync } from 'node:fs'; import { fileURLToPath } from 'node:url'; import { hydrateData } from '@workflow/core/serialization-format'; -import { getCLIRevivers } from '../../cli/src/lib/inspect/hydration.js'; -import { getWebRevivers } from '../src/lib/hydration.js'; import ts from 'typescript'; import { describe, expect, it } from 'vitest'; +import { getCLIRevivers } from '../../cli/src/lib/inspect/hydration.js'; +import { getWebRevivers } from '../src/lib/hydration.js'; const SERIALIZABLE_TYPES_PATH = fileURLToPath( new URL('../../core/src/serialization/types.ts', import.meta.url) @@ -100,6 +100,21 @@ const SERIALIZABLE_PAYLOADS: Record = { 'approval-token', 'wrun_conflicting', ], + WorkflowStartError: [ + ['WorkflowStartError', 1], + { + message: 2, + runId: 3, + stage: 4, + retryable: 5, + status: 6, + }, + 'Workflow run "wrun_queued" was queued, but start-hook admission could not be confirmed.', + 'wrun_queued', + 'admission', + false, + 403, + ], Instance: [ ['Instance', 1], { classId: 2, data: 3 }, diff --git a/packages/workflow/src/internal/errors.ts b/packages/workflow/src/internal/errors.ts index 97c941947e..d2c1578185 100644 --- a/packages/workflow/src/internal/errors.ts +++ b/packages/workflow/src/internal/errors.ts @@ -14,5 +14,6 @@ export { WorkflowRunNotCompletedError, WorkflowRunNotFoundError, WorkflowRuntimeError, + WorkflowStartError, WorkflowWorldError, } from '@workflow/errors'; diff --git a/packages/world-local/src/index.ts b/packages/world-local/src/index.ts index 4b57fbea28..79eebfeaae 100644 --- a/packages/world-local/src/index.ts +++ b/packages/world-local/src/index.ts @@ -17,7 +17,11 @@ import { import { initDataDir } from './init.js'; import { instrumentObject } from './instrumentObject.js'; import { createQueue, type DirectHandler } from './queue.js'; -import { hashToken, hookRecoveryMarkerPath } from './storage/helpers.js'; +import { + hookRecoveryMarkerPath, + hookTokenClaimPath, + readHookTokenClaim, +} from './storage/helpers.js'; import { createStorage } from './storage.js'; import { createStreamer } from './streamer.js'; @@ -69,6 +73,7 @@ export function createLocalWorld(args?: Partial): LocalWorld { const recoverActiveRuns = mergedConfig.recoverActiveRuns ?? true; return { specVersion: SPEC_VERSION_CURRENT, + experimentalStartHookAdmission: { mode: 'event-first' }, ...queue, ...storage, ...instrumentObject('world.streams', { @@ -128,9 +133,7 @@ export function createLocalWorld(args?: Partial): LocalWorld { HookSchema ); if (hook?.token) { - await deleteJSON( - path.join(hooksDir, 'tokens', `${hashToken(hook.token)}.json`) - ); + await deleteJSON(hookTokenClaimPath(basedir, hook.token)); await deleteJSON( hookRecoveryMarkerPath( basedir, @@ -142,6 +145,29 @@ export function createLocalWorld(args?: Partial): LocalWorld { } }) ); + const tokensDir = path.join(hooksDir, 'tokens'); + const tokenFiles = await fs.readdir(tokensDir).catch(() => []); + await Promise.all( + tokenFiles + .filter((tokenFile) => tokenFile.endsWith('.json')) + .map(async (tokenFile) => { + const claimPath = path.join(tokensDir, tokenFile); + const claim = await readHookTokenClaim(claimPath); + if (claim?.tag !== tag) return; + + await deleteJSON(claimPath); + if (claim.token && claim.hookId) { + await deleteJSON( + hookRecoveryMarkerPath( + basedir, + claim.token, + claim.runId, + claim.hookId + ) + ); + } + }) + ); // Delete tagged entity files across all directories const entityDirs = [ diff --git a/packages/world-local/src/storage.test.ts b/packages/world-local/src/storage.test.ts index dcf4f109c2..671c72bc50 100644 --- a/packages/world-local/src/storage.test.ts +++ b/packages/world-local/src/storage.test.ts @@ -1,13 +1,17 @@ import { promises as fs } from 'node:fs'; import os from 'node:os'; import path from 'node:path'; -import { WorkflowWorldError } from '@workflow/errors'; +import { + EntityConflictError, + HookConflictError, + WorkflowWorldError, +} from '@workflow/errors'; import type { Event, Storage } from '@workflow/world'; import { SPEC_VERSION_CURRENT, stripEventDataRefs } from '@workflow/world'; import { monotonicFactory } from 'ulid'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { writeJSON } from './fs.js'; -import { hashToken } from './storage/helpers.js'; +import { hashToken, hookRecoveryMarkerPath } from './storage/helpers.js'; import { createStorage } from './storage.js'; import { completeWait, @@ -2116,6 +2120,666 @@ describe('Storage', () => { expect(result.hook).toBeUndefined(); }); + it('should reject duplicate start hook claims before hook materialization', async () => { + const token = 'start-claim-token'; + const first = await storage.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }, + }); + expect(first.run).toBeDefined(); + const firstRun = first.run; + if (!firstRun) throw new Error('Expected run to be created'); + + await expect( + storage.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }, + }) + ).rejects.toThrow(HookConflictError); + + await expect( + storage.events.create(firstRun.runId, { + eventType: 'hook_created', + correlationId: 'hook_start', + eventData: { token }, + }) + ).resolves.toMatchObject({ + event: { eventType: 'hook_created' }, + hook: { + runId: firstRun.runId, + hookId: 'hook_start', + token, + }, + }); + }); + + it('should reserve start hook claims during resilient run_started bootstrap', async () => { + const token = 'resilient-start-claim-token'; + const runId = `wrun_${monotonicFactory()()}`; + + const started = await storage.events.create(runId, { + eventType: 'run_started', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }, + }); + + expect(started.run?.runId).toBe(runId); + await expect( + storage.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'deployment-456', + workflowName: 'test-workflow-2', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }, + }) + ).rejects.toThrow(HookConflictError); + + await expect( + storage.events.create(runId, { + eventType: 'hook_created', + correlationId: 'hook_resilient_start', + eventData: { token }, + }) + ).resolves.toMatchObject({ + event: { eventType: 'hook_created' }, + hook: { runId, hookId: 'hook_resilient_start', token }, + }); + }); + + it('should materialize only one hook for a reserved start hook token', async () => { + const token = 'concurrent-start-claim-materialize-token'; + const started = await storage.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }, + }); + const run = started.run; + if (!run) throw new Error('Expected run to be created'); + + const results = await Promise.all([ + storage.events.create(run.runId, { + eventType: 'hook_created', + correlationId: 'hook_start_a', + eventData: { token }, + }), + storage.events.create(run.runId, { + eventType: 'hook_created', + correlationId: 'hook_start_b', + eventData: { token }, + }), + ]); + + expect(results.map((result) => result.event.eventType).sort()).toEqual([ + 'hook_conflict', + 'hook_created', + ]); + expect(results.filter((result) => result.hook).length).toBe(1); + }); + + it('should repair a same-run start hook retry when the run_created event is missing', async () => { + const token = 'same-run-start-claim-token'; + const first = await storage.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }, + }); + expect(first.run).toBeDefined(); + const run = first.run; + if (!run) throw new Error('Expected run to be created'); + + await fs.unlink( + path.join( + testDir, + 'events', + `${run.runId}-${first.event.eventId}.json` + ) + ); + + const retry = await storage.events.create(run.runId, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }, + }); + + expect(retry.event.eventType).toBe('run_created'); + expect(retry.event.eventId).toBe(first.event.eventId); + expect(retry.run?.runId).toBe(run.runId); + }); + + it('should recover a stale start hook materialization lock', async () => { + const token = 'stale-materialize-lock-token'; + const first = await storage.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }, + }); + expect(first.run).toBeDefined(); + const run = first.run; + if (!run) throw new Error('Expected run to be created'); + + const lockPath = path.join( + testDir, + '.locks', + 'hooks', + `${hashToken(token)}.materialize` + ); + await fs.mkdir(path.dirname(lockPath), { recursive: true }); + await fs.writeFile(lockPath, ''); + const stale = new Date(Date.now() - 60_000); + await fs.utimes(lockPath, stale, stale); + + const result = await storage.events.create(run.runId, { + eventType: 'hook_created', + correlationId: 'hook_stale_materialize', + eventData: { token }, + }); + + expect(result.event.eventType).toBe('hook_created'); + expect(result.hook?.hookId).toBe('hook_stale_materialize'); + }); + + it('should not adopt or duplicate run_created when losing to a pre-existing hookless run', async () => { + const token = 'failed-run-create-start-claim-token'; + const run = await createRun(storage, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + }); + + // A hook-carrying run_created for a runId that already exists + // (created without a start hook) is a divergent duplicate: it must + // conflict, and it must NOT append a second run_created event. Its + // claim stays behind and expires via TTL once the run is terminal. + await expect( + storage.events.create(run.runId, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'deployment-456', + workflowName: 'test-workflow-2', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }, + }) + ).rejects.toThrow(EntityConflictError); + + const runEvents = await storage.events.list({ + runId: run.runId, + pagination: {}, + }); + expect( + runEvents.data.filter((e) => e.eventType === 'run_created') + ).toHaveLength(1); + }); + + it('should converge duplicate same-run hook-carrying run_created deliveries on one event', async () => { + const token = 'duplicate-run-create-start-claim-token'; + const request = { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }; + const started = await storage.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: request, + }); + const runId = started.run?.runId; + if (!runId) throw new Error('Expected run to be created'); + + // A duplicate delivery of the same run_created converges on the + // canonical event path and dedups instead of appending a second + // run_created or deleting the live claim. + await expect( + storage.events.create(runId, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: request, + }) + ).rejects.toThrow(EntityConflictError); + + const runEvents = await storage.events.list({ + runId, + pagination: {}, + }); + expect( + runEvents.data.filter((e) => e.eventType === 'run_created') + ).toHaveLength(1); + + // The claim survived the duplicate: another run still conflicts. + await expect( + storage.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'deployment-456', + workflowName: 'test-workflow-2', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }, + }) + ).rejects.toThrow(HookConflictError); + }); + + it('should retain a disposed start hook claim until ttl expiry', async () => { + const token = 'disposed-start-claim-token'; + const started = await storage.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }, + }); + expect(started.run).toBeDefined(); + const startedRun = started.run; + if (!startedRun) throw new Error('Expected run to be created'); + const runId = startedRun.runId; + + await storage.events.create(runId, { + eventType: 'hook_created', + correlationId: 'hook_start_disposed', + eventData: { token }, + }); + await storage.events.create(runId, { + eventType: 'hook_disposed', + correlationId: 'hook_start_disposed', + }); + + await expect( + storage.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }, + }) + ).rejects.toThrow(HookConflictError); + }); + + it('should not materialize a retained start hook claim again', async () => { + const token = 'retained-start-claim-token'; + const started = await storage.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }, + }); + const run = started.run; + if (!run) throw new Error('Expected run to be created'); + + await storage.events.create(run.runId, { + eventType: 'hook_created', + correlationId: 'hook_start_retained', + eventData: { token }, + }); + await storage.events.create(run.runId, { + eventType: 'hook_disposed', + correlationId: 'hook_start_retained', + }); + + const result = await storage.events.create(run.runId, { + eventType: 'hook_created', + correlationId: 'hook_start_retained_again', + eventData: { token }, + }); + + expect(result.event.eventType).toBe('hook_conflict'); + expect(result.hook).toBeUndefined(); + }); + + it('should retain an unmaterialized start hook claim after run failure until ttl expiry', async () => { + const token = 'failed-unmaterialized-start-claim-token'; + const started = await storage.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }, + }); + const run = started.run; + if (!run) throw new Error('Expected run to be created'); + + await storage.events.create(run.runId, { + eventType: 'run_started', + }); + await storage.events.create(run.runId, { + eventType: 'run_failed', + eventData: { error: new Uint8Array() }, + }); + + await expect( + storage.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'deployment-456', + workflowName: 'test-workflow-2', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }, + }) + ).rejects.toThrow(HookConflictError); + + const past = new Date(Date.now() - 2_000); + await writeJSON( + path.join(testDir, 'hooks', 'tokens', `${hashToken(token)}.json`), + { + token, + runId: run.runId, + ttlSeconds: 60, + createdAt: past, + expiresAt: past, + }, + { overwrite: true } + ); + + await expect( + storage.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'deployment-789', + workflowName: 'test-workflow-3', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }, + }) + ).resolves.toMatchObject({ + event: { eventType: 'run_created' }, + run: { workflowName: 'test-workflow-3' }, + }); + }); + + it('should release an unmaterialized start hook claim on cancellation', async () => { + const token = 'cancelled-unmaterialized-start-claim-token'; + const started = await storage.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }, + }); + const run = started.run; + if (!run) throw new Error('Expected run to be created'); + + // Cancel the claim-owning run before it materialized a hook: the + // token must be immediately reusable (cancel-then-retry, including + // start()'s own cleanup when queueing fails after admission). + await storage.events.create(run.runId, { + eventType: 'run_cancelled', + }); + + await expect( + storage.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'deployment-456', + workflowName: 'test-workflow-2', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }, + }) + ).resolves.toMatchObject({ + event: { eventType: 'run_created' }, + run: { workflowName: 'test-workflow-2' }, + }); + }); + + it('should keep an expired disposed start hook claim while the run is active', async () => { + const token = 'active-expired-start-claim-token'; + const started = await storage.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 1 }, + }, + }); + expect(started.run).toBeDefined(); + const run = started.run; + if (!run) throw new Error('Expected run to be created'); + + await storage.events.create(run.runId, { + eventType: 'hook_created', + correlationId: 'hook_start_expired_active', + eventData: { token }, + }); + await storage.events.create(run.runId, { + eventType: 'hook_disposed', + correlationId: 'hook_start_expired_active', + }); + + const past = new Date(Date.now() - 2_000); + await writeJSON( + path.join(testDir, 'hooks', 'tokens', `${hashToken(token)}.json`), + { + token, + runId: run.runId, + ttlSeconds: 1, + createdAt: past, + expiresAt: past, + }, + { overwrite: true } + ); + + await expect( + storage.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'deployment-456', + workflowName: 'test-workflow-2', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 1 }, + }, + }) + ).rejects.toThrow(HookConflictError); + + await storage.events.create(run.runId, { + eventType: 'run_completed', + eventData: { output: new Uint8Array() }, + }); + + await expect( + storage.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'deployment-789', + workflowName: 'test-workflow-3', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 1 }, + }, + }) + ).resolves.toMatchObject({ + event: { eventType: 'run_created' }, + run: { workflowName: 'test-workflow-3' }, + }); + }); + + it('should allow an expired orphaned start hook claim to be reused', async () => { + const token = 'expired-orphaned-start-claim-token'; + const past = new Date(Date.now() - 2_000); + await writeJSON( + path.join(testDir, 'hooks', 'tokens', `${hashToken(token)}.json`), + { + token, + runId: 'orphaned-start-run', + ttlSeconds: 1, + createdAt: past, + }, + { overwrite: true } + ); + + await expect( + storage.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 1 }, + }, + }) + ).resolves.toMatchObject({ + event: { eventType: 'run_created' }, + run: { workflowName: 'test-workflow' }, + }); + }); + + it('should allow ordinary hook creation after a retained start hook claim expires', async () => { + const token = 'expired-retained-ordinary-hook-token'; + const started = await storage.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 1 }, + }, + }); + const run = started.run; + if (!run) throw new Error('Expected run to be created'); + + await storage.events.create(run.runId, { + eventType: 'hook_created', + correlationId: 'hook_start_expired_reusable', + eventData: { token }, + }); + await storage.events.create(run.runId, { + eventType: 'hook_disposed', + correlationId: 'hook_start_expired_reusable', + }); + await storage.events.create(run.runId, { + eventType: 'run_completed', + eventData: { output: new Uint8Array() }, + }); + + const past = new Date(Date.now() - 2_000); + await writeJSON( + path.join(testDir, 'hooks', 'tokens', `${hashToken(token)}.json`), + { + token, + runId: run.runId, + hookId: 'hook_start_expired_reusable', + ttlSeconds: 1, + createdAt: past, + expiresAt: past, + }, + { overwrite: true } + ); + + const nextRun = await createRun(storage, { + deploymentId: 'deployment-456', + workflowName: 'test-workflow-2', + input: new Uint8Array(), + }); + + await expect( + storage.events.create(nextRun.runId, { + eventType: 'hook_created', + correlationId: 'hook_after_retained_expired', + eventData: { token }, + }) + ).resolves.toMatchObject({ + event: { eventType: 'hook_created' }, + hook: { + runId: nextRun.runId, + hookId: 'hook_after_retained_expired', + token, + }, + }); + }); + + it('should ignore hook recovery markers when cleaning up start hook claims', async () => { + const run = await createRun(storage, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + }); + + await writeJSON( + hookRecoveryMarkerPath( + testDir, + 'marker-token', + run.runId, + 'hook_marker' + ), + { eventId: 'wevt_marker' }, + { overwrite: true } + ); + + await expect( + updateRun(storage, run.runId, 'run_completed', { + output: new Uint8Array(), + }) + ).resolves.toMatchObject({ + status: 'completed', + }); + }); + it('should return hook_conflict event when the token claim cannot provide a run ID', async () => { const token = 'legacy-duplicate-test-token'; diff --git a/packages/world-local/src/storage/events-storage.ts b/packages/world-local/src/storage/events-storage.ts index 0f82192dbd..fa14ef07d1 100644 --- a/packages/world-local/src/storage/events-storage.ts +++ b/packages/world-local/src/storage/events-storage.ts @@ -2,6 +2,7 @@ import fs from 'node:fs/promises'; import path from 'node:path'; import { EntityConflictError, + HookConflictError, HookNotFoundError, RunExpiredError, RunNotSupportedError, @@ -12,6 +13,7 @@ import { import type { Event, EventResult, + ExperimentalStartHook, Hook, SerializedData, Step, @@ -53,11 +55,17 @@ import { import { stripEventDataRefs } from './filters.js'; import { getObjectCreatedAt, + type HookTokenClaim, hashToken, hookRecoveryMarkerPath, + hookTokenClaimPath, monotonicUlid, + readHookTokenClaim, } from './helpers.js'; -import { deleteAllHooksForRun } from './hooks-storage.js'; +import { + deleteAllHooksForRun, + settleClaimForDisposedHook, +} from './hooks-storage.js'; import { handleLegacyEvent } from './legacy.js'; import { withRunFileLock } from './runs-storage.js'; @@ -84,25 +92,11 @@ import { withRunFileLock } from './runs-storage.js'; // but a shared filesystem), exactly matching the cross-process // semantics without spawning subprocesses. -const HookTokenClaimSchema = z.object({ - // The token-claim writer below has always persisted `hookId`, but - // this read schema previously omitted it, which is the bug fixed - // by https://github.com/vercel/workflow/issues/2283. `optional()` - // is defensive: any claim file that somehow lacks the field still - // parses (yielding `undefined`) and falls through to the cross- - // hook conflict branch, matching pre-fix behavior. - hookId: z.string().optional(), - runId: z.string(), - // `eventId` is the canonical hook_created event ID the claiming - // worker committed to publishing. Persisting it here turns the - // claim file into a durable convergence key for cross-worker / - // cross-process retries (see comment on the hook_created branch). - // `optional()` for backward compatibility: a legacy claim file - // written before this field existed falls through to the recovery- - // marker upgrade path, which atomically pins a canonical eventId - // via a sidecar marker (also a `writeExclusive`). - eventId: z.string().optional(), -}); +const START_HOOK_LOCK_STALE_MS = 30_000; + +/** Whether a run status is terminal (no further lifecycle transitions). */ +const isRunTerminal = (status: string) => + ['completed', 'failed', 'cancelled'].includes(status); /** * Sidecar recovery marker that pins a canonical `hook_created` @@ -125,19 +119,196 @@ const HookRecoveryMarkerSchema = z.object({ eventId: z.string(), }); -async function readHookTokenClaim( - constraintPath: string -): Promise | null> { +/** + * Runs `fn` under THE exclusive cross-process file lock for a token's claim + * writes — one lock per token, shared by reclaim and materialize, so the two + * read-check-overwrite operations mutually exclude. Returns `undefined` + * (without running `fn`) when the lock is contended; callers treat that as + * a retryable failure and re-validate. Locks abandoned by a crashed holder + * (older than START_HOOK_LOCK_STALE_MS) are broken atomically via rename, + * so two breakers can never both enter, and the holder only removes a lock + * it still owns, so a stalled holder cannot free a breaker's lock. + */ +async function withTokenClaimLock( + basedir: string, + token: string, + fn: () => Promise +): Promise { + const lockPath = resolveWithinBase( + basedir, + '.locks', + 'hooks', + `${hashToken(token)}.claim` + ); + const owner = monotonicUlid(); + let locked = await writeExclusive(lockPath, owner); + if (!locked) { + const stat = await fs.stat(lockPath).catch(() => undefined); + if (stat && Date.now() - stat.mtimeMs > START_HOOK_LOCK_STALE_MS) { + const broken = await fs.rename(lockPath, `${lockPath}.${owner}`).then( + () => true, + () => false + ); + if (broken) { + await fs.unlink(`${lockPath}.${owner}`).catch(() => {}); + locked = await writeExclusive(lockPath, owner); + } + } + } + if (!locked) return undefined; + try { - return await readJSON(constraintPath, HookTokenClaimSchema); - } catch (error) { - if (error instanceof SyntaxError || error instanceof z.ZodError) { - return null; + return await fn(); + } finally { + const current = await fs.readFile(lockPath, 'utf8').catch(() => undefined); + if (current === owner) { + await fs.unlink(lockPath).catch(() => {}); } - throw error; } } +/** + * An expired claim may be replaced only when its owning run is gone or + * terminal — an active run keeps its token fenced even past the TTL. + */ +async function canReuseExpiredStartClaim( + basedir: string, + tag: string | undefined, + claim: HookTokenClaim +): Promise { + const expiresAt = + claim.expiresAt ?? + (claim.createdAt && claim.ttlSeconds + ? new Date(claim.createdAt.getTime() + claim.ttlSeconds * 1000) + : undefined); + if (!expiresAt || expiresAt > new Date()) return false; + + const run = await readJSONWithFallback( + basedir, + 'runs', + claim.runId, + WorkflowRunSchema, + claim.tag ?? tag + ); + if (!run) return true; + return isRunTerminal(run.status); +} + +async function replaceExpiredStartClaim( + basedir: string, + tag: string | undefined, + token: string, + currentClaim: HookTokenClaim, + nextClaim: HookTokenClaim +): Promise { + // Cheap pre-check on the caller's snapshot: an unexpired claim can never + // be reclaimed, so skip the lock + re-reads entirely. The in-lock check + // below stays authoritative. + if (!(await canReuseExpiredStartClaim(basedir, tag, currentClaim))) { + return false; + } + const replaced = await withTokenClaimLock(basedir, token, async () => { + const constraintPath = hookTokenClaimPath(basedir, token); + const latestClaim = await readHookTokenClaim(constraintPath); + if ( + !latestClaim || + !(await canReuseExpiredStartClaim(basedir, tag, latestClaim)) + ) { + return false; + } + await writeJSON(constraintPath, nextClaim, { overwrite: true }); + return true; + }); + return replaced ?? false; +} + +/** + * Binds an unmaterialized same-run start claim to a hook. Fails (returns + * false) when the claim is gone, owned by another run, already bound to a + * hook, or retained. + */ +async function materializeStartHookClaim( + basedir: string, + token: string, + nextClaim: HookTokenClaim +): Promise { + const materialized = await withTokenClaimLock(basedir, token, async () => { + const constraintPath = hookTokenClaimPath(basedir, token); + const latestClaim = await readHookTokenClaim(constraintPath); + if ( + !latestClaim || + latestClaim.runId !== nextClaim.runId || + latestClaim.hookId !== undefined || + latestClaim.expiresAt !== undefined + ) { + return false; + } + await writeJSON(constraintPath, nextClaim, { overwrite: true }); + return true; + }); + return materialized ?? false; +} + +/** + * Claims `token` for `runId`'s run_created event. Returns the canonical + * run_created eventId for the run: the caller's `eventId` when this call + * created (or reclaimed) the token, or the eventId recorded by a previous + * same-run attempt — every duplicate delivery converges on one event path, + * so the exclusive event publish downstream dedups instead of appending a + * second run_created. Throws HookConflictError when another run holds a + * live claim. + */ +async function claimStartHookToken( + basedir: string, + runId: string, + eventId: string, + startHook: ExperimentalStartHook, + tag?: string +): Promise { + const constraintPath = hookTokenClaimPath(basedir, startHook.token); + const nextClaim: HookTokenClaim = { + token: startHook.token, + runId, + ttlSeconds: startHook.ttlSeconds, + startEventId: eventId, + createdAt: new Date(), + tag, + }; + const claimed = await writeExclusive( + constraintPath, + JSON.stringify(nextClaim) + ); + if (claimed) return eventId; + + const existingClaim = await readHookTokenClaim(constraintPath); + if (existingClaim?.runId === runId) { + return existingClaim.startEventId ?? eventId; + } + if ( + existingClaim && + (await replaceExpiredStartClaim( + basedir, + tag, + startHook.token, + existingClaim, + nextClaim + )) + ) { + return eventId; + } + if (existingClaim) { + throw new HookConflictError(startHook.token, existingClaim.runId); + } + + // The exclusive write failed but the claim file was gone by the time we + // read it (concurrently reclaimed) — re-read once to converge. + const currentClaim = await readHookTokenClaim(constraintPath); + if (currentClaim?.runId === runId) { + return currentClaim.startEventId ?? eventId; + } + throw new HookConflictError(startHook.token, currentClaim?.runId); +} + async function readHookRecoveryMarker( markerPath: string ): Promise | null> { @@ -626,10 +797,6 @@ export function createEventsStorage( // specVersion is always sent by the runtime, but we provide a fallback for safety const effectiveSpecVersion = data.specVersion ?? SPEC_VERSION_CURRENT; - // Helper to check if run is in terminal state - const isRunTerminal = (status: string) => - ['completed', 'failed', 'cancelled'].includes(status); - // Helper to check if step is in terminal state const isStepTerminal = (status: string) => ['completed', 'failed', 'cancelled'].includes(status); @@ -668,6 +835,7 @@ export function createEventsStorage( executionContext?: Record; attributes?: Record; allowReservedAttributes?: true; + experimentalStartHook?: ExperimentalStartHook; }; if ( runInputData.deploymentId && @@ -703,6 +871,20 @@ export function createEventsStorage( createdAt: now, updatedAt: now, }; + // For start-hook runs, claimStartHookToken returns the + // canonical run_created eventId — duplicate deliveries of the + // same runId converge on one event path so the exclusive + // event write below dedups. + let runCreatedEventId = `evnt_${monotonicUlid()}`; + if (runInputData.experimentalStartHook) { + runCreatedEventId = await claimStartHookToken( + basedir, + effectiveRunId, + runCreatedEventId, + runInputData.experimentalStartHook, + tag + ); + } const runPath = taggedPath(basedir, 'runs', effectiveRunId, tag); const created = await writeExclusive( runPath, @@ -711,7 +893,6 @@ export function createEventsStorage( if (created) { // We created the run — also write the run_created event. - const runCreatedEventId = `evnt_${monotonicUlid()}`; const runCreatedEvent: Event = { eventType: 'run_created', runId: effectiveRunId, @@ -726,13 +907,17 @@ export function createEventsStorage( attributes: runInputData.attributes, allowReservedAttributes: runInputData.allowReservedAttributes, + experimentalStartHook: runInputData.experimentalStartHook, }, }; await storeEvent(runCreatedEvent); currentRun = createdRun; } else { // Run already exists (concurrent run_created won the - // race). Re-read it so downstream logic sees the real state. + // race). Re-read it so downstream logic sees the real + // state. Any start-hook claim stays: same-run attempts + // share the canonical event path, and a claim orphaned by + // a truly divergent writer expires via its TTL. currentRun = await readJSONWithFallback( basedir, 'runs', @@ -1011,6 +1196,7 @@ export function createEventsStorage( executionContext?: Record; attributes?: Record; allowReservedAttributes?: true; + experimentalStartHook?: ExperimentalStartHook; }; validateAttributeChanges( Object.entries(runData.attributes ?? {}).map(([key, value]) => ({ @@ -1021,6 +1207,24 @@ export function createEventsStorage( allowReservedAttributes: runData.allowReservedAttributes === true, } ); + // For start-hook runs, claimStartHookToken returns the canonical + // run_created eventId — duplicate deliveries converge on one + // event path so the exclusive event publish downstream dedups. + // `ownsStartHookClaim` records whether this call created the + // claim (canonical id === ours) or adopted a previous same-run + // attempt's. + let ownsStartHookClaim = false; + if (runData.experimentalStartHook) { + const canonicalEventId = await claimStartHookToken( + basedir, + effectiveRunId, + eventId, + runData.experimentalStartHook, + tag + ); + ownsStartHookClaim = canonicalEventId === eventId; + eventId = canonicalEventId; + } run = { runId: effectiveRunId, deploymentId: runData.deploymentId, @@ -1048,9 +1252,62 @@ export function createEventsStorage( JSON.stringify(run, jsonReplacer, 2) ); if (!created) { - throw new EntityConflictError( - `Workflow run "${effectiveRunId}" already exists` - ); + // Lost the run write to a concurrent same-run creation. For + // start-hook runs whose claim we share, converge on the winner: + // reuse the existing run and the canonical eventId, and let the + // exclusive event publish downstream dedup. The claim is left + // alone — same-run attempts share it, and a claim orphaned by a + // truly divergent writer expires via its TTL. + const startHook = runData.experimentalStartHook; + const startHookClaim = startHook + ? await readHookTokenClaim( + hookTokenClaimPath(basedir, startHook.token) + ) + : null; + const existingRun = + startHook && + !ownsStartHookClaim && + startHookClaim?.runId === effectiveRunId && + startHookClaim.startEventId + ? await readJSONWithFallback( + basedir, + 'runs', + effectiveRunId, + WorkflowRunSchema, + tag + ) + : null; + + if (existingRun && startHook && startHookClaim?.startEventId) { + eventId = startHookClaim.startEventId; + event = { + eventType: 'run_created', + runId: effectiveRunId, + eventId, + createdAt: ulidToDate(eventId.replace(/^evnt_/, '')) ?? now, + specVersion: effectiveSpecVersion, + eventData: { + deploymentId: existingRun.deploymentId, + workflowName: existingRun.workflowName, + input: existingRun.input, + executionContext: existingRun.executionContext, + attributes: existingRun.attributes, + ...(runData.allowReservedAttributes + ? { allowReservedAttributes: true as const } + : {}), + experimentalStartHook: { + token: startHook.token, + ttlSeconds: + startHookClaim.ttlSeconds ?? startHook.ttlSeconds, + }, + }, + }; + run = existingRun; + } else { + throw new EntityConflictError( + `Workflow run "${effectiveRunId}" already exists` + ); + } } } else if (data.eventType === 'run_started') { // Reuse currentRun from validation (already read above) @@ -1182,7 +1439,9 @@ export function createEventsStorage( } ); await Promise.all([ - deleteAllHooksForRun(basedir, effectiveRunId), + deleteAllHooksForRun(basedir, effectiveRunId, { + releaseUnmaterializedClaims: true, + }), deleteAllWaitsForRun(basedir, effectiveRunId), ]); } @@ -1563,17 +1822,12 @@ export function createEventsStorage( // Atomically claim the token using an exclusive-create constraint file. // This avoids the TOCTOU race of the previous read-all-then-check approach. - const constraintPath = path.join( - basedir, - 'hooks', - 'tokens', - `${hashToken(hookData.token)}.json` - ); + const constraintPath = hookTokenClaimPath(basedir, hookData.token); // Persist `eventId` in the claim so concurrent / cross- // process retries can converge on a single canonical // `hook_created` event path. See the recovery comment // below. - const tokenClaimed = await writeExclusive( + let tokenClaimed = await writeExclusive( constraintPath, JSON.stringify({ token: hookData.token, @@ -1621,11 +1875,61 @@ export function createEventsStorage( let writeHookEntityWithOverwrite = false; if (!tokenClaimed) { - const existingClaim = await readHookTokenClaim(constraintPath); + let existingClaim = await readHookTokenClaim(constraintPath); if ( existingClaim?.runId === effectiveRunId && - existingClaim.hookId === data.correlationId + existingClaim.hookId === undefined && + existingClaim.expiresAt === undefined + ) { + // Unmaterialized same-run start claim: bind it to this hook. + tokenClaimed = await materializeStartHookClaim( + basedir, + hookData.token, + { + token: hookData.token, + hookId: data.correlationId, + runId: effectiveRunId, + eventId, + ttlSeconds: existingClaim.ttlSeconds, + startEventId: existingClaim.startEventId, + createdAt: existingClaim.createdAt, + tag: existingClaim.tag, + } + ); + if (!tokenClaimed) { + // Lost a concurrent materialize. Re-read so the dedup + // branch below sees the winner's claim — if the winner was + // a duplicate of this very (runId, hookId), this converges + // on its canonical eventId instead of emitting a spurious + // self-conflict. + existingClaim = await readHookTokenClaim(constraintPath); + } + } + if ( + !tokenClaimed && + existingClaim?.expiresAt !== undefined && + (await replaceExpiredStartClaim( + basedir, + tag, + hookData.token, + existingClaim, + { + token: hookData.token, + hookId: data.correlationId, + runId: effectiveRunId, + eventId, + } + )) + ) { + tokenClaimed = true; + } + + if ( + !tokenClaimed && + existingClaim?.runId === effectiveRunId && + existingClaim.hookId === data.correlationId && + existingClaim.expiresAt === undefined ) { // Adopt a canonical eventId for the recovery write. The // outer event publish (`writeExclusive(eventPath)`) @@ -1729,7 +2033,7 @@ export function createEventsStorage( specVersion: effectiveSpecVersion, }; writeHookEntityWithOverwrite = true; - } else { + } else if (!tokenClaimed) { // Cross-hook / cross-run conflict: a different // (runId, hookId) holds this token. Create a // hook_conflict event so the workflow can fail @@ -1841,19 +2145,10 @@ export function createEventsStorage( tag ); if (existingHook) { - // Delete the token constraint file to free up the token - // for reuse, and delete this hook's recovery marker (if - // any) for disk hygiene. The marker's filename hash - // includes `(token, runId, hookId)` so different - // lifetimes never collide, but cleaning up reduces disk - // leak for hooks that go through the recovery path. - const disposedConstraintPath = path.join( - basedir, - 'hooks', - 'tokens', - `${hashToken(existingHook.token)}.json` - ); - await deleteJSON(disposedConstraintPath); + // Normal hook disposal frees the token; start-hook claims carry + // ttlSeconds and are retained so duplicate starts remain fenced + // after the active hook entity is gone. + await settleClaimForDisposedHook(basedir, existingHook, now); await deleteJSON( hookRecoveryMarkerPath( basedir, diff --git a/packages/world-local/src/storage/helpers.ts b/packages/world-local/src/storage/helpers.ts index 23156df992..6751125db1 100644 --- a/packages/world-local/src/storage/helpers.ts +++ b/packages/world-local/src/storage/helpers.ts @@ -1,7 +1,8 @@ import { createHash } from 'node:crypto'; import path from 'node:path'; import { monotonicFactory } from 'ulid'; -import { stripTag, ulidToDate } from '../fs.js'; +import { z } from 'zod'; +import { readJSON, stripTag, ulidToDate } from '../fs.js'; /** * Hash a hook token to produce a filesystem-safe constraint filename. @@ -10,6 +11,70 @@ export function hashToken(token: string): string { return createHash('sha256').update(token).digest('hex'); } +/** + * The token-claim constraint file: the single-owner record behind a hook + * token. A claim's lifecycle state is derived, not stored: + * - unmaterialized start claim: `startEventId` set, no `hookId`, no + * `expiresAt` (reserved by `start()`, no hook entity yet) + * - materialized: `hookId` set, no `expiresAt` (live hook entity) + * - retained: `expiresAt` set (hook disposed / run finished; the token + * stays fenced until `expiresAt`) + * + * Claims without `ttlSeconds` are plain `createHook` token guards with no + * retention window — they are deleted when the hook is disposed or the run + * reaches a terminal state. + */ +export const HookTokenClaimSchema = z.object({ + token: z.string().optional(), + // The token-claim writer has always persisted `hookId`, but an older + // read schema omitted it, which is the bug fixed by + // https://github.com/vercel/workflow/issues/2283. `optional()` is + // defensive: any claim file that somehow lacks the field still parses + // (yielding `undefined`) and falls through to the cross-hook conflict + // branch, matching pre-fix behavior. + hookId: z.string().optional(), + runId: z.string(), + // `eventId` is the canonical hook_created event ID the claiming worker + // committed to publishing. Persisting it here turns the claim file into + // a durable convergence key for cross-worker / cross-process retries + // (see the hook_created branch in events-storage.ts). `optional()` for + // backward compatibility: a legacy claim file written before this field + // existed falls through to the recovery-marker upgrade path, which + // atomically pins a canonical eventId via a sidecar marker. + eventId: z.string().optional(), + ttlSeconds: z.number().int().positive().optional(), + // `startEventId` is the canonical run_created event ID for claims + // reserved by `start()` — the same convergence-key role `eventId` plays + // for hook_created. + startEventId: z.string().optional(), + createdAt: z.coerce.date().optional(), + expiresAt: z.coerce.date().optional(), + tag: z.string().optional(), +}); + +export type HookTokenClaim = z.infer; + +export function hookTokenClaimPath(basedir: string, token: string): string { + return path.join(basedir, 'hooks', 'tokens', `${hashToken(token)}.json`); +} + +/** + * Tolerant claim reader: malformed or partially-written claim files parse + * to `null` (treated as "no claim") instead of failing the caller. + */ +export async function readHookTokenClaim( + constraintPath: string +): Promise { + try { + return await readJSON(constraintPath, HookTokenClaimSchema); + } catch (error) { + if (error instanceof SyntaxError || error instanceof z.ZodError) { + return null; + } + throw error; + } +} + /** * Compute the path of the recovery-marker sidecar for a specific * `(token, runId, hookId)` triple. Identity is encoded in the diff --git a/packages/world-local/src/storage/hooks-storage.ts b/packages/world-local/src/storage/hooks-storage.ts index e4c8f3c742..0c84bbc81a 100644 --- a/packages/world-local/src/storage/hooks-storage.ts +++ b/packages/world-local/src/storage/hooks-storage.ts @@ -16,9 +16,67 @@ import { paginatedFileSystemQuery, readJSON, readJSONWithFallback, + writeJSON, } from '../fs.js'; import { filterHookData } from './filters.js'; -import { hashToken, hookRecoveryMarkerPath } from './helpers.js'; +import { + type HookTokenClaim, + hookRecoveryMarkerPath, + hookTokenClaimPath, + readHookTokenClaim, +} from './helpers.js'; + +/** + * Transitions a TTL-carrying claim to retained: the token stays fenced + * until at least `hookCreatedAt + ttl` (falling back to the claim's own + * createdAt when no hook was ever created), never shrinking an existing + * retention window and never expiring before `now`. + */ +export async function retainStartHookClaim( + constraintPath: string, + claim: HookTokenClaim, + now: Date, + hookCreatedAt?: Date +): Promise { + if (!claim.ttlSeconds) return; + + const ttlBase = hookCreatedAt ?? claim.createdAt ?? now; + const expiresAt = new Date( + Math.max( + ttlBase.getTime() + claim.ttlSeconds * 1000, + claim.expiresAt?.getTime() ?? 0, + now.getTime() + ) + ); + + await writeJSON(constraintPath, { ...claim, expiresAt }, { overwrite: true }); +} + +/** + * Settles a disposed hook's token claim: TTL-carrying claims are retained + * (duplicate starts stay fenced past disposal), plain createHook guards are + * deleted to free the token. Shared by hook_disposed and terminal-run + * cleanup. + */ +export async function settleClaimForDisposedHook( + basedir: string, + hook: Pick, + now: Date +): Promise { + const constraintPath = hookTokenClaimPath(basedir, hook.token); + const claim = await readHookTokenClaim(constraintPath); + if (claim?.runId === hook.runId && claim.ttlSeconds) { + await retainStartHookClaim( + constraintPath, + { ...claim, token: hook.token }, + now, + hook.createdAt + ); + } else { + await deleteJSON(constraintPath); + } + return constraintPath; +} /** * Creates a hooks storage implementation using the filesystem. @@ -113,34 +171,69 @@ export function createHooksStorage( /** * Helper function to delete all hooks associated with a workflow run. * Called when a run reaches a terminal state. + * + * `releaseUnmaterializedClaims` (used for cancellation) additionally deletes + * start-hook claims the workflow never materialized into a hook, so + * cancel-then-retry — including `start()`'s own cleanup when queueing fails + * after admission — can reuse the token immediately. */ export async function deleteAllHooksForRun( basedir: string, - runId: string + runId: string, + opts?: { releaseUnmaterializedClaims?: boolean } ): Promise { const hooksDir = path.join(basedir, 'hooks'); const files = await listJSONFiles(hooksDir); + const tokensDir = path.join(hooksDir, 'tokens'); + const now = new Date(); + // Settle claims through the run's hook entities (retain TTL claims, free + // plain guards), and delete the hooks + recovery markers. The marker's + // filename hash includes `(token, runId, hookId)` so a leaked marker can + // never corrupt a different lifetime — cleaning it up here keeps the + // tokens/ directory from accumulating recovered-hook sidecars over time. + const settledClaimPaths = new Set(); for (const file of files) { const hookPath = path.join(hooksDir, `${file}.json`); const hook = await readJSON(hookPath, HookSchema); if (hook && hook.runId === runId) { - // Delete the token constraint file to free up the token, and - // delete the recovery marker (if any) for disk hygiene. The - // marker's filename hash includes `(token, runId, hookId)` so - // a leaked marker can never corrupt a different lifetime — but - // cleaning it up here keeps the tokens/ directory from - // accumulating recovered-hook sidecars over time. - const constraintPath = path.join( - hooksDir, - 'tokens', - `${hashToken(hook.token)}.json` + settledClaimPaths.add( + await settleClaimForDisposedHook(basedir, hook, now) ); - await deleteJSON(constraintPath); await deleteJSON( hookRecoveryMarkerPath(basedir, hook.token, hook.runId, hook.hookId) ); await deleteJSON(hookPath); } } + + // Sweep the tokens directory for claims with no hook entity: this run's + // unmaterialized start claims (retain or, on cancellation, release), plus + // opportunistic cleanup of anyone's expired retained-claim debris so the + // directory doesn't grow forever. + const tokenFiles = await listJSONFiles(tokensDir); + await Promise.all( + tokenFiles.map(async (file) => { + if (file.endsWith('.recovery')) return; + const constraintPath = path.join(tokensDir, `${file}.json`); + if (settledClaimPaths.has(constraintPath)) return; + const claim = await readHookTokenClaim(constraintPath); + if (!claim) return; + if (claim.expiresAt && claim.expiresAt <= now) { + // Retention window over — the claim is dead for every run. + await deleteJSON(constraintPath); + return; + } + if (claim.runId !== runId || !claim.ttlSeconds) return; + if ( + opts?.releaseUnmaterializedClaims && + claim.hookId === undefined && + claim.expiresAt === undefined + ) { + await deleteJSON(constraintPath); + } else { + await retainStartHookClaim(constraintPath, claim, now); + } + }) + ); } diff --git a/packages/world-local/src/tag.test.ts b/packages/world-local/src/tag.test.ts index 0ec9c39aff..d6686347ac 100644 --- a/packages/world-local/src/tag.test.ts +++ b/packages/world-local/src/tag.test.ts @@ -1,6 +1,7 @@ import { promises as fs } from 'node:fs'; import os from 'node:os'; import path from 'node:path'; +import { SPEC_VERSION_CURRENT } from '@workflow/world'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { listJSONFiles, stripTag } from './fs.js'; import { createStorage } from './storage.js'; @@ -268,6 +269,44 @@ describe('File tagging', () => { await world1.close?.(); }); + it('should not reclaim another tag active start hook claim after ttl', async () => { + const { createLocalWorld } = await import('./index.js'); + + const world0 = createLocalWorld({ dataDir: testDir, tag: 'vitest-0' }); + const world1 = createLocalWorld({ dataDir: testDir, tag: 'vitest-1' }); + await world0.start?.(); + + const token = 'cross-tag-active-start-hook-token'; + await world0.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'dep-1', + workflowName: 'wf-0', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 1 }, + }, + }); + + await new Promise((resolve) => setTimeout(resolve, 1_100)); + + await expect( + world1.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'dep-2', + workflowName: 'wf-1', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 1 }, + }, + }) + ).rejects.toThrow(); + + await world0.close?.(); + await world1.close?.(); + }); + it('should clear events, steps, hooks, and waits', async () => { const { createLocalWorld } = await import('./index.js'); @@ -351,6 +390,49 @@ describe('File tagging', () => { await world.close?.(); }); + + it('should clear unmaterialized start hook token claims', async () => { + const { createLocalWorld } = await import('./index.js'); + + const world = createLocalWorld({ dataDir: testDir, tag: 'vitest-0' }); + await world.start?.(); + + const token = 'tagged-start-hook-token'; + await world.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'dep-1', + workflowName: 'start-hook-wf', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }, + }); + + const tokensDir = path.join(testDir, 'hooks', 'tokens'); + expect(await fs.readdir(tokensDir)).toHaveLength(1); + + await world.clear(); + + expect(await fs.readdir(tokensDir)).toHaveLength(0); + await expect( + world.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'dep-2', + workflowName: 'start-hook-wf-2', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }, + }) + ).resolves.toMatchObject({ + event: { eventType: 'run_created' }, + run: { workflowName: 'start-hook-wf-2' }, + }); + + await world.close?.(); + }); }); describe('untagged clear()', () => { diff --git a/packages/world-postgres/src/drizzle/migrations/0016_add_hook_claims.sql b/packages/world-postgres/src/drizzle/migrations/0016_add_hook_claims.sql new file mode 100644 index 0000000000..03e9245477 --- /dev/null +++ b/packages/world-postgres/src/drizzle/migrations/0016_add_hook_claims.sql @@ -0,0 +1,12 @@ +CREATE TABLE IF NOT EXISTS "workflow"."workflow_hook_claims" ( + "token" varchar PRIMARY KEY NOT NULL, + "run_id" varchar NOT NULL, + "hook_id" varchar, + "ttl_seconds" integer NOT NULL, + "expires_at" timestamp, + "created_at" timestamp DEFAULT now() NOT NULL +); +--> statement-breakpoint +CREATE INDEX IF NOT EXISTS "workflow_hook_claims_run_id_index" ON "workflow"."workflow_hook_claims" USING btree ("run_id"); +--> statement-breakpoint +CREATE INDEX IF NOT EXISTS "workflow_hook_claims_expires_at_index" ON "workflow"."workflow_hook_claims" USING btree ("expires_at"); diff --git a/packages/world-postgres/src/drizzle/migrations/meta/_journal.json b/packages/world-postgres/src/drizzle/migrations/meta/_journal.json index 92b7e8d3ec..a299802e5c 100644 --- a/packages/world-postgres/src/drizzle/migrations/meta/_journal.json +++ b/packages/world-postgres/src/drizzle/migrations/meta/_journal.json @@ -113,6 +113,13 @@ "when": 1782691200000, "tag": "0015_move_enums_to_workflow_schema", "breakpoints": true + }, + { + "idx": 16, + "version": "7", + "when": 1782777600000, + "tag": "0016_add_hook_claims", + "breakpoints": true } ] } diff --git a/packages/world-postgres/src/drizzle/schema.ts b/packages/world-postgres/src/drizzle/schema.ts index 3a9be436b7..0f7b997140 100644 --- a/packages/world-postgres/src/drizzle/schema.ts +++ b/packages/world-postgres/src/drizzle/schema.ts @@ -218,6 +218,33 @@ export const hooks = schema.table( (tb) => [index().on(tb.runId), index().on(tb.token)] ); +/** + * Hook-token claims: the single-owner constraint behind hook tokens. + * + * A claim's lifecycle state is derived, not stored: + * - unmaterialized start claim: `hookId IS NULL` (reserved by `start()`, + * no hook entity yet) + * - materialized: `hookId` set, `expiresAt IS NULL` (live hook entity) + * - retained: `expiresAt` set (hook disposed / run finished; the token + * stays fenced until `expiresAt`) + * + * `ttlSeconds <= 0` marks a plain `createHook` token guard with no + * retention window — it is released when the hook is disposed or the run + * reaches a terminal state. + */ +export const hookClaims = schema.table( + 'workflow_hook_claims', + { + token: varchar('token').primaryKey(), + runId: varchar('run_id').notNull(), + hookId: varchar('hook_id'), + ttlSeconds: integer('ttl_seconds').notNull(), + expiresAt: timestamp('expires_at'), + createdAt: timestamp('created_at').defaultNow().notNull(), + }, + (tb) => [index().on(tb.runId), index().on(tb.expiresAt)] +); + export const waits = schema.table( 'workflow_waits', { diff --git a/packages/world-postgres/src/index.ts b/packages/world-postgres/src/index.ts index 9ad7565e06..3aa53def60 100644 --- a/packages/world-postgres/src/index.ts +++ b/packages/world-postgres/src/index.ts @@ -58,6 +58,7 @@ export function createWorld( return { specVersion: SPEC_VERSION_CURRENT, + experimentalStartHookAdmission: { mode: 'event-first' }, ...storage, ...streamer, ...queue, diff --git a/packages/world-postgres/src/storage.ts b/packages/world-postgres/src/storage.ts index 2d531802eb..3dbfdd0858 100644 --- a/packages/world-postgres/src/storage.ts +++ b/packages/world-postgres/src/storage.ts @@ -1,5 +1,6 @@ import { EntityConflictError, + HookConflictError, HookNotFoundError, RunExpiredError, RunNotSupportedError, @@ -12,6 +13,7 @@ import type { Event, EventResult, ExperimentalSetAttributesResult, + ExperimentalStartHook, GetEventParams, Hook, ListEventsParams, @@ -40,7 +42,18 @@ import { validateUlidTimestamp, WorkflowRunSchema, } from '@workflow/world'; -import { and, asc, desc, eq, gt, lt, notInArray, sql } from 'drizzle-orm'; +import { + and, + asc, + desc, + eq, + gt, + isNull, + lt, + notInArray, + or, + sql, +} from 'drizzle-orm'; import { monotonicFactory } from 'ulid'; import { type Drizzle, Schema } from './drizzle/index.js'; import type { SerializedContent } from './drizzle/schema.js'; @@ -329,6 +342,10 @@ async function handleLegacyEventPostgres( export function createEventsStorage(drizzle: Drizzle): Storage['events'] { const ulid = monotonicFactory(); + + /** Whether a run status is terminal (no further lifecycle transitions). */ + const isRunTerminal = (status: string) => + ['completed', 'failed', 'cancelled'].includes(status); const { events } = Schema; // Prepared statements for validation queries (performance optimization) @@ -391,6 +408,206 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { .limit(1) .prepare('events_get_wait_for_validation'); + /** + * Reads the claim for a token, lazily reclaiming dead claims. A claim is + * live while its expiry (explicit `expiresAt`, or `createdAt + ttl` for + * never-retained start claims) is in the future. Claims past expiry — and + * plain `createHook` guards (`ttlSeconds <= 0`), which have no expiry — + * additionally survive while their owning run is still active; otherwise + * they are debris from a crashed cleanup and are deleted here. + */ + async function getLiveHookClaim(token: string): Promise<{ + runId: string; + hookId: string | null; + expiresAt: Date | null; + } | null> { + // Bounded loop instead of recursion: the guarded DELETE below can miss + // when a concurrent writer replaces the row between the read and the + // delete; re-read and re-evaluate. + for (let attempt = 0; attempt < 3; attempt++) { + const [claim] = await drizzle + .select({ + runId: Schema.hookClaims.runId, + hookId: Schema.hookClaims.hookId, + expiresAt: Schema.hookClaims.expiresAt, + ttlSeconds: Schema.hookClaims.ttlSeconds, + createdAt: Schema.hookClaims.createdAt, + }) + .from(Schema.hookClaims) + .where(eq(Schema.hookClaims.token, token)) + .limit(1); + if (!claim) return null; + + const expiresAt = + claim.expiresAt ?? + (claim.ttlSeconds > 0 + ? new Date(claim.createdAt.getTime() + claim.ttlSeconds * 1000) + : null); + if (expiresAt && expiresAt > new Date()) return claim; + + const [run] = await drizzle + .select({ status: Schema.runs.status }) + .from(Schema.runs) + .where(eq(Schema.runs.runId, claim.runId)) + .limit(1); + if (run && !isRunTerminal(run.status)) { + return claim; + } + + const [deleted] = await drizzle + .delete(Schema.hookClaims) + .where( + and( + eq(Schema.hookClaims.token, token), + eq(Schema.hookClaims.runId, claim.runId), + claim.hookId + ? eq(Schema.hookClaims.hookId, claim.hookId) + : isNull(Schema.hookClaims.hookId), + claim.expiresAt + ? eq(Schema.hookClaims.expiresAt, claim.expiresAt) + : isNull(Schema.hookClaims.expiresAt) + ) + ) + .returning({ token: Schema.hookClaims.token }); + if (deleted) return null; + } + // The row kept churning under us; treat whatever is there as live — the + // caller reports a conflict, which is the safe default. + const [claim] = await drizzle + .select({ + runId: Schema.hookClaims.runId, + hookId: Schema.hookClaims.hookId, + expiresAt: Schema.hookClaims.expiresAt, + }) + .from(Schema.hookClaims) + .where(eq(Schema.hookClaims.token, token)) + .limit(1); + return claim ?? null; + } + + /** + * Transitions a finishing run's hook-token claims. Called before the + * run's hook entities are deleted, so the retention window can anchor on + * the hook's `createdAt`. Set-based statements so a concurrent + * materialize cannot interleave with a read-modify-write. + */ + async function retainHookClaimsForRun( + runId: string, + completedAt: Date, + opts?: { releaseUnmaterializedClaims?: boolean } + ): Promise { + // Two set-based statements (no read-modify-write a concurrent + // materialize could interleave with, and no transaction — they touch + // disjoint rows and are each independently atomic). + // + // 1. Release plain createHook token guards (ttlSeconds <= 0) with the + // run. Cancellation additionally releases start-hook claims the + // workflow never materialized into a hook: a cancelled run did no + // fenced work, and cancel-then-retry (including start()'s own + // cleanup when queueing fails after admission) must be able to reuse + // the token immediately. Piggyback deletion of anyone's expired + // retained-claim debris (index on expires_at) so the table does not + // grow forever — expired claims are only otherwise removed lazily on + // same-token reuse. + await drizzle.delete(Schema.hookClaims).where( + or( + and( + eq(Schema.hookClaims.runId, runId), + opts?.releaseUnmaterializedClaims + ? sql`(${Schema.hookClaims.ttlSeconds} <= 0 OR ${Schema.hookClaims.hookId} IS NULL)` + : sql`${Schema.hookClaims.ttlSeconds} <= 0` + ), + and( + lt(Schema.hookClaims.expiresAt, completedAt), + // Expired claims still fence their token while the owning run + // is active (getLiveHookClaim's liveness rule) — only sweep + // debris whose run is terminal or gone. + sql`NOT EXISTS ( + SELECT 1 FROM ${Schema.runs} r + WHERE r.id = ${Schema.hookClaims.runId} + AND r.status NOT IN ('completed', 'failed', 'cancelled') + )` + ) + ) + ); + // 2. Retain the rest until at least `hookCreatedAt + ttl` (falling back + // to the claim's own createdAt when no hook entity exists), never + // shrinking an existing retention window and never expiring before + // run completion. + await drizzle + .update(Schema.hookClaims) + .set({ + expiresAt: sql`GREATEST( + COALESCE( + (SELECT h.created_at FROM ${Schema.hooks} h WHERE h.hook_id = ${Schema.hookClaims.hookId}), + ${Schema.hookClaims.createdAt} + ) + make_interval(secs => ${Schema.hookClaims.ttlSeconds}), + COALESCE(${Schema.hookClaims.expiresAt}, ${completedAt}), + ${completedAt} + )`, + }) + .where( + and( + eq(Schema.hookClaims.runId, runId), + gt(Schema.hookClaims.ttlSeconds, 0) + ) + ); + } + + /** + * Pre-transaction ownership snapshot for a start-hook token: rejects + * tokens held by a hook entity or a live claim of ANOTHER run, and + * returns the same-run claim (if any) so callers can apply their + * duplicate-delivery semantics. getLiveHookClaim lazily reclaims dead + * claims here so the transactional insert can succeed afterwards. + */ + async function getOwnStartHookClaim(token: string, runId: string) { + const [[existingHook], existingClaim] = await Promise.all([ + getHookByToken.execute({ token }), + getLiveHookClaim(token), + ]); + if (existingHook && existingHook.runId !== runId) { + throw new HookConflictError(token, existingHook.runId); + } + if (existingClaim && existingClaim.runId !== runId) { + throw new HookConflictError(token, existingClaim.runId); + } + return existingClaim; + } + + /** + * Takes the token claim for `runId` inside `tx`. Tolerates a concurrent + * same-run claim, and a claim that vanished between the insert conflict + * and the re-read (concurrent reclaim) — in both cases the caller's run + * insert arbitrates the duplicate. Throws HookConflictError when another + * run owns the token. + */ + async function claimStartHookTokenInTx( + tx: Parameters[0]>[0], + startHook: ExperimentalStartHook, + runId: string + ): Promise { + const [createdClaim] = await tx + .insert(Schema.hookClaims) + .values({ + token: startHook.token, + runId, + ttlSeconds: startHook.ttlSeconds, + }) + .onConflictDoNothing() + .returning({ runId: Schema.hookClaims.runId }); + if (createdClaim) return; + + const [existing] = await tx + .select({ runId: Schema.hookClaims.runId }) + .from(Schema.hookClaims) + .where(eq(Schema.hookClaims.token, startHook.token)) + .limit(1); + if (existing && existing.runId !== runId) { + throw new HookConflictError(startHook.token, existing.runId); + } + } + return { async create(runId, data, params): Promise { const eventId = `wevt_${ulid()}`; @@ -425,11 +642,10 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { // the step (the caller won the create-claim). Surfaced on EventResult // as the runtime's exactly-once ownership signal. let stepCreatedLazily = false; + let eventCreatedWithRun: { createdAt: Date } | undefined; const now = new Date(); // Helper to check if run is in terminal state - const isRunTerminal = (status: string) => - ['completed', 'failed', 'cancelled'].includes(status); // Helper to check if step is in terminal state const isStepTerminal = (status: string) => @@ -478,6 +694,7 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { executionContext?: Record; attributes?: Record; allowReservedAttributes?: true; + experimentalStartHook?: ExperimentalStartHook; }; if ( runInputData.deploymentId && @@ -493,43 +710,57 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { runInputData.allowReservedAttributes === true, } ); - // Create run + run_created event atomically. The - // transaction ensures we never have an orphaned run - // without its run_created event. - const [inserted] = await drizzle - .insert(Schema.runs) - .values({ - runId: effectiveRunId, - deploymentId: runInputData.deploymentId, - workflowName: runInputData.workflowName, - specVersion: effectiveSpecVersion, - input: runInputData.input as SerializedContent, - executionContext: runInputData.executionContext as - | SerializedContent - | undefined, - attributes: runInputData.attributes, - status: 'pending', - }) - .onConflictDoNothing() - .returning(); + const deploymentId = runInputData.deploymentId; + const workflowName = runInputData.workflowName; + const input = runInputData.input; + const startHook = runInputData.experimentalStartHook; + const sameRunStartClaim = startHook + ? await getOwnStartHookClaim(startHook.token, effectiveRunId) + : null; + // Create run + optional start claim + run_created event + // atomically. The transaction ensures we never have an + // orphaned run without its run_created event or start claim. + const inserted = await drizzle.transaction(async (tx) => { + const [createdRun] = await tx + .insert(Schema.runs) + .values({ + runId: effectiveRunId, + deploymentId, + workflowName, + specVersion: effectiveSpecVersion, + input: input as SerializedContent, + executionContext: runInputData.executionContext as + | SerializedContent + | undefined, + attributes: runInputData.attributes, + status: 'pending', + }) + .onConflictDoNothing() + .returning(); + if (!createdRun) return undefined; + + if (startHook && !sameRunStartClaim) { + await claimStartHookTokenInTx(tx, startHook, effectiveRunId); + } - if (inserted) { const runCreatedEventId = `wevt_${ulid()}`; - await drizzle.insert(events).values({ + await tx.insert(events).values({ runId: effectiveRunId, eventId: runCreatedEventId, eventType: 'run_created', eventData: { - deploymentId: runInputData.deploymentId, - workflowName: runInputData.workflowName, - input: runInputData.input, + deploymentId, + workflowName, + input, executionContext: runInputData.executionContext, attributes: runInputData.attributes, allowReservedAttributes: runInputData.allowReservedAttributes, + experimentalStartHook: startHook, }, specVersion: effectiveSpecVersion, }); - } + return createdRun; + }); const createdRun = inserted; if (createdRun) { @@ -772,6 +1003,7 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { executionContext?: Record; attributes?: Record; allowReservedAttributes?: true; + experimentalStartHook?: ExperimentalStartHook; }; validateAttributeChanges( Object.entries(eventData.attributes ?? {}).map(([key, value]) => ({ @@ -782,23 +1014,69 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { allowReservedAttributes: eventData.allowReservedAttributes === true, } ); - const [runValue] = await drizzle - .insert(Schema.runs) - .values({ - runId: effectiveRunId, - deploymentId: eventData.deploymentId, - workflowName: eventData.workflowName, - // Propagate specVersion from the event to the run entity - specVersion: effectiveSpecVersion, - input: eventData.input as SerializedContent, - executionContext: eventData.executionContext as - | SerializedContent - | undefined, - attributes: eventData.attributes, - status: 'pending', - }) - .onConflictDoNothing() - .returning(); + const pendingRun = { + runId: effectiveRunId, + deploymentId: eventData.deploymentId, + workflowName: eventData.workflowName, + // Propagate specVersion from the event to the run entity + specVersion: effectiveSpecVersion, + input: eventData.input as SerializedContent, + executionContext: eventData.executionContext as + | SerializedContent + | undefined, + attributes: eventData.attributes, + status: 'pending' as const, + }; + const startHook = eventData.experimentalStartHook; + const runValues = startHook + ? await (async () => { + // Claim + run + event are committed in one transaction, so a + // live same-run claim means this exact run_created was + // already admitted — a duplicate delivery. + const sameRunStartClaim = await getOwnStartHookClaim( + startHook.token, + effectiveRunId + ); + if (sameRunStartClaim) { + throw new EntityConflictError( + `Run "${effectiveRunId}" already exists` + ); + } + const result = await drizzle.transaction(async (tx) => { + await claimStartHookTokenInTx(tx, startHook, effectiveRunId); + + const insertedRuns = await tx + .insert(Schema.runs) + .values(pendingRun) + .onConflictDoNothing() + .returning(); + if (!insertedRuns[0]) { + throw new EntityConflictError( + `Run "${effectiveRunId}" already exists` + ); + } + const [createdEvent] = await tx + .insert(events) + .values({ + runId: effectiveRunId, + eventId, + correlationId: data.correlationId, + eventType: data.eventType, + eventData: data.eventData, + specVersion: effectiveSpecVersion, + }) + .returning({ createdAt: events.createdAt }); + return { event: createdEvent, runs: insertedRuns }; + }); + eventCreatedWithRun = result.event; + return result.runs; + })() + : await drizzle + .insert(Schema.runs) + .values(pendingRun) + .onConflictDoNothing() + .returning(); + const [runValue] = runValues; if (runValue) { run = deserializeRunError(compact(runValue)); } @@ -877,7 +1155,8 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { ); } } - // Delete all hooks and waits for this run to allow token reuse + // Close active hooks/waits; retained start-hook claims expire separately. + await retainHookClaimsForRun(effectiveRunId, now); await Promise.all([ drizzle .delete(Schema.hooks) @@ -929,7 +1208,8 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { ); } } - // Delete all hooks and waits for this run to allow token reuse + // Close active hooks/waits; retained start-hook claims expire separately. + await retainHookClaimsForRun(effectiveRunId, now); await Promise.all([ drizzle .delete(Schema.hooks) @@ -974,7 +1254,13 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { ); } } - // Delete all hooks and waits for this run to allow token reuse + // Close active hooks/waits. Cancellation releases start-hook claims + // the workflow never materialized into a hook, so cancel-then-retry + // (including start()'s own cleanup when queueing fails after + // admission) can reuse the token immediately. + await retainHookClaimsForRun(effectiveRunId, now, { + releaseUnmaterializedClaims: true, + }); await Promise.all([ drizzle .delete(Schema.hooks) @@ -1352,8 +1638,8 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { } } - // Handle hook_created event: create hook entity - // Uses prepared statement for token uniqueness check (performance optimization) + // Handle hook_created event: create the hook entity and take (or + // materialize) the token claim. if (data.eventType === 'hook_created') { const eventData = (data as any).eventData as { token: string; @@ -1362,142 +1648,254 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { isSystem?: boolean; }; - // Check for duplicate token using prepared statement - const [existingHook] = await getHookByToken.execute({ - token: eventData.token, - }); - if (existingHook) { - // Idempotency: if the existing hook is the *same* (runId, hookId) - // we are trying to create, this is either a duplicate / replayed - // processing of the same hook_created (not a real conflict), or - // an orphaned hook row from a prior crashed attempt (the hook - // INSERT below landed but the events INSERT below didn't — - // these writes are not in one transaction). Distinguish by - // checking whether the `hook_created` event actually exists in - // the event log: - // - exists → real duplicate: throw EntityConflictError so the - // runtime's concurrent-replay catch path (matching the - // step_created path) swallows it, instead of producing a - // self-conflict in the event log that would later replay - // as HookConflictError. - // See https://github.com/vercel/workflow/issues/2283. - // - missing → orphaned hook row (crash between hook INSERT - // and events INSERT): skip the hook insert (the existing - // row already has the desired state) and fall through to - // the events INSERT below, completing the partial write. - if ( - existingHook.runId === effectiveRunId && - existingHook.hookId === data.correlationId - ) { - const [existingEvent] = await getHookCreatedEvent.execute({ + // Emit a hook_conflict event instead of throwing 409 — this lets + // the workflow continue and fail gracefully when the hook is + // awaited. + const emitHookConflict = async (conflictingRunId?: string) => { + const conflictEventData = { + token: eventData.token, + ...(conflictingRunId ? { conflictingRunId } : {}), + }; + const [conflictValue] = await drizzle + .insert(events) + .values({ runId: effectiveRunId, + eventId, correlationId: data.correlationId, - eventType: 'hook_created', - }); - if (existingEvent) { - throw new EntityConflictError( - `Hook "${data.correlationId}" already created` - ); - } - // Orphaned hook row: hook row exists but no hook_created - // event in the log. Skip the hook insert below (the row - // already exists with our (runId, hookId)) and let the - // outer code path emit the hook_created event, completing - // the partial write. We also re-fetch the existing hook - // row so the EventResult carries the actual persisted - // entity rather than `undefined`. - const [recoveredHookValue] = await drizzle - .select() - .from(Schema.hooks) - .where(eq(Schema.hooks.hookId, data.correlationId!)) - .limit(1); - if (recoveredHookValue) { - recoveredHookValue.metadata ||= recoveredHookValue.metadataJson; - hook = HookSchema.parse(compact(recoveredHookValue)); - } - } else { - // Cross-hook / cross-run conflict: a different - // (runId, hookId) holds this token. Create a hook_conflict - // event instead of throwing 409 — this lets the workflow - // continue and fail gracefully when the hook is awaited. - const conflictEventData = { - token: eventData.token, - conflictingRunId: existingHook.runId, - }; - - const [conflictValue] = await drizzle - .insert(events) - .values({ - runId: effectiveRunId, - eventId, - correlationId: data.correlationId, - eventType: 'hook_conflict', - eventData: conflictEventData, - specVersion: effectiveSpecVersion, - }) - .returning({ createdAt: events.createdAt }); + eventType: 'hook_conflict', + eventData: conflictEventData, + specVersion: effectiveSpecVersion, + }) + .returning({ createdAt: events.createdAt }); + if (!conflictValue) { + throw new EntityConflictError( + `Event ${eventId} could not be created` + ); + } + const parsedConflict = EventSchema.parse({ + eventType: 'hook_conflict' as const, + correlationId: data.correlationId, + eventData: conflictEventData, + ...conflictValue, + runId: effectiveRunId, + eventId, + specVersion: effectiveSpecVersion, + }); + const resolveData = params?.resolveData ?? 'all'; + return { + event: stripEventDataRefs(parsedConflict, resolveData), + run, + step, + hook: undefined, + }; + }; - if (!conflictValue) { + // Snapshot current token ownership. getLiveHookClaim lazily + // reclaims dead claims, so a `null` claim means the token is free. + const [[existingHook], existingClaim] = await Promise.all([ + getHookByToken.execute({ token: eventData.token }), + getLiveHookClaim(eventData.token), + ]); + + const isOwnHookRow = (row: typeof existingHook) => + row?.runId === effectiveRunId && row.hookId === data.correlationId; + + // Adopt our own already-persisted hook row: either a replayed + // duplicate of this hook_created (the hook_created event exists → + // throw EntityConflictError so the runtime's dedup catch path + // swallows it instead of recording a self-conflict, see + // https://github.com/vercel/workflow/issues/2283), or an orphaned + // row from a crash between the entity write and the events INSERT + // (event missing → reuse the row and fall through to the events + // INSERT below, completing the partial write). + const adoptOwnHookRow = async () => { + const [existingEvent] = await getHookCreatedEvent.execute({ + runId: effectiveRunId, + correlationId: data.correlationId, + eventType: 'hook_created', + }); + if (existingEvent) { + throw new EntityConflictError( + `Hook "${data.correlationId}" already created` + ); + } + // Repair the claim if it does not reflect the materialized hook + // yet (rows written before the claim update and hook insert were + // transactional). + await drizzle + .update(Schema.hookClaims) + .set({ hookId: data.correlationId! }) + .where( + and( + eq(Schema.hookClaims.token, eventData.token), + eq(Schema.hookClaims.runId, effectiveRunId), + isNull(Schema.hookClaims.hookId) + ) + ); + // Re-fetch the full row so the EventResult carries the actual + // persisted entity. + const [recoveredHookValue] = await drizzle + .select() + .from(Schema.hooks) + .where(eq(Schema.hooks.hookId, data.correlationId!)) + .limit(1); + if (recoveredHookValue) { + recoveredHookValue.metadata ||= recoveredHookValue.metadataJson; + hook = HookSchema.parse(compact(recoveredHookValue)); + } + }; + + if (isOwnHookRow(existingHook)) { + await adoptOwnHookRow(); + } else if (existingHook) { + // A different (runId, hookId) hook entity holds this token. + return await emitHookConflict(existingHook.runId); + } else if ( + existingClaim && + !( + existingClaim.runId === effectiveRunId && + existingClaim.hookId === null && + existingClaim.expiresAt === null + ) + ) { + // The token is held by a claim we cannot materialize: another + // run's claim, or our own claim already bound to a different hook + // or retained after disposal. + return await emitHookConflict(existingClaim.runId); + } else { + // The token is free, or reserved for this run by start(): create + // the hook row and take (or materialize) the claim in one + // transaction. ttlSeconds 0 marks a plain createHook guard with + // no retention window. + const materializesStartClaim = existingClaim !== null; + const hookValues = { + runId: effectiveRunId, + hookId: data.correlationId!, + token: eventData.token, + metadata: eventData.metadata as SerializedContent, + ownerId: '', // TODO: get from context + projectId: '', // TODO: get from context + environment: '', // TODO: get from context + // Propagate specVersion from the event to the hook entity + specVersion: effectiveSpecVersion, + isWebhook: eventData.isWebhook, + isSystem: eventData.isSystem ?? false, + }; + const hookValue = await drizzle.transaction(async (tx) => { + const [claimed] = materializesStartClaim + ? await tx + .update(Schema.hookClaims) + .set({ hookId: data.correlationId! }) + .where( + and( + eq(Schema.hookClaims.token, eventData.token), + eq(Schema.hookClaims.runId, effectiveRunId), + isNull(Schema.hookClaims.hookId), + isNull(Schema.hookClaims.expiresAt) + ) + ) + .returning({ token: Schema.hookClaims.token }) + : await tx + .insert(Schema.hookClaims) + .values({ + token: eventData.token, + runId: effectiveRunId, + hookId: data.correlationId!, + ttlSeconds: 0, + }) + .onConflictDoNothing() + .returning({ token: Schema.hookClaims.token }); + if (!claimed) return undefined; + const [createdHook] = await tx + .insert(Schema.hooks) + .values(hookValues) + .onConflictDoNothing() + .returning(); + if (!createdHook) { throw new EntityConflictError( - `Event ${eventId} could not be created` + `Hook "${data.correlationId}" already created` ); } + return createdHook; + }); - const conflictResult = { - eventType: 'hook_conflict' as const, - correlationId: data.correlationId, - eventData: conflictEventData, - ...conflictValue, - runId: effectiveRunId, - eventId, - }; - const parsedConflict = EventSchema.parse(conflictResult); - const resolveData = params?.resolveData ?? 'all'; - return { - event: stripEventDataRefs(parsedConflict, resolveData), - run, - step, - hook: undefined, - }; - } - } else { - const [hookValue] = await drizzle - .insert(Schema.hooks) - .values({ - runId: effectiveRunId, - hookId: data.correlationId!, - token: eventData.token, - metadata: eventData.metadata as SerializedContent, - ownerId: '', // TODO: get from context - projectId: '', // TODO: get from context - environment: '', // TODO: get from context - // Propagate specVersion from the event to the hook entity - specVersion: effectiveSpecVersion, - isWebhook: eventData.isWebhook, - isSystem: eventData.isSystem ?? false, - }) - .onConflictDoNothing() - .returning(); if (hookValue) { hookValue.metadata ||= hookValue.metadataJson; hook = HookSchema.parse(compact(hookValue)); + } else { + // Lost the claim to a concurrent writer between the snapshot + // and the transaction. Re-read and converge: our own + // (runId, hookId) winning concurrently is a duplicate/orphan; + // anything else is a conflict. + const [[racedHook], racedClaim] = await Promise.all([ + getHookByToken.execute({ token: eventData.token }), + getLiveHookClaim(eventData.token), + ]); + if (isOwnHookRow(racedHook)) { + await adoptOwnHookRow(); + } else { + return await emitHookConflict( + racedHook?.runId ?? racedClaim?.runId + ); + } } } } - // Handle hook_disposed event: delete hook entity atomically. - // Uses DELETE ... RETURNING to ensure only one concurrent caller - // succeeds — if no rows are returned, the hook was already disposed. + // Handle hook_disposed event: delete the hook entity and settle its + // token claim atomically. The DELETE ... RETURNING ensures only one + // concurrent disposer succeeds (a losing tx rolls back its claim + // write); claims with a retention TTL stay behind as retained + // constraints so duplicate starts remain fenced, plain guards are + // released. if (data.eventType === 'hook_disposed' && data.correlationId) { - const [deleted] = await drizzle - .delete(Schema.hooks) - .where(eq(Schema.hooks.hookId, data.correlationId)) - .returning({ hookId: Schema.hooks.hookId }); - if (!deleted) { - throw new EntityConflictError( - `Hook "${data.correlationId}" already disposed` - ); - } + const hookId = data.correlationId; + await drizzle.transaction(async (tx) => { + const [disposed] = await tx + .delete(Schema.hooks) + .where(eq(Schema.hooks.hookId, hookId)) + .returning({ + runId: Schema.hooks.runId, + token: Schema.hooks.token, + createdAt: Schema.hooks.createdAt, + }); + if (!disposed) { + throw new EntityConflictError(`Hook "${hookId}" already disposed`); + } + // Settle the claim with two guarded statements instead of a + // SELECT-then-branch: release plain guards (no retention TTL), + // retain TTL-carrying claims until at least hookCreatedAt + ttl — + // never shrinking an existing window and never expiring before + // now. Both guard on the disposed hook's run so a claim owned by + // another run is left alone. For plain hooks (the common case) + // the UPDATE is a no-op index probe. + await tx + .delete(Schema.hookClaims) + .where( + and( + eq(Schema.hookClaims.token, disposed.token), + eq(Schema.hookClaims.runId, disposed.runId), + sql`${Schema.hookClaims.ttlSeconds} <= 0` + ) + ); + await tx + .update(Schema.hookClaims) + .set({ + hookId, + expiresAt: sql`GREATEST( + ${disposed.createdAt}::timestamp + make_interval(secs => ${Schema.hookClaims.ttlSeconds}), + COALESCE(${Schema.hookClaims.expiresAt}, ${now}::timestamp), + ${now}::timestamp + )`, + }) + .where( + and( + eq(Schema.hookClaims.token, disposed.token), + eq(Schema.hookClaims.runId, disposed.runId), + gt(Schema.hookClaims.ttlSeconds, 0) + ) + ); + }); } // Handle wait_created event: create wait entity @@ -1605,54 +2003,56 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { storedEventData = undefined; } - let value: { createdAt: Date } | undefined; - try { - [value] = await drizzle - .insert(events) - .values({ - runId: effectiveRunId, - eventId, - correlationId: data.correlationId, - eventType: data.eventType, - eventData: storedEventData, - specVersion: effectiveSpecVersion, - }) - .returning({ createdAt: events.createdAt }); - } catch (err) { - // Translate unique-violation on the correlated-event partial index - // (workflow_events_entity_creation_unique) into EntityConflictError - // so the runtime's existing dedup catch path can handle it. Without - // this, two concurrent invocations producing identical - // correlationIds (e.g. snapshot runtime deterministic ULIDs) would - // surface as unhandled DB errors instead of dedup signals. - // Drizzle wraps the underlying pg error in DrizzleQueryError; the - // pg error (with .code === '23505') lives on .cause. We additionally - // gate on the violated constraint name so other 23505 violations on - // these event types (e.g. the events primary key, or any future - // unique constraint we might add) don't get misclassified as a - // correlationId conflict. - const isDeduplicatedCorrelatedEvent = - data.eventType === 'step_created' || - data.eventType === 'hook_created' || - data.eventType === 'wait_created' || - (data.eventType === 'attr_set' && - data.eventData.writer.type === 'workflow'); - const pgErr = (err as { code?: string; constraint?: string }).code - ? (err as { code?: string; constraint?: string }) - : ((err as { cause?: { code?: string; constraint?: string } }) - .cause ?? {}); - const pgCode = pgErr.code; - const pgConstraint = pgErr.constraint; - if ( - isDeduplicatedCorrelatedEvent && - pgCode === '23505' && - pgConstraint === 'workflow_events_entity_creation_unique' - ) { - throw new EntityConflictError( - `${data.eventType} for correlationId "${data.correlationId}" already exists in run "${effectiveRunId}"` - ); + let value: { createdAt: Date } | undefined = eventCreatedWithRun; + if (!value) { + try { + [value] = await drizzle + .insert(events) + .values({ + runId: effectiveRunId, + eventId, + correlationId: data.correlationId, + eventType: data.eventType, + eventData: storedEventData, + specVersion: effectiveSpecVersion, + }) + .returning({ createdAt: events.createdAt }); + } catch (err) { + // Translate unique-violation on the correlated-event partial index + // (workflow_events_entity_creation_unique) into EntityConflictError + // so the runtime's existing dedup catch path can handle it. Without + // this, two concurrent invocations producing identical + // correlationIds (e.g. snapshot runtime deterministic ULIDs) would + // surface as unhandled DB errors instead of dedup signals. + // Drizzle wraps the underlying pg error in DrizzleQueryError; the + // pg error (with .code === '23505') lives on .cause. We additionally + // gate on the violated constraint name so other 23505 violations on + // these event types (e.g. the events primary key, or any future + // unique constraint we might add) don't get misclassified as a + // correlationId conflict. + const isDeduplicatedCorrelatedEvent = + data.eventType === 'step_created' || + data.eventType === 'hook_created' || + data.eventType === 'wait_created' || + (data.eventType === 'attr_set' && + data.eventData.writer.type === 'workflow'); + const pgErr = (err as { code?: string; constraint?: string }).code + ? (err as { code?: string; constraint?: string }) + : ((err as { cause?: { code?: string; constraint?: string } }) + .cause ?? {}); + const pgCode = pgErr.code; + const pgConstraint = pgErr.constraint; + if ( + isDeduplicatedCorrelatedEvent && + pgCode === '23505' && + pgConstraint === 'workflow_events_entity_creation_unique' + ) { + throw new EntityConflictError( + `${data.eventType} for correlationId "${data.correlationId}" already exists in run "${effectiveRunId}"` + ); + } + throw err; } - throw err; } if (!value) { throw new EntityConflictError(`Event ${eventId} could not be created`); diff --git a/packages/world-postgres/test/storage.test.ts b/packages/world-postgres/test/storage.test.ts index 2a92cabc51..ff9574c5ee 100644 --- a/packages/world-postgres/test/storage.test.ts +++ b/packages/world-postgres/test/storage.test.ts @@ -1,9 +1,12 @@ import { execSync } from 'node:child_process'; import { PostgreSqlContainer } from '@testcontainers/postgresql'; +import { EntityConflictError, HookConflictError } from '@workflow/errors'; import type { Hook, Step, WorkflowRun } from '@workflow/world'; import { SPEC_VERSION_CURRENT } from '@workflow/world'; import { encode } from 'cbor-x'; +import { eq } from 'drizzle-orm'; import { Pool } from 'pg'; +import { monotonicFactory } from 'ulid'; import { afterAll, beforeAll, @@ -34,6 +37,7 @@ async function createRun( input: Uint8Array; executionContext?: Record; attributes?: Record; + experimentalStartHook?: { token: string; ttlSeconds: number }; } ): Promise { const result = await events.create(null, { @@ -1491,6 +1495,435 @@ describe('Storage (Postgres integration)', () => { expect(result.hook).toBeUndefined(); }); + it('should reject duplicate start hook claims before hook materialization', async () => { + const token = 'postgres-start-claim-token'; + const first = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }); + + await expect( + createRun(events, { + deploymentId: 'deployment-456', + workflowName: 'test-workflow-2', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }) + ).rejects.toThrow(HookConflictError); + + await expect( + events.create(first.runId, { + eventType: 'hook_created' as const, + correlationId: 'hook_start', + eventData: { token }, + }) + ).resolves.toMatchObject({ + event: { eventType: 'hook_created' }, + hook: { + runId: first.runId, + hookId: 'hook_start', + token, + }, + }); + }); + + it('should reserve start hook claims during resilient run_started bootstrap', async () => { + const token = 'postgres-resilient-start-claim-token'; + const runId = `wrun_${monotonicFactory()()}`; + + const started = await events.create(runId, { + eventType: 'run_started' as const, + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }, + }); + + expect(started.run?.runId).toBe(runId); + await expect( + createRun(events, { + deploymentId: 'deployment-456', + workflowName: 'test-workflow-2', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }) + ).rejects.toThrow(HookConflictError); + + await expect( + events.create(runId, { + eventType: 'hook_created' as const, + correlationId: 'hook_resilient_start', + eventData: { token }, + }) + ).resolves.toMatchObject({ + event: { eventType: 'hook_created' }, + hook: { runId, hookId: 'hook_resilient_start', token }, + }); + }); + + it('should materialize only one hook for a reserved start hook token', async () => { + const token = 'postgres-concurrent-start-claim-materialize-token'; + const first = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }); + + const results = await Promise.all([ + events.create(first.runId, { + eventType: 'hook_created' as const, + correlationId: 'hook_start_a', + eventData: { token }, + }), + events.create(first.runId, { + eventType: 'hook_created' as const, + correlationId: 'hook_start_b', + eventData: { token }, + }), + ]); + + expect(results.map((result) => result.event.eventType).sort()).toEqual([ + 'hook_conflict', + 'hook_created', + ]); + const persistedHooks = await drizzle + .select() + .from(DrizzleSchema.hooks) + .where(eq(DrizzleSchema.hooks.token, token)); + expect(persistedHooks).toHaveLength(1); + }); + + it('should reject experimental starts when an active ordinary hook owns the token', async () => { + const token = 'postgres-active-ordinary-hook-token'; + await createHook(events, testRunId, { + hookId: 'hook_active_ordinary', + token, + }); + + await expect( + createRun(events, { + deploymentId: 'deployment-456', + workflowName: 'test-workflow-2', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }) + ).rejects.toThrow(HookConflictError); + }); + + it('should treat a same-run start hook retry as a run conflict', async () => { + const token = 'postgres-same-run-start-claim-token'; + const first = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }); + + await expect( + events.create(first.runId, { + eventType: 'run_created' as const, + eventData: { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }, + }) + ).rejects.toThrow(EntityConflictError); + }); + + it('should not duplicate the run_created event on a concurrent same-run retry', async () => { + const token = 'postgres-same-run-concurrent-start-claim-token'; + const request = { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }; + const runId = `wrun_${monotonicFactory()()}`; + + // Two deliveries of the same run_created (same runId + token) must + // admit exactly once; the loser gets EntityConflictError — not a + // HookConflictError against its own run. + const results = await Promise.allSettled([ + events.create(runId, { + eventType: 'run_created' as const, + eventData: request, + }), + events.create(runId, { + eventType: 'run_created' as const, + eventData: request, + }), + ]); + + const fulfilled = results.filter((r) => r.status === 'fulfilled'); + const rejected = results.filter( + (r): r is PromiseRejectedResult => r.status === 'rejected' + ); + expect(fulfilled).toHaveLength(1); + expect(rejected).toHaveLength(1); + expect(rejected[0]?.reason).toMatchObject({ + name: 'EntityConflictError', + }); + + const createdEvents = await drizzle + .select() + .from(DrizzleSchema.events) + .where(eq(DrizzleSchema.events.runId, runId)); + expect(createdEvents).toHaveLength(1); + }); + + it('should retain a disposed start hook claim until ttl expiry', async () => { + const token = 'postgres-disposed-start-claim-token'; + const first = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }); + + await events.create(first.runId, { + eventType: 'hook_created' as const, + correlationId: 'hook_start_disposed', + eventData: { token }, + }); + await events.create(first.runId, { + eventType: 'hook_disposed' as const, + correlationId: 'hook_start_disposed', + }); + + await expect( + createRun(events, { + deploymentId: 'deployment-456', + workflowName: 'test-workflow-2', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }) + ).rejects.toThrow(HookConflictError); + }); + + it('should not materialize a retained start hook claim again', async () => { + const token = 'postgres-retained-start-claim-token'; + const first = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }); + + await events.create(first.runId, { + eventType: 'hook_created' as const, + correlationId: 'hook_start_retained', + eventData: { token }, + }); + await events.create(first.runId, { + eventType: 'hook_disposed' as const, + correlationId: 'hook_start_retained', + }); + + const result = await events.create(first.runId, { + eventType: 'hook_created' as const, + correlationId: 'hook_start_retained_again', + eventData: { token }, + }); + + expect(result.event.eventType).toBe('hook_conflict'); + expect(result.hook).toBeUndefined(); + }); + + it('should retain a recovered start hook claim after orphaned hook row materialization', async () => { + const token = 'postgres-recovered-start-claim-token'; + const hookId = 'hook_recovered_start_claim'; + const first = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }); + + await drizzle.insert(DrizzleSchema.hooks).values({ + runId: first.runId, + hookId, + token, + ownerId: '', + projectId: '', + environment: '', + specVersion: SPEC_VERSION_CURRENT, + isWebhook: false, + isSystem: false, + }); + + await expect( + events.create(first.runId, { + eventType: 'hook_created' as const, + correlationId: hookId, + eventData: { token }, + }) + ).resolves.toMatchObject({ + event: { eventType: 'hook_created' }, + hook: { hookId, token }, + }); + + const [materializedClaim] = await drizzle + .select() + .from(DrizzleSchema.hookClaims) + .where(eq(DrizzleSchema.hookClaims.token, token)) + .limit(1); + expect(materializedClaim).toMatchObject({ + runId: first.runId, + hookId, + expiresAt: null, + }); + + await updateRun(events, first.runId, 'run_completed', { + output: new Uint8Array(), + }); + + const [retainedClaim] = await drizzle + .select() + .from(DrizzleSchema.hookClaims) + .where(eq(DrizzleSchema.hookClaims.token, token)) + .limit(1); + expect(retainedClaim).toMatchObject({ + runId: first.runId, + hookId, + }); + expect(retainedClaim?.expiresAt?.getTime()).toBeGreaterThan(Date.now()); + + await expect( + createRun(events, { + deploymentId: 'deployment-456', + workflowName: 'test-workflow-2', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }) + ).rejects.toThrow(HookConflictError); + }); + + it('should retain an unmaterialized start hook claim after run failure until ttl expiry', async () => { + const token = 'postgres-failed-unmaterialized-start-claim-token'; + const first = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }); + + await updateRun(events, first.runId, 'run_failed', { + error: new Uint8Array(), + }); + + await expect( + createRun(events, { + deploymentId: 'deployment-456', + workflowName: 'test-workflow-2', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }) + ).rejects.toThrow(HookConflictError); + + const past = new Date(Date.now() - 2_000); + await drizzle + .update(DrizzleSchema.hookClaims) + .set({ createdAt: past, expiresAt: past }) + .where(eq(DrizzleSchema.hookClaims.token, token)); + + await expect( + createRun(events, { + deploymentId: 'deployment-789', + workflowName: 'test-workflow-3', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }) + ).resolves.toMatchObject({ + workflowName: 'test-workflow-3', + }); + }); + + it('should release an unmaterialized start hook claim on cancellation', async () => { + const token = 'postgres-cancelled-unmaterialized-start-claim-token'; + const first = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }); + + // Cancel the claim-owning run before it materialized a hook: the + // token must be immediately reusable (cancel-then-retry, including + // start()'s own cleanup when queueing fails after admission). + await events.create(first.runId, { + eventType: 'run_cancelled' as const, + }); + + await expect( + createRun(events, { + deploymentId: 'deployment-456', + workflowName: 'test-workflow-2', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }) + ).resolves.toMatchObject({ + workflowName: 'test-workflow-2', + }); + }); + + it('should keep an expired disposed start hook claim while the run is active', async () => { + const token = 'postgres-active-expired-start-claim-token'; + const first = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 1 }, + }); + + await events.create(first.runId, { + eventType: 'hook_created' as const, + correlationId: 'hook_start_expired_active', + eventData: { token }, + }); + await events.create(first.runId, { + eventType: 'hook_disposed' as const, + correlationId: 'hook_start_expired_active', + }); + + const past = new Date(Date.now() - 2_000); + await drizzle + .update(DrizzleSchema.hookClaims) + .set({ createdAt: past, expiresAt: past }) + .where(eq(DrizzleSchema.hookClaims.token, token)); + + await expect( + createRun(events, { + deploymentId: 'deployment-456', + workflowName: 'test-workflow-2', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 1 }, + }) + ).rejects.toThrow(HookConflictError); + + await updateRun(events, first.runId, 'run_completed', { + output: new Uint8Array(), + }); + + await expect( + createRun(events, { + deploymentId: 'deployment-789', + workflowName: 'test-workflow-3', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 1 }, + }) + ).resolves.toMatchObject({ + workflowName: 'test-workflow-3', + }); + }); + it('should allow token reuse after hook is disposed', async () => { const token = 'reusable-token-test'; diff --git a/packages/world-vercel/src/events-v4.test.ts b/packages/world-vercel/src/events-v4.test.ts index 284a887ed6..aaf6cf08d4 100644 --- a/packages/world-vercel/src/events-v4.test.ts +++ b/packages/world-vercel/src/events-v4.test.ts @@ -1,5 +1,6 @@ import { EntityConflictError, + HookConflictError, RunExpiredError, ThrottleError, TooEarlyError, @@ -15,6 +16,19 @@ import { } from './events-v4.js'; import { encodeFrame, V4_FRAME_CONTENT_TYPE } from './frames.js'; +function decodePostedMeta(rawBody: unknown): Record { + const bytes = + typeof rawBody === 'string' + ? new TextEncoder().encode(rawBody) + : new Uint8Array(rawBody as ArrayBufferLike); + const metaLen = new DataView( + bytes.buffer, + bytes.byteOffset, + bytes.byteLength + ).getUint32(0, false); + return decode(bytes.subarray(4, 4 + metaLen)) as Record; +} + /** * The v4 client must preserve the typed-error contract of the v3 * `makeRequest` path — the workflow runtime branches on these types @@ -32,6 +46,25 @@ describe('throwForErrorResponse', () => { expect(() => call(409)).toThrowError(EntityConflictError); }); + it('maps hook-conflict 409s to HookConflictError', () => { + try { + call( + 409, + JSON.stringify({ + code: 'hook_conflict', + message: 'hook conflict', + token: 'order:123', + conflictingRunId: 'wrun_owner', + }) + ); + expect.unreachable(); + } catch (err) { + expect(HookConflictError.is(err)).toBe(true); + expect((err as HookConflictError).token).toBe('order:123'); + expect((err as HookConflictError).conflictingRunId).toBe('wrun_owner'); + } + }); + it('maps 410 to RunExpiredError (terminal run — runtime must not retry)', () => { expect(() => call(410)).toThrowError(RunExpiredError); }); @@ -349,24 +382,7 @@ describe('createWorkflowRunEventV4 over HTTP', () => { const agent = new MockAgent(); agent.disableNetConnect(); - // Decode the posted frame's CBOR meta block: - // u32_be(meta_len) || cbor_meta || u32_be(body_len) || body let capturedMeta: Record | undefined; - const captureMeta = (rawBody: unknown) => { - const bytes = - typeof rawBody === 'string' - ? new TextEncoder().encode(rawBody) - : new Uint8Array(rawBody as ArrayBufferLike); - const metaLen = new DataView( - bytes.buffer, - bytes.byteOffset, - bytes.byteLength - ).getUint32(0, false); - capturedMeta = decode(bytes.subarray(4, 4 + metaLen)) as Record< - string, - unknown - >; - }; agent .get(origin) @@ -377,7 +393,7 @@ describe('createWorkflowRunEventV4 over HTTP', () => { .reply( 200, (opts: { body?: unknown }) => { - captureMeta(opts.body); + capturedMeta = decodePostedMeta(opts.body); return encode({ run: { runId: 'wrun_1', status: 'running' } }); }, { @@ -404,6 +420,50 @@ describe('createWorkflowRunEventV4 over HTTP', () => { agent.assertNoPendingInterceptors(); }); + it('forwards experimentalStartHook in the frame meta', async () => { + const origin = 'https://vercel-workflow.com'; + const agent = new MockAgent(); + agent.disableNetConnect(); + + let capturedMeta: Record | undefined; + const startHook = { token: 'order:123', ttlSeconds: 60 }; + + agent + .get(origin) + .intercept({ + path: '/api/v4/runs/wrun_1/events/run_created', + method: 'POST', + }) + .reply( + 200, + (opts: { body?: unknown }) => { + capturedMeta = decodePostedMeta(opts.body); + return encode({ run: { runId: 'wrun_1', status: 'pending' } }); + }, + { + headers: { + 'x-wf-event-id': 'evnt_1', + 'x-wf-run-id': 'wrun_1', + 'x-wf-created-at': '2026-06-10T00:00:00.000Z', + }, + } + ); + + await createWorkflowRunEventV4( + { + runId: 'wrun_1', + eventType: 'run_created', + specVersion: 5, + experimentalStartHook: startHook, + }, + { token: 'test-token', dispatcher: agent } + ); + + expect(capturedMeta?.eventType).toBe('run_created'); + expect(capturedMeta?.experimentalStartHook).toEqual(startHook); + agent.assertNoPendingInterceptors(); + }); + it('omits skipPreload from the frame meta when not set (default / old SDK parity)', async () => { const origin = 'https://vercel-workflow.com'; const agent = new MockAgent(); diff --git a/packages/world-vercel/src/events-v4.ts b/packages/world-vercel/src/events-v4.ts index 41532605ce..5d8aa4b071 100644 --- a/packages/world-vercel/src/events-v4.ts +++ b/packages/world-vercel/src/events-v4.ts @@ -21,6 +21,7 @@ * bytes — this module stays at the wire-bytes layer. */ +import type { ExperimentalStartHook } from '@workflow/world'; import { decode } from 'cbor-x'; import { decodeFrames, encodeFrame, V4_FRAME_CONTENT_TYPE } from './frames.js'; import { getEventsDispatcher } from './http-client.js'; @@ -136,6 +137,9 @@ export interface CreateEventV4Input { * resilient-start path). Validated server-side against the attribute * key/value/count caps. */ attributes?: Record; + /** Experimental start-hook token claim carried by run_created and the + * resilient-start run_started fallback. */ + experimentalStartHook?: ExperimentalStartHook; /** attr_set's attribute change list ({key, value|null} entries). */ changes?: Array>; /** attr_set's writer provenance ({type:'workflow'} or @@ -219,6 +223,9 @@ function buildPostFrameMeta( meta.executionContext = input.executionContext; } if (input.attributes !== undefined) meta.attributes = input.attributes; + if (input.experimentalStartHook !== undefined) { + meta.experimentalStartHook = input.experimentalStartHook; + } if (input.changes !== undefined) meta.changes = input.changes; if (input.writer !== undefined) meta.writer = input.writer; if (input.allowReservedAttributes !== undefined) { @@ -247,10 +254,21 @@ function errorFromV4Response( ): Error { let message = `v4 ${opName} failed: HTTP ${statusCode}`; let code: string | undefined; + let token: string | undefined; + let conflictingRunId: string | undefined; try { - const json = JSON.parse(errorBody) as { message?: string; code?: string }; + const json = JSON.parse(errorBody) as { + message?: string; + code?: string; + token?: string; + conflictingRunId?: string; + }; if (typeof json.message === 'string') message = json.message; if (typeof json.code === 'string') code = json.code; + if (typeof json.token === 'string') token = json.token; + if (typeof json.conflictingRunId === 'string') { + conflictingRunId = json.conflictingRunId; + } } catch { // body wasn't JSON — keep the default message, append raw text below if (errorBody) message += ` ${errorBody}`; @@ -265,6 +283,8 @@ function errorFromV4Response( retryAfter, code, url, + token, + conflictingRunId, mitigated: readHeader(responseHeaders, 'x-vercel-mitigated'), }); } diff --git a/packages/world-vercel/src/events.test.ts b/packages/world-vercel/src/events.test.ts index 2faba17bd0..5d77a7fb08 100644 --- a/packages/world-vercel/src/events.test.ts +++ b/packages/world-vercel/src/events.test.ts @@ -177,6 +177,34 @@ describe('splitEventDataForV4 attribute fields', () => { expect(meta.attributes).toEqual({ sourceAtStart: 'api' }); }); + it('carries experimental start hooks on run_created and resilient-start run_started', () => { + const startHook = { token: 'order:123', ttlSeconds: 60 }; + + const created = splitEventDataForV4({ + eventType: 'run_created', + specVersion: 4, + eventData: { + deploymentId: 'dpl_1', + workflowName: 'wf', + input: new TextEncoder().encode('[]'), + experimentalStartHook: startHook, + }, + } as AnyEventRequest); + const started = splitEventDataForV4({ + eventType: 'run_started', + specVersion: 4, + eventData: { + input: new TextEncoder().encode('[]'), + deploymentId: 'dpl_1', + workflowName: 'wf', + experimentalStartHook: startHook, + }, + } as AnyEventRequest); + + expect(created.meta.experimentalStartHook).toEqual(startHook); + expect(started.meta.experimentalStartHook).toEqual(startHook); + }); + it('lifts workflowName into the frame meta on outcome events (step_completed/step_created), keeping the payload in the body', () => { // The backend keys payload refs by workflow name; carrying it in the // frame meta lets the v4 POST handler skip the per-step run lookup. diff --git a/packages/world-vercel/src/events.ts b/packages/world-vercel/src/events.ts index 41dcd0d1c3..6b082befac 100644 --- a/packages/world-vercel/src/events.ts +++ b/packages/world-vercel/src/events.ts @@ -40,6 +40,7 @@ import { type EventResult, EventSchema, EventTypeSchema, + type ExperimentalStartHook, type GetEventParams, type ListEventsByCorrelationIdParams, type ListEventsParams, @@ -184,6 +185,8 @@ interface SplitEventData { executionContext?: Record; /** Initial run attributes (run_created / resilient-start run_started). */ attributes?: Record; + /** Experimental start-hook token claim (run_created / resilient start). */ + experimentalStartHook?: ExperimentalStartHook; /** attr_set change list, included verbatim in frame meta. */ changes?: Array>; /** attr_set writer provenance, included verbatim in frame meta. */ @@ -215,6 +218,7 @@ type MetaSourceField = | 'errorCode' | 'executionContext' | 'attributes' + | 'experimentalStartHook' | 'changes' | 'writer' | 'allowReservedAttributes'; @@ -331,6 +335,14 @@ export function splitEventDataForV4(data: AnyEventRequest): SplitEventData { ) { meta.attributes = eventData.attributes as Record; } + if ( + eventData.experimentalStartHook !== undefined && + eventData.experimentalStartHook !== null && + typeof eventData.experimentalStartHook === 'object' + ) { + meta.experimentalStartHook = + eventData.experimentalStartHook as ExperimentalStartHook; + } if (Array.isArray(eventData.changes)) { meta.changes = eventData.changes as Array>; } diff --git a/packages/world-vercel/src/http-core.ts b/packages/world-vercel/src/http-core.ts index 18c5dff6e6..4e535372b9 100644 --- a/packages/world-vercel/src/http-core.ts +++ b/packages/world-vercel/src/http-core.ts @@ -19,6 +19,7 @@ import { getVercelOidcToken } from '@vercel/oidc'; import { EntityConflictError, + HookConflictError, RunExpiredError, ThrottleError, TooEarlyError, @@ -165,9 +166,19 @@ export function errorForResponse( code?: string; url?: string; mitigated?: string | null; + /** Hook token from a 409 `hook_conflict` response body. */ + token?: string; + /** Owning run from a 409 `hook_conflict` response body. */ + conflictingRunId?: string; } = {} ): Error { const { retryAfter, code, url, mitigated } = opts; + // A 409 carrying the server's hook_conflict code is a start-hook token + // conflict, not a generic entity conflict — surface it as the typed error + // the runtime and start() branch on. + if (status === 409 && code === 'hook_conflict') { + return new HookConflictError(opts.token ?? '', opts.conflictingRunId); + } if (status === 409) return new EntityConflictError(message); if (status === 410) return new RunExpiredError(message); if (status === 425) return new TooEarlyError(message, { retryAfter }); diff --git a/packages/world-vercel/src/index.ts b/packages/world-vercel/src/index.ts index 17e13f3c54..33c795d3f9 100644 --- a/packages/world-vercel/src/index.ts +++ b/packages/world-vercel/src/index.ts @@ -9,6 +9,9 @@ import { createStorage } from './storage.js'; import { createStreamer } from './streamer.js'; import type { APIConfig } from './utils.js'; +const EXPERIMENTAL_START_HOOK_TTL_MAX_SECONDS = 30 * 24 * 60 * 60; +const EXPERIMENTAL_START_HOOK_TOKEN_MAX_BYTES = 255; + export { createAnalytics } from './analytics.js'; export { createGetEncryptionKeyForRun, @@ -41,6 +44,11 @@ export function createVercelWorld(config?: APIConfig): World { // `process.exit(1)` is an acceptable response to an exhausted replay // budget. processExitTriggersQueueRedelivery: true, + experimentalStartHookAdmission: { + mode: 'queue-first', + maxTtlSeconds: EXPERIMENTAL_START_HOOK_TTL_MAX_SECONDS, + maxTokenBytes: EXPERIMENTAL_START_HOOK_TOKEN_MAX_BYTES, + }, ...createQueue(config), ...createStorage(config), analytics: createAnalytics(config), diff --git a/packages/world-vercel/src/utils.ts b/packages/world-vercel/src/utils.ts index aaaeee0cdf..5134f3fc74 100644 --- a/packages/world-vercel/src/utils.ts +++ b/packages/world-vercel/src/utils.ts @@ -412,13 +412,16 @@ export async function makeRequest({ }); if (!response.ok) { - const errorData: { message?: string; code?: string; error?: string } = - await parseResponseBody(response) - .then( - (r) => - r.data as { message?: string; code?: string; error?: string } - ) - .catch(() => ({})); + type ErrorBody = { + message?: string; + code?: string; + error?: string; + token?: string; + conflictingRunId?: string; + }; + const errorData: ErrorBody = await parseResponseBody(response) + .then((r) => r.data as ErrorBody) + .catch(() => ({})); const errorCode = errorData.code ?? errorData.error; logCurlRepro(request.method, url, headers); @@ -440,6 +443,8 @@ export async function makeRequest({ url, code: errorCode, retryAfter, + token: errorData.token, + conflictingRunId: errorData.conflictingRunId, mitigated: response.headers.get('x-vercel-mitigated'), }); span?.setAttributes({ diff --git a/packages/world/src/events.ts b/packages/world/src/events.ts index 00d7eb017d..2fc33d0b7c 100644 --- a/packages/world/src/events.ts +++ b/packages/world/src/events.ts @@ -276,6 +276,20 @@ const AttrSetEventSchema = BaseEventSchema.extend({ }), }); +/** + * Start-hook claim data: `start()` reserves `token` atomically with run + * admission, fencing duplicate starts for `ttlSeconds` past hook creation / + * run completion. Rides on `run_created` (and the resilient-start + * `run_started` fallback + queue runInput) so every admission path validates + * the same shape. + */ +export const ExperimentalStartHookSchema = z.object({ + token: z.string().min(1), + ttlSeconds: z.number().int().positive(), +}); + +export type ExperimentalStartHook = z.infer; + // ============================================================================= // Run lifecycle events // ============================================================================= @@ -293,6 +307,7 @@ const RunCreatedEventSchema = BaseEventSchema.extend({ executionContext: z.record(z.string(), z.any()).optional(), attributes: z.record(z.string(), z.string()).optional(), allowReservedAttributes: z.literal(true).optional(), + experimentalStartHook: ExperimentalStartHookSchema.optional(), }), }); @@ -315,6 +330,7 @@ const RunStartedEventSchema = BaseEventSchema.extend({ executionContext: z.record(z.string(), z.any()).optional(), attributes: z.record(z.string(), z.string()).optional(), allowReservedAttributes: z.literal(true).optional(), + experimentalStartHook: ExperimentalStartHookSchema.optional(), }) .optional(), }); diff --git a/packages/world/src/index.ts b/packages/world/src/index.ts index 8052636320..2e8189477d 100644 --- a/packages/world/src/index.ts +++ b/packages/world/src/index.ts @@ -27,6 +27,7 @@ export { EVENT_DATA_REF_FIELDS, EventSchema, EventTypeSchema, + ExperimentalStartHookSchema, stripEventDataRefs, } from './events.js'; export type * from './hooks.js'; diff --git a/packages/world/src/interfaces.ts b/packages/world/src/interfaces.ts index fa045f93e3..f2cb40dfd8 100644 --- a/packages/world/src/interfaces.ts +++ b/packages/world/src/interfaces.ts @@ -293,6 +293,14 @@ export interface World extends Queue, Streamer, Storage { */ specVersion: number; + /** + * How this World atomically reserves `run_created.eventData.experimentalStartHook` + * tokens with run admission. + */ + experimentalStartHookAdmission?: + | { mode: 'event-first' } + | { mode: 'queue-first'; maxTtlSeconds: number; maxTokenBytes: number }; + /** * Whether calling `process.exit(1)` from a queue handler is observed by * the World as a delivery failure that will be retried. diff --git a/packages/world/src/queue.ts b/packages/world/src/queue.ts index f60ae034a7..e0765eff57 100644 --- a/packages/world/src/queue.ts +++ b/packages/world/src/queue.ts @@ -115,6 +115,15 @@ export const RunInputSchema = z.object({ executionContext: z.record(z.string(), z.any()).optional(), /** Initial plaintext run attributes, for resilient run creation. */ attributes: z.record(z.string(), z.string()).optional(), + /** Start-hook claim data, for resilient run creation. Keep aligned with + * `ExperimentalStartHookSchema` in events.ts (this file uses `zod/v4`, so + * the schema instance cannot be shared). */ + experimentalStartHook: z + .object({ + token: z.string().min(1), + ttlSeconds: z.number().int().positive(), + }) + .optional(), /** * Permits reserved `$`-prefixed keys in `attributes`, mirrored from the * `start()` option so resilient run creation validates the same way as diff --git a/workbench/example/workflows/99_e2e.ts b/workbench/example/workflows/99_e2e.ts index 304578eebb..d7c1287fc0 100644 --- a/workbench/example/workflows/99_e2e.ts +++ b/workbench/example/workflows/99_e2e.ts @@ -778,6 +778,26 @@ export async function hookClaimOnlyMutexWorkflow( }; } +export async function experimentalStartHookWorkflow(token: string) { + 'use workflow'; + + using hook = createHook<{ message: string }>({ token }); + + const conflict = await hook.getConflict(); + if (conflict) { + return { + role: 'duplicate' as const, + conflictRunId: conflict.runId, + }; + } + + const payload = await hook; + return { + role: 'owner' as const, + message: payload.message, + }; +} + /** * "Adopt the owner's result" strategy: the duplicate run waits for the * active owner to finish and returns the owner's result, so callers