From de1af624a1de7cbbf3f8b3847609f80d165629be Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Sat, 27 Jun 2026 04:58:19 -0700 Subject: [PATCH 1/4] Add experimental idempotent start hooks --- .changeset/atomic-start-hooks.md | 9 + packages/core/e2e/e2e.test.ts | 32 + packages/core/src/runtime.ts | 2 + packages/core/src/runtime/start.test.ts | 223 ++++- packages/core/src/runtime/start.ts | 324 +++++-- packages/world-local/src/index.ts | 35 + packages/world-local/src/storage.test.ts | 582 +++++++++++- .../world-local/src/storage/events-storage.ts | 398 ++++++++- .../world-local/src/storage/hooks-storage.ts | 73 +- packages/world-local/src/tag.test.ts | 82 ++ .../migrations/0015_add_hook_claims.sql | 13 + .../src/drizzle/migrations/meta/_journal.json | 7 + packages/world-postgres/src/drizzle/schema.ts | 16 + packages/world-postgres/src/index.ts | 1 + packages/world-postgres/src/storage.ts | 841 ++++++++++++++---- packages/world-postgres/test/storage.test.ts | 407 +++++++++ packages/world-vercel/src/events.ts | 11 +- packages/world/src/events.ts | 7 + packages/world/src/interfaces.ts | 6 + packages/world/src/queue.ts | 7 + workbench/example/workflows/99_e2e.ts | 20 + 21 files changed, 2834 insertions(+), 262 deletions(-) create mode 100644 .changeset/atomic-start-hooks.md create mode 100644 packages/world-postgres/src/drizzle/migrations/0015_add_hook_claims.sql diff --git a/.changeset/atomic-start-hooks.md b/.changeset/atomic-start-hooks.md new file mode 100644 index 0000000000..08deadd0d5 --- /dev/null +++ b/.changeset/atomic-start-hooks.md @@ -0,0 +1,9 @@ +--- +"@workflow/core": minor +"@workflow/world": minor +"@workflow/world-local": minor +"@workflow/world-postgres": minor +"@workflow/world-vercel": patch +--- + +Add experimental atomic start-hook idempotency for worlds that opt into start-time hook admission. diff --git a/packages/core/e2e/e2e.test.ts b/packages/core/e2e/e2e.test.ts index a0c6c0d719..cbbd57fc40 100644 --- a/packages/core/e2e/e2e.test.ts +++ b/packages/core/e2e/e2e.test.ts @@ -1790,6 +1790,38 @@ describe('e2e', () => { } ); + test.skipIf(!!process.env.WORKFLOW_VERCEL_ENV)( + 'experimentalWithHook rejects duplicate starts before hook materialization', + { timeout: 60_000 }, + async () => { + const token = Math.random().toString(36).slice(2); + const workflow = await e2e('experimentalStartHookWorkflow'); + const startOptions = { + experimentalWithHook: { + token, + experimentalTtl: 60_000, + }, + }; + + const run = await start(workflow, [token], startOptions); + + const duplicateError = await start(workflow, [token], startOptions).catch( + (error: unknown) => error + ); + expect(HookConflictError.is(duplicateError)).toBe(true); + assert(HookConflictError.is(duplicateError)); + expect(duplicateError.conflictingRunId).toBe(run.runId); + + const hook = await waitForHook(token, { runId: run.runId }); + await resumeHook(hook, { message: 'accepted' }); + + await expect(run.returnValue).resolves.toEqual({ + role: 'owner', + message: 'accepted', + }); + } + ); + test('hookGetConflictWorkflow - awaiting hook.getConflict() registers hook without payload', async () => { const token = Math.random().toString(36).slice(2); const customData = Math.random().toString(36).slice(2); diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 49b2dd5281..29490bd5e3 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -763,6 +763,8 @@ export function workflowEntrypoint( workflowName: runInput.workflowName, executionContext: runInput.executionContext, attributes: runInput.attributes, + experimentalStartHook: + runInput.experimentalStartHook, allowReservedAttributes: runInput.allowReservedAttributes, }, diff --git a/packages/core/src/runtime/start.test.ts b/packages/core/src/runtime/start.test.ts index 1b3968f861..df244cc419 100644 --- a/packages/core/src/runtime/start.test.ts +++ b/packages/core/src/runtime/start.test.ts @@ -1,4 +1,9 @@ -import { WorkflowRuntimeError, WorkflowWorldError } from '@workflow/errors'; +import { + EntityConflictError, + HookConflictError, + WorkflowRuntimeError, + WorkflowWorldError, +} from '@workflow/errors'; import { SPEC_VERSION_CURRENT, SPEC_VERSION_LEGACY, @@ -106,6 +111,7 @@ describe('start', () => { }); afterEach(() => { + vi.useRealTimers(); setWorld(undefined); vi.clearAllMocks(); }); @@ -134,6 +140,7 @@ describe('start', () => { // Mock world with specVersion 3 → uses it setWorld({ specVersion: SPEC_VERSION_CURRENT, + supportsExperimentalStartHook: true, getDeploymentId: vi.fn().mockResolvedValue('deploy_123'), events: { create: mockEventsCreate }, queue: mockQueue, @@ -226,6 +233,220 @@ describe('start', () => { ); }); + it('seeds experimental start hook data and queues only after run creation succeeds', async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-01-01T00:00:00.000Z')); + + const validWorkflow = Object.assign(() => Promise.resolve('result'), { + workflowId: 'test-workflow', + }); + setWorld({ + specVersion: SPEC_VERSION_CURRENT, + supportsExperimentalStartHook: true, + getDeploymentId: vi.fn().mockResolvedValue('deploy_123'), + events: { create: mockEventsCreate }, + queue: mockQueue, + } as any); + + await start(validWorkflow, [], { + experimentalWithHook: { + token: 'order:123', + experimental_ttl: '30 days', + }, + }); + + expect(mockEventsCreate).toHaveBeenCalledWith( + expect.any(String), + expect.objectContaining({ + eventData: expect.objectContaining({ + experimentalStartHook: { + token: 'order:123', + ttlSeconds: 30 * 24 * 60 * 60, + }, + }), + }), + expect.anything() + ); + expect( + mockQueue.mock.calls[0]?.[1].runInput.experimentalStartHook + ).toEqual({ + token: 'order:123', + ttlSeconds: 30 * 24 * 60 * 60, + }); + expect(mockEventsCreate.mock.invocationCallOrder[0]).toBeLessThan( + mockQueue.mock.invocationCallOrder[0] + ); + + vi.useRealTimers(); + }); + + it('does not queue when experimental start hook admission conflicts', async () => { + const validWorkflow = Object.assign(() => Promise.resolve('result'), { + workflowId: 'test-workflow', + }); + mockEventsCreate.mockRejectedValueOnce( + new HookConflictError('order:123', 'wrun_conflicting') + ); + setWorld({ + specVersion: SPEC_VERSION_CURRENT, + supportsExperimentalStartHook: true, + getDeploymentId: vi.fn().mockResolvedValue('deploy_123'), + events: { create: mockEventsCreate }, + queue: mockQueue, + } as any); + + await expect( + start(validWorkflow, [], { + experimentalWithHook: { + token: 'order:123', + experimentalTtl: '30 days', + }, + }) + ).rejects.toThrow(HookConflictError); + expect(mockQueue).not.toHaveBeenCalled(); + }); + + it('rejects non-string experimental start hook tokens', async () => { + const validWorkflow = Object.assign(() => Promise.resolve('result'), { + workflowId: 'test-workflow', + }); + + await expect( + start(validWorkflow, [], { + experimentalWithHook: { + token: 123, + experimentalTtl: '30 days', + } as any, + }) + ).rejects.toThrow(/token must be a non-empty string/); + expect(mockEventsCreate).not.toHaveBeenCalled(); + expect(mockQueue).not.toHaveBeenCalled(); + }); + + it('cancels the admitted run when experimental start hook queueing fails', async () => { + const validWorkflow = Object.assign(() => Promise.resolve('result'), { + workflowId: 'test-workflow', + }); + const queueError = new Error('queue unavailable'); + mockQueue.mockRejectedValueOnce(queueError); + setWorld({ + specVersion: SPEC_VERSION_CURRENT, + supportsExperimentalStartHook: true, + getDeploymentId: vi.fn().mockResolvedValue('deploy_123'), + events: { create: mockEventsCreate }, + queue: mockQueue, + } as any); + + await expect( + start(validWorkflow, [], { + experimentalWithHook: { + token: 'order:123', + experimentalTtl: '30 days', + }, + }) + ).rejects.toThrow(queueError); + + expect(mockEventsCreate).toHaveBeenCalledTimes(2); + expect(mockEventsCreate.mock.calls[0]?.[1].eventType).toBe('run_created'); + expect(mockEventsCreate.mock.calls[1]?.[1]).toMatchObject({ + eventType: 'run_cancelled', + specVersion: SPEC_VERSION_CURRENT, + }); + }); + + it('does not queue when experimental start hook admission is unconfirmed after retry', async () => { + const validWorkflow = Object.assign(() => Promise.resolve('result'), { + workflowId: 'test-workflow', + }); + const admissionError = new WorkflowWorldError('response lost', { + status: 500, + }); + mockEventsCreate + .mockRejectedValueOnce(admissionError) + .mockRejectedValueOnce(admissionError); + setWorld({ + specVersion: SPEC_VERSION_CURRENT, + supportsExperimentalStartHook: true, + getDeploymentId: vi.fn().mockResolvedValue('deploy_123'), + events: { create: mockEventsCreate }, + queue: mockQueue, + } as any); + + await expect( + start(validWorkflow, [], { + experimentalWithHook: { + token: 'order:123', + experimentalTtl: '30 days', + }, + }) + ).rejects.toThrow(admissionError); + + expect(mockEventsCreate).toHaveBeenCalledTimes(2); + expect(mockQueue).not.toHaveBeenCalled(); + }); + + it('queues a confirmed experimental start hook run after a retryable admission response', async () => { + const validWorkflow = Object.assign(() => Promise.resolve('result'), { + workflowId: 'test-workflow', + }); + const admissionError = new WorkflowWorldError('response lost', { + status: 500, + }); + const mockEventsList = vi.fn().mockResolvedValue({ + data: [{ eventType: 'run_created' }], + }); + mockEventsCreate + .mockRejectedValueOnce(admissionError) + .mockRejectedValueOnce(new EntityConflictError('Run already exists')); + setWorld({ + specVersion: SPEC_VERSION_CURRENT, + supportsExperimentalStartHook: true, + getDeploymentId: vi.fn().mockResolvedValue('deploy_123'), + events: { create: mockEventsCreate, list: mockEventsList }, + queue: mockQueue, + } as any); + + const run = await start(validWorkflow, [], { + experimentalWithHook: { + token: 'order:123', + experimentalTtl: '30 days', + }, + }); + + expect(run.runId).toBe(mockEventsCreate.mock.calls[0]?.[0]); + expect(mockEventsCreate).toHaveBeenCalledTimes(2); + expect(mockEventsList).toHaveBeenCalledWith({ + runId: run.runId, + pagination: { limit: 1, sortOrder: 'asc' }, + resolveData: 'none', + }); + expect(mockQueue).toHaveBeenCalledTimes(1); + expect(mockQueue.mock.calls[0]?.[1].runId).toBe(run.runId); + }); + + it('rejects experimental start hooks when the world has not opted in', async () => { + const validWorkflow = Object.assign(() => Promise.resolve('result'), { + workflowId: 'test-workflow', + }); + setWorld({ + specVersion: SPEC_VERSION_CURRENT, + getDeploymentId: vi.fn().mockResolvedValue('deploy_123'), + events: { create: mockEventsCreate }, + queue: mockQueue, + } as any); + + await expect( + start(validWorkflow, [], { + experimentalWithHook: { + token: 'order:123', + experimentalTtl: '30 days', + }, + }) + ).rejects.toThrow(/supports experimental start-hook admission/); + expect(mockEventsCreate).not.toHaveBeenCalled(); + expect(mockQueue).not.toHaveBeenCalled(); + }); + it('rejects initial attributes for pre-v4 runs', async () => { const validWorkflow = Object.assign(() => Promise.resolve('result'), { workflowId: 'test-workflow', diff --git a/packages/core/src/runtime/start.ts b/packages/core/src/runtime/start.ts index 88c8838074..151edebf79 100644 --- a/packages/core/src/runtime/start.ts +++ b/packages/core/src/runtime/start.ts @@ -4,6 +4,7 @@ import { WorkflowRuntimeError, WorkflowWorldError, } from '@workflow/errors'; +import { parseDurationToDate } from '@workflow/utils'; import { workflowDisplayName } from '@workflow/utils/parse-name'; import type { WorkflowInvokePayload, World } from '@workflow/world'; import { @@ -13,6 +14,7 @@ import { SPEC_VERSION_SUPPORTS_COMPRESSION, SPEC_VERSION_SUPPORTS_EVENT_SOURCING, } from '@workflow/world'; +import type { StringValue } from 'ms'; import { monotonicFactory } from 'ulid'; import { normalizeAttributeChanges } from '../attribute-changes.js'; import { getRunCapabilities } from '../capabilities.js'; @@ -61,6 +63,79 @@ export function _resetLatestNoOpWarnForTests(): void { hasWarnedLatestNoOp = false; } +type ExperimentalStartHookOptions = { + token: string; +} & ( + | { experimentalTtl: StringValue | number; experimental_ttl?: never } + | { experimental_ttl: StringValue | number; experimentalTtl?: never } +); + +function normalizeExperimentalStartHook(hook: ExperimentalStartHookOptions): { + token: string; + ttlSeconds: number; +} { + if (typeof hook.token !== 'string' || hook.token.length === 0) { + throw new WorkflowRuntimeError( + 'experimentalWithHook.token must be a non-empty string.' + ); + } + + if ( + hook.experimentalTtl !== undefined && + hook.experimental_ttl !== undefined + ) { + throw new WorkflowRuntimeError( + 'experimentalWithHook accepts either experimentalTtl or experimental_ttl, not both.' + ); + } + + const ttl = hook.experimentalTtl ?? hook.experimental_ttl; + if (ttl === undefined) { + throw new WorkflowRuntimeError( + 'experimentalWithHook.experimentalTtl must be provided.' + ); + } + + const ttlMilliseconds = parseDurationToDate(ttl).getTime() - Date.now(); + if (!Number.isFinite(ttlMilliseconds) || ttlMilliseconds <= 0) { + throw new WorkflowRuntimeError( + 'experimentalWithHook.experimentalTtl must be a positive duration.' + ); + } + + const ttlSeconds = Math.max(1, Math.floor(ttlMilliseconds / 1000)); + return { token: hook.token, ttlSeconds }; +} + +async function hasDurableRunCreatedEvent( + world: World, + runId: string +): Promise { + const page = await world.events.list({ + runId, + pagination: { limit: 1, sortOrder: 'asc' }, + resolveData: 'none', + }); + return page.data.some((event) => event.eventType === 'run_created'); +} + +function getRunCreatedResult( + runId: string, + result: Awaited> +) { + if (!result.run) { + throw new WorkflowRuntimeError( + "Missing 'run' in server response for 'run_created' event" + ); + } + if (result.run.runId !== runId) { + throw new WorkflowRuntimeError( + `Server returned different runId than requested: expected ${runId}, got ${result.run.runId}` + ); + } + return result.run; +} + export interface StartOptionsBase { /** * The world to use for the workflow run creation, @@ -93,6 +168,14 @@ export interface StartOptionsBase { * the `experimental_setAttributes` option of the same name. */ allowReservedAttributes?: boolean; + + /** + * EXPERIMENTAL: atomically reserve a hook token as part of run admission. + * + * If another active or retained run already owns the token, `start()` throws + * `HookConflictError` and does not queue a new workflow run. + */ + experimentalWithHook?: ExperimentalStartHookOptions; } export interface StartOptionsWithDeploymentId extends StartOptionsBase { @@ -289,6 +372,22 @@ export async function start( world.specVersion ?? SPEC_VERSION_SUPPORTS_EVENT_SOURCING; const v1Compat = isLegacySpecVersion(specVersion); + const experimentalStartHook = opts.experimentalWithHook + ? normalizeExperimentalStartHook(opts.experimentalWithHook) + : undefined; + if (experimentalStartHook && v1Compat) { + throw new WorkflowRuntimeError( + 'experimentalWithHook requires an event-sourced World.' + ); + } + if ( + experimentalStartHook && + world.supportsExperimentalStartHook !== true + ) { + throw new WorkflowRuntimeError( + 'experimentalWithHook requires a World that supports experimental start-hook admission.' + ); + } const allowReservedAttributes = opts.allowReservedAttributes === true; let attributes: Record | undefined; if (opts.attributes && Object.keys(opts.attributes).length > 0) { @@ -326,6 +425,9 @@ export async function start( : {}), } : {}; + const startHookSeed = experimentalStartHook + ? { experimentalStartHook } + : {}; // Resolve encryption key for the new run. The runId has already been // generated above (client-generated ULID) and will be used for both @@ -366,91 +468,154 @@ export async function start( features: { encryption: !!encryptionKey }, }; - // Call events.create (run_created) and queue in parallel. - // If events.create fails with 429/5xx, the run was still accepted - // via the queue and creation will be re-tried async by the runtime. - const [runCreatedResult, queueResult] = await Promise.allSettled([ - world.events.create( - runId, - { - eventType: 'run_created', - specVersion, - eventData: { - deploymentId: deploymentId, - workflowName: workflowName, - input: workflowArguments, - executionContext, - ...attributeSeed, - }, - }, - { v1Compat } - ), - world.queue( - getWorkflowQueueName(workflowName), - { - runId, - traceCarrier, - ...(specVersion >= SPEC_VERSION_SUPPORTS_CBOR_QUEUE_TRANSPORT - ? { - runInput: { - input: workflowArguments, - deploymentId, - workflowName, - specVersion, - executionContext, - ...attributeSeed, - }, - } - : {}), - } satisfies WorkflowInvokePayload, - { - deploymentId, - specVersion, - } - ), - ]); - - // Queue failure is always fatal — the run was not enqueued - if (queueResult.status === 'rejected') { - throw queueResult.reason; - } + const runCreatedEvent = { + eventType: 'run_created' as const, + specVersion, + eventData: { + deploymentId: deploymentId, + workflowName: workflowName, + input: workflowArguments, + executionContext, + ...attributeSeed, + ...startHookSeed, + }, + }; + const queuePayload = { + runId, + traceCarrier, + ...(specVersion >= SPEC_VERSION_SUPPORTS_CBOR_QUEUE_TRANSPORT + ? { + runInput: { + input: workflowArguments, + deploymentId, + workflowName, + specVersion, + executionContext, + ...attributeSeed, + ...startHookSeed, + }, + } + : {}), + } satisfies WorkflowInvokePayload; - // Handle events.create result let resilientStart = false; - if (runCreatedResult.status === 'rejected') { - const err = runCreatedResult.reason; - if (EntityConflictError.is(err)) { - // 409: The run already exists. This can happen in extreme cases where - // the run creation call gets a cold start or other slowdown, and the queue - // + run_started call completes faster. We expect this to be <=1% of cases. - // In this case, we can safely return. - } else if (isRetryableStartError(err)) { - // 429 (ThrottleError) and 5xx (WorkflowWorldError with status >= 500) - // are retryable — the run was accepted via the queue and creation - // will be re-tried by the runtime when it calls run_started. - resilientStart = true; + let createdRunForSpan: + | NonNullable>['run']> + | undefined; + if (experimentalStartHook) { + try { + const result = await world.events.create(runId, runCreatedEvent, { + v1Compat, + }); + createdRunForSpan = getRunCreatedResult(runId, result); + } catch (error) { + if (!isRetryableStartError(error)) throw error; + try { + const result = await world.events.create(runId, runCreatedEvent, { + v1Compat, + }); + createdRunForSpan = getRunCreatedResult(runId, result); + } catch (retryError) { + if ( + !EntityConflictError.is(retryError) && + !isRetryableStartError(retryError) + ) { + throw retryError; + } + try { + if (!(await hasDurableRunCreatedEvent(world, runId))) { + throw error; + } + } catch { + throw error; + } + } runtimeLogger.warn( - 'Run creation event failed, but the run was accepted via the queue. ' + - 'The run_created event will be re-tried async by the runtime.', - { workflowRunId: runId, error: err.message } + 'Run creation response failed after start-hook admission, but the run_created event is durable. Queueing the confirmed run.', + { workflowRunId: runId, error: (error as Error).message } ); - } else { - throw err; + } + + try { + await world.queue(getWorkflowQueueName(workflowName), queuePayload, { + deploymentId, + specVersion, + }); + } catch (error) { + try { + await world.events.create( + runId, + { + eventType: 'run_cancelled', + specVersion, + }, + { v1Compat } + ); + } catch (cleanupError) { + runtimeLogger.warn( + 'Run queueing failed after start-hook admission, and cleanup also failed.', + { + workflowRunId: runId, + error: (cleanupError as Error).message, + } + ); + } + throw error; } } else { - const result = runCreatedResult.value; - // Assert that the run was created - if (!result.run) { - throw new WorkflowRuntimeError( - "Missing 'run' in server response for 'run_created' event" - ); + // Call events.create (run_created) and queue in parallel. + // If events.create fails with 429/5xx, the run was still accepted + // via the queue and creation will be re-tried async by the runtime. + const [runCreatedResult, queueResult] = await Promise.allSettled([ + world.events.create(runId, runCreatedEvent, { v1Compat }), + world.queue(getWorkflowQueueName(workflowName), queuePayload, { + deploymentId, + specVersion, + }), + ]); + + // Queue failure is always fatal — the run was not enqueued + if (queueResult.status === 'rejected') { + throw queueResult.reason; } - // Verify server accepted our runId - if (!v1Compat && result.run.runId !== runId) { - throw new WorkflowRuntimeError( - `Server returned different runId than requested: expected ${runId}, got ${result.run.runId}` - ); + // Handle events.create result + if (runCreatedResult.status === 'rejected') { + const err = runCreatedResult.reason; + if (EntityConflictError.is(err)) { + // 409: The run already exists. This can happen in extreme cases where + // the run creation call gets a cold start or other slowdown, and the queue + // + run_started call completes faster. We expect this to be <=1% of cases. + // In this case, we can safely return. + } else if (isRetryableStartError(err)) { + // 429 (ThrottleError) and 5xx (WorkflowWorldError with status >= 500) + // are retryable — the run was accepted via the queue and creation + // will be re-tried by the runtime when it calls run_started. + resilientStart = true; + runtimeLogger.warn( + 'Run creation event failed, but the run was accepted via the queue. ' + + 'The run_created event will be re-tried async by the runtime.', + { workflowRunId: runId, error: err.message } + ); + } else { + throw err; + } + } else { + const result = runCreatedResult.value; + // Assert that the run was created + if (!result.run) { + throw new WorkflowRuntimeError( + "Missing 'run' in server response for 'run_created' event" + ); + } + + // Verify server accepted our runId + if (!v1Compat && result.run.runId !== runId) { + throw new WorkflowRuntimeError( + `Server returned different runId than requested: expected ${runId}, got ${result.run.runId}` + ); + } + createdRunForSpan = result.run; } } @@ -471,9 +636,8 @@ export async function start( span?.setAttributes({ ...Attribute.WorkflowRunId(runId), ...Attribute.DeploymentId(deploymentId), - ...(runCreatedResult.status === 'fulfilled' && - runCreatedResult.value.run - ? Attribute.WorkflowRunStatus(runCreatedResult.value.run.status) + ...(createdRunForSpan + ? Attribute.WorkflowRunStatus(createdRunForSpan.status) : {}), }); diff --git a/packages/world-local/src/index.ts b/packages/world-local/src/index.ts index 8ecb4f5727..dacda103b2 100644 --- a/packages/world-local/src/index.ts +++ b/packages/world-local/src/index.ts @@ -66,6 +66,7 @@ export function createLocalWorld(args?: Partial): LocalWorld { const recoverActiveRuns = mergedConfig.recoverActiveRuns ?? true; return { specVersion: SPEC_VERSION_CURRENT, + supportsExperimentalStartHook: true, ...queue, ...storage, ...instrumentObject('world.streams', { @@ -137,6 +138,40 @@ export function createLocalWorld(args?: Partial): LocalWorld { } }) ); + const tokensDir = path.join(hooksDir, 'tokens'); + const tokenFiles = await fs.readdir(tokensDir).catch(() => []); + await Promise.all( + tokenFiles + .filter((tokenFile) => tokenFile.endsWith('.json')) + .map(async (tokenFile) => { + const claimPath = path.join(tokensDir, tokenFile); + const claim = JSON.parse( + await fs.readFile(claimPath, 'utf8') + ) as { + tag?: unknown; + token?: unknown; + runId?: unknown; + hookId?: unknown; + }; + if (claim.tag !== tag) return; + + await deleteJSON(claimPath); + if ( + typeof claim.token === 'string' && + typeof claim.runId === 'string' && + typeof claim.hookId === 'string' + ) { + await deleteJSON( + hookRecoveryMarkerPath( + basedir, + claim.token, + claim.runId, + claim.hookId + ) + ); + } + }) + ); // Delete tagged entity files across all directories const entityDirs = [ diff --git a/packages/world-local/src/storage.test.ts b/packages/world-local/src/storage.test.ts index 0ac277bbe1..895be20981 100644 --- a/packages/world-local/src/storage.test.ts +++ b/packages/world-local/src/storage.test.ts @@ -1,13 +1,17 @@ import { promises as fs } from 'node:fs'; import os from 'node:os'; import path from 'node:path'; -import { WorkflowWorldError } from '@workflow/errors'; +import { + EntityConflictError, + HookConflictError, + WorkflowWorldError, +} from '@workflow/errors'; import type { Event, Storage } from '@workflow/world'; import { SPEC_VERSION_CURRENT, stripEventDataRefs } from '@workflow/world'; import { monotonicFactory } from 'ulid'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { writeJSON } from './fs.js'; -import { hashToken } from './storage/helpers.js'; +import { hashToken, hookRecoveryMarkerPath } from './storage/helpers.js'; import { createStorage } from './storage.js'; import { completeWait, @@ -1932,6 +1936,580 @@ describe('Storage', () => { expect(result.hook).toBeUndefined(); }); + it('should reject duplicate start hook claims before hook materialization', async () => { + const token = 'start-claim-token'; + const first = await storage.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }, + }); + expect(first.run).toBeDefined(); + const firstRun = first.run; + if (!firstRun) throw new Error('Expected run to be created'); + + await expect( + storage.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }, + }) + ).rejects.toThrow(HookConflictError); + + await expect( + storage.events.create(firstRun.runId, { + eventType: 'hook_created', + correlationId: 'hook_start', + eventData: { token }, + }) + ).resolves.toMatchObject({ + event: { eventType: 'hook_created' }, + hook: { + runId: firstRun.runId, + hookId: 'hook_start', + token, + }, + }); + }); + + it('should reserve start hook claims during resilient run_started bootstrap', async () => { + const token = 'resilient-start-claim-token'; + const runId = `wrun_${monotonicFactory()()}`; + + const started = await storage.events.create(runId, { + eventType: 'run_started', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }, + }); + + expect(started.run?.runId).toBe(runId); + await expect( + storage.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'deployment-456', + workflowName: 'test-workflow-2', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }, + }) + ).rejects.toThrow(HookConflictError); + + await expect( + storage.events.create(runId, { + eventType: 'hook_created', + correlationId: 'hook_resilient_start', + eventData: { token }, + }) + ).resolves.toMatchObject({ + event: { eventType: 'hook_created' }, + hook: { runId, hookId: 'hook_resilient_start', token }, + }); + }); + + it('should materialize only one hook for a reserved start hook token', async () => { + const token = 'concurrent-start-claim-materialize-token'; + const started = await storage.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }, + }); + const run = started.run; + if (!run) throw new Error('Expected run to be created'); + + const results = await Promise.all([ + storage.events.create(run.runId, { + eventType: 'hook_created', + correlationId: 'hook_start_a', + eventData: { token }, + }), + storage.events.create(run.runId, { + eventType: 'hook_created', + correlationId: 'hook_start_b', + eventData: { token }, + }), + ]); + + expect(results.map((result) => result.event.eventType).sort()).toEqual([ + 'hook_conflict', + 'hook_created', + ]); + expect(results.filter((result) => result.hook).length).toBe(1); + }); + + it('should repair a same-run start hook retry when the run_created event is missing', async () => { + const token = 'same-run-start-claim-token'; + const first = await storage.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }, + }); + expect(first.run).toBeDefined(); + const run = first.run; + if (!run) throw new Error('Expected run to be created'); + + await fs.unlink( + path.join( + testDir, + 'events', + `${run.runId}-${first.event.eventId}.json` + ) + ); + + const retry = await storage.events.create(run.runId, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }, + }); + + expect(retry.event.eventType).toBe('run_created'); + expect(retry.event.eventId).toBe(first.event.eventId); + expect(retry.run?.runId).toBe(run.runId); + }); + + it('should recover a stale start hook materialization lock', async () => { + const token = 'stale-materialize-lock-token'; + const first = await storage.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }, + }); + expect(first.run).toBeDefined(); + const run = first.run; + if (!run) throw new Error('Expected run to be created'); + + const lockPath = path.join( + testDir, + '.locks', + 'hooks', + `${hashToken(token)}.materialize` + ); + await fs.mkdir(path.dirname(lockPath), { recursive: true }); + await fs.writeFile(lockPath, ''); + const stale = new Date(Date.now() - 60_000); + await fs.utimes(lockPath, stale, stale); + + const result = await storage.events.create(run.runId, { + eventType: 'hook_created', + correlationId: 'hook_stale_materialize', + eventData: { token }, + }); + + expect(result.event.eventType).toBe('hook_created'); + expect(result.hook?.hookId).toBe('hook_stale_materialize'); + }); + + it('should clean up start hook claims when run creation loses', async () => { + const token = 'failed-run-create-start-claim-token'; + const run = await createRun(storage, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + }); + + await expect( + storage.events.create(run.runId, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'deployment-456', + workflowName: 'test-workflow-2', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }, + }) + ).rejects.toThrow(EntityConflictError); + + await expect( + storage.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'deployment-789', + workflowName: 'test-workflow-3', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }, + }) + ).resolves.toMatchObject({ + event: { eventType: 'run_created' }, + run: { workflowName: 'test-workflow-3' }, + }); + }); + + it('should retain a disposed start hook claim until ttl expiry', async () => { + const token = 'disposed-start-claim-token'; + const started = await storage.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }, + }); + expect(started.run).toBeDefined(); + const startedRun = started.run; + if (!startedRun) throw new Error('Expected run to be created'); + const runId = startedRun.runId; + + await storage.events.create(runId, { + eventType: 'hook_created', + correlationId: 'hook_start_disposed', + eventData: { token }, + }); + await storage.events.create(runId, { + eventType: 'hook_disposed', + correlationId: 'hook_start_disposed', + }); + + await expect( + storage.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }, + }) + ).rejects.toThrow(HookConflictError); + }); + + it('should not materialize a retained start hook claim again', async () => { + const token = 'retained-start-claim-token'; + const started = await storage.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }, + }); + const run = started.run; + if (!run) throw new Error('Expected run to be created'); + + await storage.events.create(run.runId, { + eventType: 'hook_created', + correlationId: 'hook_start_retained', + eventData: { token }, + }); + await storage.events.create(run.runId, { + eventType: 'hook_disposed', + correlationId: 'hook_start_retained', + }); + + const result = await storage.events.create(run.runId, { + eventType: 'hook_created', + correlationId: 'hook_start_retained_again', + eventData: { token }, + }); + + expect(result.event.eventType).toBe('hook_conflict'); + expect(result.hook).toBeUndefined(); + }); + + it('should retain an unmaterialized start hook claim until ttl expiry', async () => { + const token = 'cancelled-unmaterialized-start-claim-token'; + const started = await storage.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }, + }); + const run = started.run; + if (!run) throw new Error('Expected run to be created'); + + await storage.events.create(run.runId, { + eventType: 'run_cancelled', + }); + + await expect( + storage.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'deployment-456', + workflowName: 'test-workflow-2', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }, + }) + ).rejects.toThrow(HookConflictError); + + const past = new Date(Date.now() - 2_000); + await writeJSON( + path.join(testDir, 'hooks', 'tokens', `${hashToken(token)}.json`), + { + token, + runId: run.runId, + ttlSeconds: 60, + createdAt: past, + expiresAt: past, + phase: 'retained', + }, + { overwrite: true } + ); + + await expect( + storage.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'deployment-789', + workflowName: 'test-workflow-3', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }, + }) + ).resolves.toMatchObject({ + event: { eventType: 'run_created' }, + run: { workflowName: 'test-workflow-3' }, + }); + }); + + it('should keep an expired disposed start hook claim while the run is active', async () => { + const token = 'active-expired-start-claim-token'; + const started = await storage.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 1 }, + }, + }); + expect(started.run).toBeDefined(); + const run = started.run; + if (!run) throw new Error('Expected run to be created'); + + await storage.events.create(run.runId, { + eventType: 'hook_created', + correlationId: 'hook_start_expired_active', + eventData: { token }, + }); + await storage.events.create(run.runId, { + eventType: 'hook_disposed', + correlationId: 'hook_start_expired_active', + }); + + const past = new Date(Date.now() - 2_000); + await writeJSON( + path.join(testDir, 'hooks', 'tokens', `${hashToken(token)}.json`), + { + token, + runId: run.runId, + ttlSeconds: 1, + createdAt: past, + expiresAt: past, + }, + { overwrite: true } + ); + + await expect( + storage.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'deployment-456', + workflowName: 'test-workflow-2', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 1 }, + }, + }) + ).rejects.toThrow(HookConflictError); + + await storage.events.create(run.runId, { + eventType: 'run_completed', + eventData: { output: new Uint8Array() }, + }); + + await expect( + storage.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'deployment-789', + workflowName: 'test-workflow-3', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 1 }, + }, + }) + ).resolves.toMatchObject({ + event: { eventType: 'run_created' }, + run: { workflowName: 'test-workflow-3' }, + }); + }); + + it('should allow an expired orphaned start hook claim to be reused', async () => { + const token = 'expired-orphaned-start-claim-token'; + const past = new Date(Date.now() - 2_000); + await writeJSON( + path.join(testDir, 'hooks', 'tokens', `${hashToken(token)}.json`), + { + token, + runId: 'orphaned-start-run', + ttlSeconds: 1, + createdAt: past, + phase: 'start_claim', + }, + { overwrite: true } + ); + + await expect( + storage.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 1 }, + }, + }) + ).resolves.toMatchObject({ + event: { eventType: 'run_created' }, + run: { workflowName: 'test-workflow' }, + }); + }); + + it('should allow ordinary hook creation after a retained start hook claim expires', async () => { + const token = 'expired-retained-ordinary-hook-token'; + const started = await storage.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 1 }, + }, + }); + const run = started.run; + if (!run) throw new Error('Expected run to be created'); + + await storage.events.create(run.runId, { + eventType: 'hook_created', + correlationId: 'hook_start_expired_reusable', + eventData: { token }, + }); + await storage.events.create(run.runId, { + eventType: 'hook_disposed', + correlationId: 'hook_start_expired_reusable', + }); + await storage.events.create(run.runId, { + eventType: 'run_completed', + eventData: { output: new Uint8Array() }, + }); + + const past = new Date(Date.now() - 2_000); + await writeJSON( + path.join(testDir, 'hooks', 'tokens', `${hashToken(token)}.json`), + { + token, + runId: run.runId, + hookId: 'hook_start_expired_reusable', + ttlSeconds: 1, + createdAt: past, + expiresAt: past, + phase: 'retained', + }, + { overwrite: true } + ); + + const nextRun = await createRun(storage, { + deploymentId: 'deployment-456', + workflowName: 'test-workflow-2', + input: new Uint8Array(), + }); + + await expect( + storage.events.create(nextRun.runId, { + eventType: 'hook_created', + correlationId: 'hook_after_retained_expired', + eventData: { token }, + }) + ).resolves.toMatchObject({ + event: { eventType: 'hook_created' }, + hook: { + runId: nextRun.runId, + hookId: 'hook_after_retained_expired', + token, + }, + }); + }); + + it('should ignore hook recovery markers when cleaning up start hook claims', async () => { + const run = await createRun(storage, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + }); + + await writeJSON( + hookRecoveryMarkerPath( + testDir, + 'marker-token', + run.runId, + 'hook_marker' + ), + { eventId: 'wevt_marker' }, + { overwrite: true } + ); + + await expect( + updateRun(storage, run.runId, 'run_completed', { + output: new Uint8Array(), + }) + ).resolves.toMatchObject({ + status: 'completed', + }); + }); + it('should return hook_conflict event when the token claim cannot provide a run ID', async () => { const token = 'legacy-duplicate-test-token'; diff --git a/packages/world-local/src/storage/events-storage.ts b/packages/world-local/src/storage/events-storage.ts index 6813d3d4de..500e379276 100644 --- a/packages/world-local/src/storage/events-storage.ts +++ b/packages/world-local/src/storage/events-storage.ts @@ -2,6 +2,7 @@ import fs from 'node:fs/promises'; import path from 'node:path'; import { EntityConflictError, + HookConflictError, HookNotFoundError, RunExpiredError, RunNotSupportedError, @@ -83,6 +84,7 @@ import { withRunFileLock } from './runs-storage.js'; // semantics without spawning subprocesses. const HookTokenClaimSchema = z.object({ + token: z.string().optional(), // The token-claim writer below has always persisted `hookId`, but // this read schema previously omitted it, which is the bug fixed // by https://github.com/vercel/workflow/issues/2283. `optional()` @@ -100,8 +102,16 @@ const HookTokenClaimSchema = z.object({ // marker upgrade path, which atomically pins a canonical eventId // via a sidecar marker (also a `writeExclusive`). eventId: z.string().optional(), + ttlSeconds: z.number().int().positive().optional(), + startEventId: z.string().optional(), + createdAt: z.coerce.date().optional(), + expiresAt: z.coerce.date().optional(), + phase: z.enum(['start_claim', 'materialized', 'retained']).optional(), + tag: z.string().optional(), }); +const START_HOOK_MATERIALIZE_LOCK_STALE_MS = 30_000; + /** * Sidecar recovery marker that pins a canonical `hook_created` * eventId for a legacy token claim — one written by a version of @@ -136,6 +146,191 @@ async function readHookTokenClaim( } } +function hookTokenClaimPath(basedir: string, token: string): string { + return path.join(basedir, 'hooks', 'tokens', `${hashToken(token)}.json`); +} + +function hookTokenReclaimLockPath( + basedir: string, + token: string, + tag: string | undefined +): string { + const lockName = tag + ? `${hashToken(token)}.reclaim.${tag}` + : `${hashToken(token)}.reclaim`; + return resolveWithinBase(basedir, '.locks', 'hooks', lockName); +} + +function hookTokenMaterializeLockPath( + basedir: string, + token: string, + tag: string | undefined +): string { + const lockName = tag + ? `${hashToken(token)}.materialize.${tag}` + : `${hashToken(token)}.materialize`; + return resolveWithinBase(basedir, '.locks', 'hooks', lockName); +} + +async function canReuseExpiredStartClaim( + basedir: string, + tag: string | undefined, + claim: z.infer +): Promise { + const expiresAt = + claim.expiresAt ?? + (claim.createdAt && claim.ttlSeconds + ? new Date(claim.createdAt.getTime() + claim.ttlSeconds * 1000) + : undefined); + if (!expiresAt || expiresAt > new Date()) return false; + + const run = await readJSONWithFallback( + basedir, + 'runs', + claim.runId, + WorkflowRunSchema, + claim.tag ?? tag + ); + if (!run) return true; + return ['completed', 'failed', 'cancelled'].includes(run.status); +} + +async function replaceExpiredStartClaim( + basedir: string, + tag: string | undefined, + token: string, + nextClaim: z.infer +): Promise { + const lockPath = hookTokenReclaimLockPath(basedir, token, undefined); + let locked = await writeExclusive(lockPath, ''); + if (!locked) { + const stat = await fs.stat(lockPath).catch(() => undefined); + if ( + stat && + Date.now() - stat.mtimeMs > START_HOOK_MATERIALIZE_LOCK_STALE_MS + ) { + await fs.unlink(lockPath).catch(() => {}); + locked = await writeExclusive(lockPath, ''); + } + } + if (!locked) return false; + + try { + const constraintPath = hookTokenClaimPath(basedir, token); + const latestClaim = await readHookTokenClaim(constraintPath); + if ( + !latestClaim || + !(await canReuseExpiredStartClaim(basedir, tag, latestClaim)) + ) { + return false; + } + await writeJSON(constraintPath, nextClaim, { overwrite: true }); + return true; + } finally { + await fs.unlink(lockPath).catch(() => {}); + } +} + +async function materializeStartHookClaim( + basedir: string, + tag: string | undefined, + token: string, + nextClaim: z.infer +): Promise { + const lockPath = hookTokenMaterializeLockPath(basedir, token, tag); + let locked = await writeExclusive(lockPath, ''); + if (!locked) { + const stat = await fs.stat(lockPath).catch(() => undefined); + if ( + stat && + Date.now() - stat.mtimeMs > START_HOOK_MATERIALIZE_LOCK_STALE_MS + ) { + await fs.unlink(lockPath).catch(() => {}); + locked = await writeExclusive(lockPath, ''); + } + } + if (!locked) return false; + + try { + const constraintPath = hookTokenClaimPath(basedir, token); + const latestClaim = await readHookTokenClaim(constraintPath); + if ( + !latestClaim || + latestClaim.runId !== nextClaim.runId || + latestClaim.hookId !== undefined || + latestClaim.phase === 'retained' + ) { + return false; + } + await writeJSON(constraintPath, nextClaim, { overwrite: true }); + return true; + } finally { + await fs.unlink(lockPath).catch(() => {}); + } +} + +async function claimStartHookToken( + basedir: string, + runId: string, + eventId: string, + startHook: { token: string; ttlSeconds: number }, + tag?: string +): Promise { + const constraintPath = hookTokenClaimPath(basedir, startHook.token); + const createdAt = new Date(); + const claimed = await writeExclusive( + constraintPath, + JSON.stringify({ + token: startHook.token, + runId, + ttlSeconds: startHook.ttlSeconds, + startEventId: eventId, + createdAt, + phase: 'start_claim', + tag, + }) + ); + if (claimed) return true; + + const existingClaim = await readHookTokenClaim(constraintPath); + if (existingClaim?.runId === runId) return false; + if ( + existingClaim && + (await replaceExpiredStartClaim(basedir, tag, startHook.token, { + token: startHook.token, + runId, + ttlSeconds: startHook.ttlSeconds, + startEventId: eventId, + createdAt, + phase: 'start_claim', + tag, + })) + ) { + return true; + } + + const currentClaim = await readHookTokenClaim(constraintPath); + throw new HookConflictError(startHook.token, currentClaim?.runId); +} + +async function deleteOwnedStartHookClaim( + basedir: string, + token: string, + runId: string, + eventId: string +): Promise { + const constraintPath = hookTokenClaimPath(basedir, token); + const latestClaim = await readHookTokenClaim(constraintPath); + if ( + latestClaim?.runId === runId && + latestClaim.startEventId === eventId && + latestClaim.phase === 'start_claim' && + !latestClaim.hookId + ) { + await deleteJSON(constraintPath); + } +} + async function readHookRecoveryMarker( markerPath: string ): Promise | null> { @@ -548,6 +743,10 @@ export function createEventsStorage( executionContext?: Record; attributes?: Record; allowReservedAttributes?: true; + experimentalStartHook?: { + token: string; + ttlSeconds: number; + }; }; if ( runInputData.deploymentId && @@ -583,6 +782,17 @@ export function createEventsStorage( createdAt: now, updatedAt: now, }; + const runCreatedEventId = `evnt_${monotonicUlid()}`; + let ownsStartHookClaim = false; + if (runInputData.experimentalStartHook) { + ownsStartHookClaim = await claimStartHookToken( + basedir, + effectiveRunId, + runCreatedEventId, + runInputData.experimentalStartHook, + tag + ); + } const runPath = taggedPath(basedir, 'runs', effectiveRunId, tag); const created = await writeExclusive( runPath, @@ -591,7 +801,6 @@ export function createEventsStorage( if (created) { // We created the run — also write the run_created event. - const runCreatedEventId = `evnt_${monotonicUlid()}`; const runCreatedEvent: Event = { eventType: 'run_created', runId: effectiveRunId, @@ -606,6 +815,7 @@ export function createEventsStorage( attributes: runInputData.attributes, allowReservedAttributes: runInputData.allowReservedAttributes, + experimentalStartHook: runInputData.experimentalStartHook, }, }; const createdCompositeKey = `${effectiveRunId}-${runCreatedEventId}`; @@ -615,6 +825,14 @@ export function createEventsStorage( ); currentRun = createdRun; } else { + if (ownsStartHookClaim && runInputData.experimentalStartHook) { + await deleteOwnedStartHookClaim( + basedir, + runInputData.experimentalStartHook.token, + effectiveRunId, + runCreatedEventId + ); + } // Run already exists (concurrent run_created won the // race). Re-read it so downstream logic sees the real state. currentRun = await readJSONWithFallback( @@ -899,6 +1117,10 @@ export function createEventsStorage( executionContext?: Record; attributes?: Record; allowReservedAttributes?: true; + experimentalStartHook?: { + token: string; + ttlSeconds: number; + }; }; validateAttributeChanges( Object.entries(runData.attributes ?? {}).map(([key, value]) => ({ @@ -909,6 +1131,16 @@ export function createEventsStorage( allowReservedAttributes: runData.allowReservedAttributes === true, } ); + let ownsStartHookClaim = false; + if (runData.experimentalStartHook) { + ownsStartHookClaim = await claimStartHookToken( + basedir, + effectiveRunId, + eventId, + runData.experimentalStartHook, + tag + ); + } run = { runId: effectiveRunId, deploymentId: runData.deploymentId, @@ -936,9 +1168,74 @@ export function createEventsStorage( JSON.stringify(run, jsonReplacer, 2) ); if (!created) { - throw new EntityConflictError( - `Workflow run "${effectiveRunId}" already exists` - ); + const startHookClaim = runData.experimentalStartHook + ? await readHookTokenClaim( + hookTokenClaimPath( + basedir, + runData.experimentalStartHook.token + ) + ) + : null; + const startEventId = startHookClaim?.startEventId; + const existingRun = + runData.experimentalStartHook && + !ownsStartHookClaim && + startHookClaim?.runId === effectiveRunId && + startHookClaim.phase === 'start_claim' && + startEventId + ? await readJSONWithFallback( + basedir, + 'runs', + effectiveRunId, + WorkflowRunSchema, + tag + ) + : null; + + if ( + existingRun && + runData.experimentalStartHook && + startHookClaim && + startEventId + ) { + eventId = startEventId; + event = { + eventType: 'run_created', + runId: effectiveRunId, + eventId, + createdAt: ulidToDate(eventId.replace(/^evnt_/, '')) ?? now, + specVersion: effectiveSpecVersion, + eventData: { + deploymentId: existingRun.deploymentId, + workflowName: existingRun.workflowName, + input: existingRun.input, + executionContext: existingRun.executionContext, + attributes: existingRun.attributes, + ...(runData.allowReservedAttributes + ? { allowReservedAttributes: true as const } + : {}), + experimentalStartHook: { + token: runData.experimentalStartHook.token, + ttlSeconds: + startHookClaim.ttlSeconds ?? + runData.experimentalStartHook.ttlSeconds, + }, + }, + }; + run = existingRun; + } else { + if (ownsStartHookClaim && runData.experimentalStartHook) { + await deleteOwnedStartHookClaim( + basedir, + runData.experimentalStartHook.token, + effectiveRunId, + eventId + ); + } + throw new EntityConflictError( + `Workflow run "${effectiveRunId}" already exists` + ); + } } } else if (data.eventType === 'run_started') { // Reuse currentRun from validation (already read above) @@ -1451,17 +1748,12 @@ export function createEventsStorage( // Atomically claim the token using an exclusive-create constraint file. // This avoids the TOCTOU race of the previous read-all-then-check approach. - const constraintPath = path.join( - basedir, - 'hooks', - 'tokens', - `${hashToken(hookData.token)}.json` - ); + const constraintPath = hookTokenClaimPath(basedir, hookData.token); // Persist `eventId` in the claim so concurrent / cross- // process retries can converge on a single canonical // `hook_created` event path. See the recovery comment // below. - const tokenClaimed = await writeExclusive( + let tokenClaimed = await writeExclusive( constraintPath, JSON.stringify({ token: hookData.token, @@ -1513,7 +1805,44 @@ export function createEventsStorage( if ( existingClaim?.runId === effectiveRunId && - existingClaim.hookId === data.correlationId + existingClaim.hookId === undefined && + existingClaim.phase !== 'retained' + ) { + tokenClaimed = await materializeStartHookClaim( + basedir, + tag, + hookData.token, + { + token: hookData.token, + hookId: data.correlationId, + runId: effectiveRunId, + eventId, + ttlSeconds: existingClaim.ttlSeconds, + startEventId: existingClaim.startEventId, + createdAt: existingClaim.createdAt, + phase: 'materialized', + tag: existingClaim.tag, + } + ); + } + if ( + !tokenClaimed && + existingClaim?.phase === 'retained' && + (await replaceExpiredStartClaim(basedir, tag, hookData.token, { + token: hookData.token, + hookId: data.correlationId, + runId: effectiveRunId, + eventId, + })) + ) { + tokenClaimed = true; + } + + if ( + !tokenClaimed && + existingClaim?.runId === effectiveRunId && + existingClaim.hookId === data.correlationId && + existingClaim.phase !== 'retained' ) { // Adopt a canonical eventId for the recovery write. The // outer event publish (`writeExclusive(eventPath)`) @@ -1617,7 +1946,7 @@ export function createEventsStorage( specVersion: effectiveSpecVersion, }; writeHookEntityWithOverwrite = true; - } else { + } else if (!tokenClaimed) { // Cross-hook / cross-run conflict: a different // (runId, hookId) holds this token. Create a // hook_conflict event so the workflow can fail @@ -1730,19 +2059,40 @@ export function createEventsStorage( tag ); if (existingHook) { - // Delete the token constraint file to free up the token - // for reuse, and delete this hook's recovery marker (if - // any) for disk hygiene. The marker's filename hash - // includes `(token, runId, hookId)` so different - // lifetimes never collide, but cleaning up reduces disk - // leak for hooks that go through the recovery path. - const disposedConstraintPath = path.join( + // Normal hook disposal frees the token. Start-hook claims carry + // ttlSeconds and are retained so duplicate starts remain fenced + // after the active hook entity is gone. + const disposedConstraintPath = hookTokenClaimPath( basedir, - 'hooks', - 'tokens', - `${hashToken(existingHook.token)}.json` + existingHook.token ); - await deleteJSON(disposedConstraintPath); + const claim = await readHookTokenClaim(disposedConstraintPath); + if (claim?.runId === existingHook.runId && claim.ttlSeconds) { + await writeJSON( + disposedConstraintPath, + { + token: existingHook.token, + hookId: existingHook.hookId, + runId: existingHook.runId, + eventId: claim.eventId, + ttlSeconds: claim.ttlSeconds, + startEventId: claim.startEventId, + createdAt: claim.createdAt, + phase: 'retained', + tag: claim.tag, + expiresAt: new Date( + Math.max( + existingHook.createdAt.getTime() + + claim.ttlSeconds * 1000, + Date.now() + ) + ), + }, + { overwrite: true } + ); + } else { + await deleteJSON(disposedConstraintPath); + } await deleteJSON( hookRecoveryMarkerPath( basedir, diff --git a/packages/world-local/src/storage/hooks-storage.ts b/packages/world-local/src/storage/hooks-storage.ts index e4c8f3c742..30104b128c 100644 --- a/packages/world-local/src/storage/hooks-storage.ts +++ b/packages/world-local/src/storage/hooks-storage.ts @@ -8,6 +8,7 @@ import type { Storage, } from '@workflow/world'; import { HookSchema } from '@workflow/world'; +import { z } from 'zod'; import { DEFAULT_RESOLVE_DATA_OPTION } from '../config.js'; import { assertSafeEntityId, @@ -16,10 +17,59 @@ import { paginatedFileSystemQuery, readJSON, readJSONWithFallback, + writeJSON, } from '../fs.js'; import { filterHookData } from './filters.js'; import { hashToken, hookRecoveryMarkerPath } from './helpers.js'; +const HookTokenClaimSchema = z.object({ + token: z.string().optional(), + hookId: z.string().optional(), + runId: z.string(), + eventId: z.string().optional(), + ttlSeconds: z.number().int().positive().optional(), + startEventId: z.string().optional(), + createdAt: z.coerce.date().optional(), + expiresAt: z.coerce.date().optional(), + phase: z.enum(['start_claim', 'materialized', 'retained']).optional(), + tag: z.string().optional(), +}); + +async function retainStartHookClaim( + constraintPath: string, + claim: z.infer, + now: Date, + hookCreatedAt?: Date +): Promise { + if (!claim.ttlSeconds) return; + + const ttlBase = hookCreatedAt ?? claim.createdAt ?? now; + const expiresAt = new Date( + Math.max( + ttlBase.getTime() + claim.ttlSeconds * 1000, + claim.expiresAt?.getTime() ?? 0, + now.getTime() + ) + ); + + await writeJSON( + constraintPath, + { + token: claim.token, + hookId: claim.hookId, + runId: claim.runId, + eventId: claim.eventId, + ttlSeconds: claim.ttlSeconds, + startEventId: claim.startEventId, + createdAt: claim.createdAt, + phase: 'retained', + tag: claim.tag, + expiresAt, + }, + { overwrite: true } + ); +} + /** * Creates a hooks storage implementation using the filesystem. * Implements the Storage['hooks'] interface with hook CRUD operations. @@ -120,6 +170,8 @@ export async function deleteAllHooksForRun( ): Promise { const hooksDir = path.join(basedir, 'hooks'); const files = await listJSONFiles(hooksDir); + const tokensDir = path.join(hooksDir, 'tokens'); + const now = new Date(); for (const file of files) { const hookPath = path.join(hooksDir, `${file}.json`); @@ -136,11 +188,30 @@ export async function deleteAllHooksForRun( 'tokens', `${hashToken(hook.token)}.json` ); - await deleteJSON(constraintPath); + const claim = await readJSON(constraintPath, HookTokenClaimSchema); + if (claim?.runId === runId && claim.ttlSeconds) { + await retainStartHookClaim( + constraintPath, + { ...claim, token: hook.token }, + now, + hook.createdAt + ); + } else { + await deleteJSON(constraintPath); + } await deleteJSON( hookRecoveryMarkerPath(basedir, hook.token, hook.runId, hook.hookId) ); await deleteJSON(hookPath); } } + + for (const file of await listJSONFiles(tokensDir)) { + if (file.endsWith('.recovery')) continue; + const constraintPath = path.join(tokensDir, `${file}.json`); + const claim = await readJSON(constraintPath, HookTokenClaimSchema); + if (claim?.runId === runId && claim.ttlSeconds) { + await retainStartHookClaim(constraintPath, claim, now); + } + } } diff --git a/packages/world-local/src/tag.test.ts b/packages/world-local/src/tag.test.ts index 6376ae9032..34841ba70d 100644 --- a/packages/world-local/src/tag.test.ts +++ b/packages/world-local/src/tag.test.ts @@ -1,6 +1,7 @@ import { promises as fs } from 'node:fs'; import os from 'node:os'; import path from 'node:path'; +import { SPEC_VERSION_CURRENT } from '@workflow/world'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { listJSONFiles, stripTag } from './fs.js'; import { createStorage } from './storage.js'; @@ -268,6 +269,44 @@ describe('File tagging', () => { await world1.close?.(); }); + it('should not reclaim another tag active start hook claim after ttl', async () => { + const { createLocalWorld } = await import('./index.js'); + + const world0 = createLocalWorld({ dataDir: testDir, tag: 'vitest-0' }); + const world1 = createLocalWorld({ dataDir: testDir, tag: 'vitest-1' }); + await world0.start?.(); + + const token = 'cross-tag-active-start-hook-token'; + await world0.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'dep-1', + workflowName: 'wf-0', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 1 }, + }, + }); + + await new Promise((resolve) => setTimeout(resolve, 1_100)); + + await expect( + world1.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'dep-2', + workflowName: 'wf-1', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 1 }, + }, + }) + ).rejects.toThrow(); + + await world0.close?.(); + await world1.close?.(); + }); + it('should clear events, steps, hooks, and waits', async () => { const { createLocalWorld } = await import('./index.js'); @@ -351,6 +390,49 @@ describe('File tagging', () => { await world.close?.(); }); + + it('should clear unmaterialized start hook token claims', async () => { + const { createLocalWorld } = await import('./index.js'); + + const world = createLocalWorld({ dataDir: testDir, tag: 'vitest-0' }); + await world.start?.(); + + const token = 'tagged-start-hook-token'; + await world.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'dep-1', + workflowName: 'start-hook-wf', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }, + }); + + const tokensDir = path.join(testDir, 'hooks', 'tokens'); + expect(await fs.readdir(tokensDir)).toHaveLength(1); + + await world.clear(); + + expect(await fs.readdir(tokensDir)).toHaveLength(0); + await expect( + world.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'dep-2', + workflowName: 'start-hook-wf-2', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }, + }) + ).resolves.toMatchObject({ + event: { eventType: 'run_created' }, + run: { workflowName: 'start-hook-wf-2' }, + }); + + await world.close?.(); + }); }); describe('full lifecycle with tags', () => { diff --git a/packages/world-postgres/src/drizzle/migrations/0015_add_hook_claims.sql b/packages/world-postgres/src/drizzle/migrations/0015_add_hook_claims.sql new file mode 100644 index 0000000000..040135ae45 --- /dev/null +++ b/packages/world-postgres/src/drizzle/migrations/0015_add_hook_claims.sql @@ -0,0 +1,13 @@ +CREATE TABLE IF NOT EXISTS "workflow"."workflow_hook_claims" ( + "token" varchar PRIMARY KEY NOT NULL, + "run_id" varchar NOT NULL, + "hook_id" varchar, + "phase" varchar NOT NULL, + "ttl_seconds" integer NOT NULL, + "expires_at" timestamp, + "created_at" timestamp DEFAULT now() NOT NULL +); +--> statement-breakpoint +CREATE INDEX IF NOT EXISTS "workflow_hook_claims_run_id_index" ON "workflow"."workflow_hook_claims" USING btree ("run_id"); +--> statement-breakpoint +CREATE INDEX IF NOT EXISTS "workflow_hook_claims_expires_at_index" ON "workflow"."workflow_hook_claims" USING btree ("expires_at"); diff --git a/packages/world-postgres/src/drizzle/migrations/meta/_journal.json b/packages/world-postgres/src/drizzle/migrations/meta/_journal.json index 8341c19475..0257b8799a 100644 --- a/packages/world-postgres/src/drizzle/migrations/meta/_journal.json +++ b/packages/world-postgres/src/drizzle/migrations/meta/_journal.json @@ -106,6 +106,13 @@ "when": 1780185600000, "tag": "0014_add_attr_set_event_unique_index", "breakpoints": true + }, + { + "idx": 15, + "version": "7", + "when": 1782604800000, + "tag": "0015_add_hook_claims", + "breakpoints": true } ] } diff --git a/packages/world-postgres/src/drizzle/schema.ts b/packages/world-postgres/src/drizzle/schema.ts index 1fde7ff1b7..6710cdd0df 100644 --- a/packages/world-postgres/src/drizzle/schema.ts +++ b/packages/world-postgres/src/drizzle/schema.ts @@ -219,6 +219,22 @@ export const hooks = schema.table( (tb) => [index().on(tb.runId), index().on(tb.token)] ); +export const hookClaims = schema.table( + 'workflow_hook_claims', + { + token: varchar('token').primaryKey(), + runId: varchar('run_id').notNull(), + hookId: varchar('hook_id'), + phase: varchar('phase') + .$type<'start_claim' | 'materialized' | 'retained'>() + .notNull(), + ttlSeconds: integer('ttl_seconds').notNull(), + expiresAt: timestamp('expires_at'), + createdAt: timestamp('created_at').defaultNow().notNull(), + }, + (tb) => [index().on(tb.runId), index().on(tb.expiresAt)] +); + export const waits = schema.table( 'workflow_waits', { diff --git a/packages/world-postgres/src/index.ts b/packages/world-postgres/src/index.ts index 9ad7565e06..8a26583607 100644 --- a/packages/world-postgres/src/index.ts +++ b/packages/world-postgres/src/index.ts @@ -58,6 +58,7 @@ export function createWorld( return { specVersion: SPEC_VERSION_CURRENT, + supportsExperimentalStartHook: true, ...storage, ...streamer, ...queue, diff --git a/packages/world-postgres/src/storage.ts b/packages/world-postgres/src/storage.ts index 2d531802eb..f7ffb080a0 100644 --- a/packages/world-postgres/src/storage.ts +++ b/packages/world-postgres/src/storage.ts @@ -1,5 +1,6 @@ import { EntityConflictError, + HookConflictError, HookNotFoundError, RunExpiredError, RunNotSupportedError, @@ -40,7 +41,17 @@ import { validateUlidTimestamp, WorkflowRunSchema, } from '@workflow/world'; -import { and, asc, desc, eq, gt, lt, notInArray, sql } from 'drizzle-orm'; +import { + and, + asc, + desc, + eq, + gt, + isNull, + lt, + notInArray, + sql, +} from 'drizzle-orm'; import { monotonicFactory } from 'ulid'; import { type Drizzle, Schema } from './drizzle/index.js'; import type { SerializedContent } from './drizzle/schema.js'; @@ -391,6 +402,118 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { .limit(1) .prepare('events_get_wait_for_validation'); + async function getLiveHookClaim(token: string): Promise<{ + runId: string; + phase: 'start_claim' | 'materialized' | 'retained'; + expiresAt: Date | null; + } | null> { + const [claim] = await drizzle + .select({ + runId: Schema.hookClaims.runId, + phase: Schema.hookClaims.phase, + expiresAt: Schema.hookClaims.expiresAt, + ttlSeconds: Schema.hookClaims.ttlSeconds, + createdAt: Schema.hookClaims.createdAt, + }) + .from(Schema.hookClaims) + .where(eq(Schema.hookClaims.token, token)) + .limit(1); + if (!claim) return null; + const expiresAt = + claim.expiresAt ?? + (claim.ttlSeconds > 0 + ? new Date(claim.createdAt.getTime() + claim.ttlSeconds * 1000) + : null); + if (expiresAt && expiresAt <= new Date()) { + const [run] = await drizzle + .select({ status: Schema.runs.status }) + .from(Schema.runs) + .where(eq(Schema.runs.runId, claim.runId)) + .limit(1); + if (run && !['completed', 'failed', 'cancelled'].includes(run.status)) { + return claim; + } + const [deleted] = await drizzle + .delete(Schema.hookClaims) + .where( + and( + eq(Schema.hookClaims.token, token), + eq(Schema.hookClaims.runId, claim.runId), + eq(Schema.hookClaims.phase, claim.phase), + claim.expiresAt + ? eq(Schema.hookClaims.expiresAt, claim.expiresAt) + : isNull(Schema.hookClaims.expiresAt) + ) + ) + .returning({ token: Schema.hookClaims.token }); + return deleted ? null : getLiveHookClaim(token); + } + return claim; + } + + async function retainHookClaimsForRun( + runId: string, + completedAt: Date + ): Promise { + const hooksForRun = await drizzle + .select({ + token: Schema.hooks.token, + createdAt: Schema.hooks.createdAt, + }) + .from(Schema.hooks) + .where(eq(Schema.hooks.runId, runId)); + const hookCreatedAtByToken = new Map( + hooksForRun.map((hook) => [hook.token, hook.createdAt]) + ); + + const claimsForRun = await drizzle + .select() + .from(Schema.hookClaims) + .where(eq(Schema.hookClaims.runId, runId)); + + for (const claim of claimsForRun) { + const hookCreatedAt = hookCreatedAtByToken.get(claim.token); + if (claim.ttlSeconds <= 0) { + await drizzle + .delete(Schema.hookClaims) + .where( + and( + eq(Schema.hookClaims.token, claim.token), + eq(Schema.hookClaims.runId, runId), + eq(Schema.hookClaims.phase, claim.phase) + ) + ); + continue; + } + + const ttlBase = hookCreatedAt ?? claim.createdAt; + const expiresAt = new Date( + Math.max( + ttlBase.getTime() + claim.ttlSeconds * 1000, + claim.expiresAt?.getTime() ?? 0, + completedAt.getTime() + ) + ); + + await drizzle + .update(Schema.hookClaims) + .set({ + expiresAt, + phase: 'retained', + }) + .where( + and( + eq(Schema.hookClaims.token, claim.token), + eq(Schema.hookClaims.runId, runId), + eq(Schema.hookClaims.phase, claim.phase), + claim.hookId + ? eq(Schema.hookClaims.hookId, claim.hookId) + : isNull(Schema.hookClaims.hookId) + ) + ); + } + } + return { async create(runId, data, params): Promise { const eventId = `wevt_${ulid()}`; @@ -425,6 +548,7 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { // the step (the caller won the create-claim). Surfaced on EventResult // as the runtime's exactly-once ownership signal. let stepCreatedLazily = false; + let eventCreatedWithRun: { createdAt: Date } | undefined; const now = new Date(); // Helper to check if run is in terminal state @@ -478,6 +602,10 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { executionContext?: Record; attributes?: Record; allowReservedAttributes?: true; + experimentalStartHook?: { + token: string; + ttlSeconds: number; + }; }; if ( runInputData.deploymentId && @@ -493,43 +621,98 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { runInputData.allowReservedAttributes === true, } ); - // Create run + run_created event atomically. The - // transaction ensures we never have an orphaned run - // without its run_created event. - const [inserted] = await drizzle - .insert(Schema.runs) - .values({ - runId: effectiveRunId, - deploymentId: runInputData.deploymentId, - workflowName: runInputData.workflowName, - specVersion: effectiveSpecVersion, - input: runInputData.input as SerializedContent, - executionContext: runInputData.executionContext as - | SerializedContent - | undefined, - attributes: runInputData.attributes, - status: 'pending', - }) - .onConflictDoNothing() - .returning(); + const deploymentId = runInputData.deploymentId; + const workflowName = runInputData.workflowName; + const input = runInputData.input; + const startHook = runInputData.experimentalStartHook; + let hasSameRunStartClaim = false; + if (startHook) { + const [existingHook] = await getHookByToken.execute({ + token: startHook.token, + }); + if (existingHook && existingHook.runId !== effectiveRunId) { + throw new HookConflictError( + startHook.token, + existingHook.runId + ); + } + const existingClaim = await getLiveHookClaim(startHook.token); + if (existingClaim) { + if (existingClaim.runId !== effectiveRunId) { + throw new HookConflictError( + startHook.token, + existingClaim.runId + ); + } + hasSameRunStartClaim = true; + } + } + // Create run + optional start claim + run_created event + // atomically. The transaction ensures we never have an + // orphaned run without its run_created event or start claim. + const inserted = await drizzle.transaction(async (tx) => { + const [createdRun] = await tx + .insert(Schema.runs) + .values({ + runId: effectiveRunId, + deploymentId, + workflowName, + specVersion: effectiveSpecVersion, + input: input as SerializedContent, + executionContext: runInputData.executionContext as + | SerializedContent + | undefined, + attributes: runInputData.attributes, + status: 'pending', + }) + .onConflictDoNothing() + .returning(); + if (!createdRun) return undefined; + + if (startHook && !hasSameRunStartClaim) { + const [createdClaim] = await tx + .insert(Schema.hookClaims) + .values({ + token: startHook.token, + runId: effectiveRunId, + phase: 'start_claim', + ttlSeconds: startHook.ttlSeconds, + }) + .onConflictDoNothing() + .returning({ token: Schema.hookClaims.token }); + if (!createdClaim) { + const [existing] = await tx + .select({ runId: Schema.hookClaims.runId }) + .from(Schema.hookClaims) + .where(eq(Schema.hookClaims.token, startHook.token)) + .limit(1); + if (existing?.runId !== effectiveRunId) { + throw new HookConflictError( + startHook.token, + existing?.runId + ); + } + } + } - if (inserted) { const runCreatedEventId = `wevt_${ulid()}`; - await drizzle.insert(events).values({ + await tx.insert(events).values({ runId: effectiveRunId, eventId: runCreatedEventId, eventType: 'run_created', eventData: { - deploymentId: runInputData.deploymentId, - workflowName: runInputData.workflowName, - input: runInputData.input, + deploymentId, + workflowName, + input, executionContext: runInputData.executionContext, attributes: runInputData.attributes, allowReservedAttributes: runInputData.allowReservedAttributes, + experimentalStartHook: startHook, }, specVersion: effectiveSpecVersion, }); - } + return createdRun; + }); const createdRun = inserted; if (createdRun) { @@ -772,6 +955,10 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { executionContext?: Record; attributes?: Record; allowReservedAttributes?: true; + experimentalStartHook?: { + token: string; + ttlSeconds: number; + }; }; validateAttributeChanges( Object.entries(eventData.attributes ?? {}).map(([key, value]) => ({ @@ -782,23 +969,147 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { allowReservedAttributes: eventData.allowReservedAttributes === true, } ); - const [runValue] = await drizzle - .insert(Schema.runs) - .values({ - runId: effectiveRunId, - deploymentId: eventData.deploymentId, - workflowName: eventData.workflowName, - // Propagate specVersion from the event to the run entity - specVersion: effectiveSpecVersion, - input: eventData.input as SerializedContent, - executionContext: eventData.executionContext as - | SerializedContent - | undefined, - attributes: eventData.attributes, - status: 'pending', - }) - .onConflictDoNothing() - .returning(); + const runValues = eventData.experimentalStartHook + ? await (async () => { + const startHook = eventData.experimentalStartHook!; + const [existingHook] = await getHookByToken.execute({ + token: startHook.token, + }); + if (existingHook && existingHook.runId !== effectiveRunId) { + throw new HookConflictError( + startHook.token, + existingHook.runId + ); + } + const existingClaim = await getLiveHookClaim(startHook.token); + if (existingClaim) { + if (existingClaim.runId === effectiveRunId) { + const [existingEvent] = await drizzle + .select({ eventId: events.eventId }) + .from(events) + .where( + and( + eq(events.runId, effectiveRunId), + eq(events.eventType, 'run_created') + ) + ) + .orderBy(asc(events.eventId)) + .limit(1); + if (existingEvent) { + throw new EntityConflictError( + `Run "${effectiveRunId}" already exists` + ); + } + + const insertedRuns = await drizzle + .insert(Schema.runs) + .values({ + runId: effectiveRunId, + deploymentId: eventData.deploymentId, + workflowName: eventData.workflowName, + // Propagate specVersion from the event to the run entity + specVersion: effectiveSpecVersion, + input: eventData.input as SerializedContent, + executionContext: eventData.executionContext as + | SerializedContent + | undefined, + attributes: eventData.attributes, + status: 'pending', + }) + .onConflictDoNothing() + .returning(); + if (!insertedRuns[0]) { + const [existingRun] = await drizzle + .select() + .from(Schema.runs) + .where(eq(Schema.runs.runId, effectiveRunId)) + .limit(1); + if (existingRun) return [existingRun]; + throw new WorkflowRunNotFoundError(effectiveRunId); + } + return insertedRuns; + } + throw new HookConflictError( + startHook.token, + existingClaim.runId + ); + } + const result = await drizzle.transaction(async (tx) => { + const [createdClaim] = await tx + .insert(Schema.hookClaims) + .values({ + token: startHook.token, + runId: effectiveRunId, + phase: 'start_claim', + ttlSeconds: startHook.ttlSeconds, + }) + .onConflictDoNothing() + .returning({ runId: Schema.hookClaims.runId }); + if (!createdClaim) { + const [existing] = await tx + .select({ runId: Schema.hookClaims.runId }) + .from(Schema.hookClaims) + .where(eq(Schema.hookClaims.token, startHook.token)) + .limit(1); + throw new HookConflictError(startHook.token, existing?.runId); + } + + const insertedRuns = await tx + .insert(Schema.runs) + .values({ + runId: effectiveRunId, + deploymentId: eventData.deploymentId, + workflowName: eventData.workflowName, + // Propagate specVersion from the event to the run entity + specVersion: effectiveSpecVersion, + input: eventData.input as SerializedContent, + executionContext: eventData.executionContext as + | SerializedContent + | undefined, + attributes: eventData.attributes, + status: 'pending', + }) + .onConflictDoNothing() + .returning(); + if (!insertedRuns[0]) { + throw new EntityConflictError( + `Run "${effectiveRunId}" already exists` + ); + } + const [createdEvent] = await tx + .insert(events) + .values({ + runId: effectiveRunId, + eventId, + correlationId: data.correlationId, + eventType: data.eventType, + eventData: data.eventData, + specVersion: effectiveSpecVersion, + }) + .returning({ createdAt: events.createdAt }); + return { event: createdEvent, runs: insertedRuns }; + }); + eventCreatedWithRun = result.event; + return result.runs; + })() + : await drizzle + .insert(Schema.runs) + .values({ + runId: effectiveRunId, + deploymentId: eventData.deploymentId, + workflowName: eventData.workflowName, + // Propagate specVersion from the event to the run entity + specVersion: effectiveSpecVersion, + input: eventData.input as SerializedContent, + executionContext: eventData.executionContext as + | SerializedContent + | undefined, + attributes: eventData.attributes, + status: 'pending', + }) + .onConflictDoNothing() + .returning(); + const [runValue] = runValues; if (runValue) { run = deserializeRunError(compact(runValue)); } @@ -877,7 +1188,8 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { ); } } - // Delete all hooks and waits for this run to allow token reuse + // Close active hooks/waits; retained start-hook claims expire separately. + await retainHookClaimsForRun(effectiveRunId, now); await Promise.all([ drizzle .delete(Schema.hooks) @@ -929,7 +1241,8 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { ); } } - // Delete all hooks and waits for this run to allow token reuse + // Close active hooks/waits; retained start-hook claims expire separately. + await retainHookClaimsForRun(effectiveRunId, now); await Promise.all([ drizzle .delete(Schema.hooks) @@ -974,7 +1287,8 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { ); } } - // Delete all hooks and waits for this run to allow token reuse + // Close active hooks/waits; retained start-hook claims expire separately. + await retainHookClaimsForRun(effectiveRunId, now); await Promise.all([ drizzle .delete(Schema.hooks) @@ -1366,7 +1680,12 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { const [existingHook] = await getHookByToken.execute({ token: eventData.token, }); - if (existingHook) { + const existingClaim = await getLiveHookClaim(eventData.token); + const materializesStartClaim = + !existingHook && + existingClaim?.runId === effectiveRunId && + existingClaim.phase === 'start_claim'; + if (existingHook || existingClaim?.runId !== undefined) { // Idempotency: if the existing hook is the *same* (runId, hookId) // we are trying to create, this is either a duplicate / replayed // processing of the same hook_created (not a real conflict), or @@ -1386,7 +1705,7 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { // row already has the desired state) and fall through to // the events INSERT below, completing the partial write. if ( - existingHook.runId === effectiveRunId && + existingHook?.runId === effectiveRunId && existingHook.hookId === data.correlationId ) { const [existingEvent] = await getHookCreatedEvent.execute({ @@ -1399,6 +1718,21 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { `Hook "${data.correlationId}" already created` ); } + if (existingClaim?.phase === 'start_claim') { + await drizzle + .update(Schema.hookClaims) + .set({ + hookId: data.correlationId!, + phase: 'materialized', + }) + .where( + and( + eq(Schema.hookClaims.token, eventData.token), + eq(Schema.hookClaims.runId, effectiveRunId), + eq(Schema.hookClaims.phase, 'start_claim') + ) + ); + } // Orphaned hook row: hook row exists but no hook_created // event in the log. Skip the hook insert below (the row // already exists with our (runId, hookId)) and let the @@ -1420,65 +1754,225 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { // (runId, hookId) holds this token. Create a hook_conflict // event instead of throwing 409 — this lets the workflow // continue and fail gracefully when the hook is awaited. - const conflictEventData = { - token: eventData.token, - conflictingRunId: existingHook.runId, - }; + const conflictingRunId = + existingHook?.runId ?? existingClaim!.runId; + if (materializesStartClaim) { + // This run reserved the token during start(); materialize the + // actual workflow hook below. + } else { + const conflictEventData = { + token: eventData.token, + conflictingRunId, + }; - const [conflictValue] = await drizzle - .insert(events) - .values({ + const [conflictValue] = await drizzle + .insert(events) + .values({ + runId: effectiveRunId, + eventId, + correlationId: data.correlationId, + eventType: 'hook_conflict', + eventData: conflictEventData, + specVersion: effectiveSpecVersion, + }) + .returning({ createdAt: events.createdAt }); + + if (!conflictValue) { + throw new EntityConflictError( + `Event ${eventId} could not be created` + ); + } + + const conflictResult = { + eventType: 'hook_conflict' as const, + correlationId: data.correlationId, + eventData: conflictEventData, + ...conflictValue, runId: effectiveRunId, eventId, + }; + const parsedConflict = EventSchema.parse(conflictResult); + const resolveData = params?.resolveData ?? 'all'; + return { + event: stripEventDataRefs(parsedConflict, resolveData), + run, + step, + hook: undefined, + }; + } + } + } + if (!existingHook || existingClaim?.runId === effectiveRunId) { + const hookValues = { + runId: effectiveRunId, + hookId: data.correlationId!, + token: eventData.token, + metadata: eventData.metadata as SerializedContent, + ownerId: '', // TODO: get from context + projectId: '', // TODO: get from context + environment: '', // TODO: get from context + // Propagate specVersion from the event to the hook entity + specVersion: effectiveSpecVersion, + isWebhook: eventData.isWebhook, + isSystem: eventData.isSystem ?? false, + }; + const hookValue = + !existingHook && !existingClaim + ? await drizzle.transaction(async (tx) => { + const [createdClaim] = await tx + .insert(Schema.hookClaims) + .values({ + token: eventData.token, + runId: effectiveRunId, + hookId: data.correlationId!, + phase: 'materialized', + ttlSeconds: 0, + }) + .onConflictDoNothing() + .returning({ token: Schema.hookClaims.token }); + if (!createdClaim) return undefined; + const [createdHook] = await tx + .insert(Schema.hooks) + .values(hookValues) + .onConflictDoNothing() + .returning(); + if (!createdHook) { + throw new EntityConflictError( + `Hook "${data.correlationId}" already created` + ); + } + return createdHook; + }) + : materializesStartClaim + ? await drizzle.transaction(async (tx) => { + const [claimed] = await tx + .update(Schema.hookClaims) + .set({ + hookId: data.correlationId!, + phase: 'materialized', + }) + .where( + and( + eq(Schema.hookClaims.token, eventData.token), + eq(Schema.hookClaims.runId, effectiveRunId), + eq(Schema.hookClaims.phase, 'start_claim'), + isNull(Schema.hookClaims.hookId) + ) + ) + .returning({ token: Schema.hookClaims.token }); + if (!claimed) return undefined; + const [createdHook] = await tx + .insert(Schema.hooks) + .values(hookValues) + .onConflictDoNothing() + .returning(); + if (!createdHook) { + throw new EntityConflictError( + `Hook "${data.correlationId}" already created` + ); + } + return createdHook; + }) + : ( + await drizzle + .insert(Schema.hooks) + .values(hookValues) + .onConflictDoNothing() + .returning() + )[0]; + if ( + !hookValue && + ((!existingHook && !existingClaim) || materializesStartClaim) + ) { + const racedClaim = await getLiveHookClaim(eventData.token); + const [racedHook] = await getHookByToken.execute({ + token: eventData.token, + }); + if ( + racedHook?.runId === effectiveRunId && + racedHook.hookId === data.correlationId + ) { + const [existingEvent] = await getHookCreatedEvent.execute({ + runId: effectiveRunId, + correlationId: data.correlationId, + eventType: 'hook_created', + }); + if (existingEvent) { + throw new EntityConflictError( + `Hook "${data.correlationId}" already created` + ); + } + const [recoveredHookValue] = await drizzle + .select() + .from(Schema.hooks) + .where(eq(Schema.hooks.hookId, data.correlationId!)) + .limit(1); + if (recoveredHookValue) { + recoveredHookValue.metadata ||= recoveredHookValue.metadataJson; + hook = HookSchema.parse(compact(recoveredHookValue)); + } + } else { + const conflictEventData = { + token: eventData.token, + ...((racedHook?.runId ?? racedClaim?.runId) + ? { + conflictingRunId: racedHook?.runId ?? racedClaim?.runId, + } + : {}), + }; + const [conflictValue] = await drizzle + .insert(events) + .values({ + runId: effectiveRunId, + eventId, + correlationId: data.correlationId, + eventType: 'hook_conflict', + eventData: conflictEventData, + specVersion: effectiveSpecVersion, + }) + .returning({ createdAt: events.createdAt }); + if (!conflictValue) { + throw new EntityConflictError( + `Event ${eventId} could not be created` + ); + } + const parsedConflict = EventSchema.parse({ + eventType: 'hook_conflict' as const, correlationId: data.correlationId, - eventType: 'hook_conflict', eventData: conflictEventData, + ...conflictValue, + runId: effectiveRunId, + eventId, specVersion: effectiveSpecVersion, - }) - .returning({ createdAt: events.createdAt }); - - if (!conflictValue) { - throw new EntityConflictError( - `Event ${eventId} could not be created` - ); + }); + const resolveData = params?.resolveData ?? 'all'; + return { + event: stripEventDataRefs(parsedConflict, resolveData), + run, + step, + hook: undefined, + }; } - - const conflictResult = { - eventType: 'hook_conflict' as const, - correlationId: data.correlationId, - eventData: conflictEventData, - ...conflictValue, - runId: effectiveRunId, - eventId, - }; - const parsedConflict = EventSchema.parse(conflictResult); - const resolveData = params?.resolveData ?? 'all'; - return { - event: stripEventDataRefs(parsedConflict, resolveData), - run, - step, - hook: undefined, - }; } - } else { - const [hookValue] = await drizzle - .insert(Schema.hooks) - .values({ - runId: effectiveRunId, - hookId: data.correlationId!, - token: eventData.token, - metadata: eventData.metadata as SerializedContent, - ownerId: '', // TODO: get from context - projectId: '', // TODO: get from context - environment: '', // TODO: get from context - // Propagate specVersion from the event to the hook entity - specVersion: effectiveSpecVersion, - isWebhook: eventData.isWebhook, - isSystem: eventData.isSystem ?? false, - }) - .onConflictDoNothing() - .returning(); if (hookValue) { + if ( + existingClaim?.phase === 'start_claim' && + !materializesStartClaim + ) { + await drizzle + .update(Schema.hookClaims) + .set({ + hookId: data.correlationId!, + phase: 'materialized', + }) + .where( + and( + eq(Schema.hookClaims.token, eventData.token), + eq(Schema.hookClaims.runId, effectiveRunId), + eq(Schema.hookClaims.phase, 'start_claim') + ) + ); + } hookValue.metadata ||= hookValue.metadataJson; hook = HookSchema.parse(compact(hookValue)); } @@ -1489,15 +1983,54 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { // Uses DELETE ... RETURNING to ensure only one concurrent caller // succeeds — if no rows are returned, the hook was already disposed. if (data.eventType === 'hook_disposed' && data.correlationId) { - const [deleted] = await drizzle - .delete(Schema.hooks) - .where(eq(Schema.hooks.hookId, data.correlationId)) - .returning({ hookId: Schema.hooks.hookId }); - if (!deleted) { - throw new EntityConflictError( - `Hook "${data.correlationId}" already disposed` - ); + const hookId = data.correlationId; + const [hookToDispose] = await drizzle + .select({ + hookId: Schema.hooks.hookId, + token: Schema.hooks.token, + createdAt: Schema.hooks.createdAt, + }) + .from(Schema.hooks) + .where(eq(Schema.hooks.hookId, hookId)) + .limit(1); + if (!hookToDispose) { + throw new EntityConflictError(`Hook "${hookId}" already disposed`); } + const [claim] = await drizzle + .select() + .from(Schema.hookClaims) + .where(eq(Schema.hookClaims.token, hookToDispose.token)) + .limit(1); + await drizzle.transaction(async (tx) => { + if (!claim || claim.ttlSeconds <= 0) { + await tx + .delete(Schema.hookClaims) + .where(eq(Schema.hookClaims.token, hookToDispose.token)); + } else { + const expiresAt = new Date( + Math.max( + hookToDispose.createdAt.getTime() + claim.ttlSeconds * 1000, + now.getTime() + ) + ); + await tx + .update(Schema.hookClaims) + .set({ + expiresAt, + hookId: hookToDispose.hookId, + phase: 'retained', + }) + .where(eq(Schema.hookClaims.token, hookToDispose.token)); + } + + const [deleted] = await tx + .delete(Schema.hooks) + .where(eq(Schema.hooks.hookId, hookId)) + .returning({ hookId: Schema.hooks.hookId }); + if (!deleted) { + throw new EntityConflictError(`Hook "${hookId}" already disposed`); + } + }); } // Handle wait_created event: create wait entity @@ -1605,54 +2138,56 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { storedEventData = undefined; } - let value: { createdAt: Date } | undefined; - try { - [value] = await drizzle - .insert(events) - .values({ - runId: effectiveRunId, - eventId, - correlationId: data.correlationId, - eventType: data.eventType, - eventData: storedEventData, - specVersion: effectiveSpecVersion, - }) - .returning({ createdAt: events.createdAt }); - } catch (err) { - // Translate unique-violation on the correlated-event partial index - // (workflow_events_entity_creation_unique) into EntityConflictError - // so the runtime's existing dedup catch path can handle it. Without - // this, two concurrent invocations producing identical - // correlationIds (e.g. snapshot runtime deterministic ULIDs) would - // surface as unhandled DB errors instead of dedup signals. - // Drizzle wraps the underlying pg error in DrizzleQueryError; the - // pg error (with .code === '23505') lives on .cause. We additionally - // gate on the violated constraint name so other 23505 violations on - // these event types (e.g. the events primary key, or any future - // unique constraint we might add) don't get misclassified as a - // correlationId conflict. - const isDeduplicatedCorrelatedEvent = - data.eventType === 'step_created' || - data.eventType === 'hook_created' || - data.eventType === 'wait_created' || - (data.eventType === 'attr_set' && - data.eventData.writer.type === 'workflow'); - const pgErr = (err as { code?: string; constraint?: string }).code - ? (err as { code?: string; constraint?: string }) - : ((err as { cause?: { code?: string; constraint?: string } }) - .cause ?? {}); - const pgCode = pgErr.code; - const pgConstraint = pgErr.constraint; - if ( - isDeduplicatedCorrelatedEvent && - pgCode === '23505' && - pgConstraint === 'workflow_events_entity_creation_unique' - ) { - throw new EntityConflictError( - `${data.eventType} for correlationId "${data.correlationId}" already exists in run "${effectiveRunId}"` - ); + let value: { createdAt: Date } | undefined = eventCreatedWithRun; + if (!value) { + try { + [value] = await drizzle + .insert(events) + .values({ + runId: effectiveRunId, + eventId, + correlationId: data.correlationId, + eventType: data.eventType, + eventData: storedEventData, + specVersion: effectiveSpecVersion, + }) + .returning({ createdAt: events.createdAt }); + } catch (err) { + // Translate unique-violation on the correlated-event partial index + // (workflow_events_entity_creation_unique) into EntityConflictError + // so the runtime's existing dedup catch path can handle it. Without + // this, two concurrent invocations producing identical + // correlationIds (e.g. snapshot runtime deterministic ULIDs) would + // surface as unhandled DB errors instead of dedup signals. + // Drizzle wraps the underlying pg error in DrizzleQueryError; the + // pg error (with .code === '23505') lives on .cause. We additionally + // gate on the violated constraint name so other 23505 violations on + // these event types (e.g. the events primary key, or any future + // unique constraint we might add) don't get misclassified as a + // correlationId conflict. + const isDeduplicatedCorrelatedEvent = + data.eventType === 'step_created' || + data.eventType === 'hook_created' || + data.eventType === 'wait_created' || + (data.eventType === 'attr_set' && + data.eventData.writer.type === 'workflow'); + const pgErr = (err as { code?: string; constraint?: string }).code + ? (err as { code?: string; constraint?: string }) + : ((err as { cause?: { code?: string; constraint?: string } }) + .cause ?? {}); + const pgCode = pgErr.code; + const pgConstraint = pgErr.constraint; + if ( + isDeduplicatedCorrelatedEvent && + pgCode === '23505' && + pgConstraint === 'workflow_events_entity_creation_unique' + ) { + throw new EntityConflictError( + `${data.eventType} for correlationId "${data.correlationId}" already exists in run "${effectiveRunId}"` + ); + } + throw err; } - throw err; } if (!value) { throw new EntityConflictError(`Event ${eventId} could not be created`); diff --git a/packages/world-postgres/test/storage.test.ts b/packages/world-postgres/test/storage.test.ts index 2a92cabc51..6a0a9f64e2 100644 --- a/packages/world-postgres/test/storage.test.ts +++ b/packages/world-postgres/test/storage.test.ts @@ -1,9 +1,12 @@ import { execSync } from 'node:child_process'; import { PostgreSqlContainer } from '@testcontainers/postgresql'; +import { EntityConflictError, HookConflictError } from '@workflow/errors'; import type { Hook, Step, WorkflowRun } from '@workflow/world'; import { SPEC_VERSION_CURRENT } from '@workflow/world'; import { encode } from 'cbor-x'; +import { eq } from 'drizzle-orm'; import { Pool } from 'pg'; +import { monotonicFactory } from 'ulid'; import { afterAll, beforeAll, @@ -34,6 +37,7 @@ async function createRun( input: Uint8Array; executionContext?: Record; attributes?: Record; + experimentalStartHook?: { token: string; ttlSeconds: number }; } ): Promise { const result = await events.create(null, { @@ -1491,6 +1495,409 @@ describe('Storage (Postgres integration)', () => { expect(result.hook).toBeUndefined(); }); + it('should reject duplicate start hook claims before hook materialization', async () => { + const token = 'postgres-start-claim-token'; + const first = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }); + + await expect( + createRun(events, { + deploymentId: 'deployment-456', + workflowName: 'test-workflow-2', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }) + ).rejects.toThrow(HookConflictError); + + await expect( + events.create(first.runId, { + eventType: 'hook_created' as const, + correlationId: 'hook_start', + eventData: { token }, + }) + ).resolves.toMatchObject({ + event: { eventType: 'hook_created' }, + hook: { + runId: first.runId, + hookId: 'hook_start', + token, + }, + }); + }); + + it('should reserve start hook claims during resilient run_started bootstrap', async () => { + const token = 'postgres-resilient-start-claim-token'; + const runId = `wrun_${monotonicFactory()()}`; + + const started = await events.create(runId, { + eventType: 'run_started' as const, + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }, + }); + + expect(started.run?.runId).toBe(runId); + await expect( + createRun(events, { + deploymentId: 'deployment-456', + workflowName: 'test-workflow-2', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }) + ).rejects.toThrow(HookConflictError); + + await expect( + events.create(runId, { + eventType: 'hook_created' as const, + correlationId: 'hook_resilient_start', + eventData: { token }, + }) + ).resolves.toMatchObject({ + event: { eventType: 'hook_created' }, + hook: { runId, hookId: 'hook_resilient_start', token }, + }); + }); + + it('should materialize only one hook for a reserved start hook token', async () => { + const token = 'postgres-concurrent-start-claim-materialize-token'; + const first = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }); + + const results = await Promise.all([ + events.create(first.runId, { + eventType: 'hook_created' as const, + correlationId: 'hook_start_a', + eventData: { token }, + }), + events.create(first.runId, { + eventType: 'hook_created' as const, + correlationId: 'hook_start_b', + eventData: { token }, + }), + ]); + + expect(results.map((result) => result.event.eventType).sort()).toEqual([ + 'hook_conflict', + 'hook_created', + ]); + const persistedHooks = await drizzle + .select() + .from(DrizzleSchema.hooks) + .where(eq(DrizzleSchema.hooks.token, token)); + expect(persistedHooks).toHaveLength(1); + }); + + it('should reject experimental starts when an active ordinary hook owns the token', async () => { + const token = 'postgres-active-ordinary-hook-token'; + await createHook(events, testRunId, { + hookId: 'hook_active_ordinary', + token, + }); + + await expect( + createRun(events, { + deploymentId: 'deployment-456', + workflowName: 'test-workflow-2', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }) + ).rejects.toThrow(HookConflictError); + }); + + it('should treat a same-run start hook retry as a run conflict', async () => { + const token = 'postgres-same-run-start-claim-token'; + const first = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }); + + await expect( + events.create(first.runId, { + eventType: 'run_created' as const, + eventData: { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }, + }) + ).rejects.toThrow(EntityConflictError); + }); + + it('should repair a same-run start hook claim when the run_created event is missing', async () => { + const token = 'postgres-missing-run-created-start-claim-token'; + const runId = `wrun_${monotonicFactory()()}`; + + await drizzle.insert(DrizzleSchema.runs).values({ + runId, + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + status: 'pending', + input: new Uint8Array(), + attributes: {}, + specVersion: SPEC_VERSION_CURRENT, + }); + await drizzle.insert(DrizzleSchema.hookClaims).values({ + token, + runId, + phase: 'start_claim', + ttlSeconds: 60, + }); + + await expect( + events.create(runId, { + eventType: 'run_created' as const, + eventData: { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }, + }) + ).resolves.toMatchObject({ + event: { eventType: 'run_created' }, + run: { runId }, + }); + + const repairedEvents = await drizzle + .select() + .from(DrizzleSchema.events) + .where(eq(DrizzleSchema.events.runId, runId)); + expect(repairedEvents).toHaveLength(1); + }); + + it('should retain a disposed start hook claim until ttl expiry', async () => { + const token = 'postgres-disposed-start-claim-token'; + const first = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }); + + await events.create(first.runId, { + eventType: 'hook_created' as const, + correlationId: 'hook_start_disposed', + eventData: { token }, + }); + await events.create(first.runId, { + eventType: 'hook_disposed' as const, + correlationId: 'hook_start_disposed', + }); + + await expect( + createRun(events, { + deploymentId: 'deployment-456', + workflowName: 'test-workflow-2', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }) + ).rejects.toThrow(HookConflictError); + }); + + it('should not materialize a retained start hook claim again', async () => { + const token = 'postgres-retained-start-claim-token'; + const first = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }); + + await events.create(first.runId, { + eventType: 'hook_created' as const, + correlationId: 'hook_start_retained', + eventData: { token }, + }); + await events.create(first.runId, { + eventType: 'hook_disposed' as const, + correlationId: 'hook_start_retained', + }); + + const result = await events.create(first.runId, { + eventType: 'hook_created' as const, + correlationId: 'hook_start_retained_again', + eventData: { token }, + }); + + expect(result.event.eventType).toBe('hook_conflict'); + expect(result.hook).toBeUndefined(); + }); + + it('should retain a recovered start hook claim after orphaned hook row materialization', async () => { + const token = 'postgres-recovered-start-claim-token'; + const hookId = 'hook_recovered_start_claim'; + const first = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }); + + await drizzle.insert(DrizzleSchema.hooks).values({ + runId: first.runId, + hookId, + token, + ownerId: '', + projectId: '', + environment: '', + specVersion: SPEC_VERSION_CURRENT, + isWebhook: false, + isSystem: false, + }); + + await expect( + events.create(first.runId, { + eventType: 'hook_created' as const, + correlationId: hookId, + eventData: { token }, + }) + ).resolves.toMatchObject({ + event: { eventType: 'hook_created' }, + hook: { hookId, token }, + }); + + const [materializedClaim] = await drizzle + .select() + .from(DrizzleSchema.hookClaims) + .where(eq(DrizzleSchema.hookClaims.token, token)) + .limit(1); + expect(materializedClaim).toMatchObject({ + runId: first.runId, + hookId, + phase: 'materialized', + }); + + await updateRun(events, first.runId, 'run_completed', { + output: new Uint8Array(), + }); + + const [retainedClaim] = await drizzle + .select() + .from(DrizzleSchema.hookClaims) + .where(eq(DrizzleSchema.hookClaims.token, token)) + .limit(1); + expect(retainedClaim).toMatchObject({ + runId: first.runId, + hookId, + phase: 'retained', + }); + expect(retainedClaim?.expiresAt?.getTime()).toBeGreaterThan(Date.now()); + + await expect( + createRun(events, { + deploymentId: 'deployment-456', + workflowName: 'test-workflow-2', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }) + ).rejects.toThrow(HookConflictError); + }); + + it('should retain an unmaterialized start hook claim until ttl expiry', async () => { + const token = 'postgres-cancelled-unmaterialized-start-claim-token'; + const first = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }); + + await events.create(first.runId, { + eventType: 'run_cancelled' as const, + }); + + await expect( + createRun(events, { + deploymentId: 'deployment-456', + workflowName: 'test-workflow-2', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }) + ).rejects.toThrow(HookConflictError); + + const past = new Date(Date.now() - 2_000); + await drizzle + .update(DrizzleSchema.hookClaims) + .set({ createdAt: past, expiresAt: past }) + .where(eq(DrizzleSchema.hookClaims.token, token)); + + await expect( + createRun(events, { + deploymentId: 'deployment-789', + workflowName: 'test-workflow-3', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }) + ).resolves.toMatchObject({ + workflowName: 'test-workflow-3', + }); + }); + + it('should keep an expired disposed start hook claim while the run is active', async () => { + const token = 'postgres-active-expired-start-claim-token'; + const first = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 1 }, + }); + + await events.create(first.runId, { + eventType: 'hook_created' as const, + correlationId: 'hook_start_expired_active', + eventData: { token }, + }); + await events.create(first.runId, { + eventType: 'hook_disposed' as const, + correlationId: 'hook_start_expired_active', + }); + + const past = new Date(Date.now() - 2_000); + await drizzle + .update(DrizzleSchema.hookClaims) + .set({ createdAt: past, expiresAt: past }) + .where(eq(DrizzleSchema.hookClaims.token, token)); + + await expect( + createRun(events, { + deploymentId: 'deployment-456', + workflowName: 'test-workflow-2', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 1 }, + }) + ).rejects.toThrow(HookConflictError); + + await updateRun(events, first.runId, 'run_completed', { + output: new Uint8Array(), + }); + + await expect( + createRun(events, { + deploymentId: 'deployment-789', + workflowName: 'test-workflow-3', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 1 }, + }) + ).resolves.toMatchObject({ + workflowName: 'test-workflow-3', + }); + }); + it('should allow token reuse after hook is disposed', async () => { const token = 'reusable-token-test'; diff --git a/packages/world-vercel/src/events.ts b/packages/world-vercel/src/events.ts index 5aab8d0cb4..8e3eb46c67 100644 --- a/packages/world-vercel/src/events.ts +++ b/packages/world-vercel/src/events.ts @@ -214,6 +214,12 @@ type MetaSourceField = | 'writer' | 'allowReservedAttributes'; +// Vercel start-hook admission cannot be enabled until workflow-server and +// VQS enqueue are one recoverable backend operation. `createVercelWorld` +// does not advertise the capability, so core `start()` rejects before this +// field could reach the v4 event path. +type UnsupportedSourceField = 'experimentalStartHook'; + /** * Compile-time guard that the v4 `eventData` wire allowlist is exhaustive * against the @workflow/world event schemas. @@ -228,7 +234,10 @@ type MetaSourceField = * the constraint '[never, never]'` — the historical "silently dropped" * footgun, now a build break that names the field. */ -type Unhandled = Exclude; +type Unhandled = Exclude< + EventDataField, + PayloadField | MetaSourceField | UnsupportedSourceField +>; type Stale = Exclude; function assertEventDataWireContractExhaustive< _Check extends [never, never], diff --git a/packages/world/src/events.ts b/packages/world/src/events.ts index 00d7eb017d..facd200e92 100644 --- a/packages/world/src/events.ts +++ b/packages/world/src/events.ts @@ -276,6 +276,11 @@ const AttrSetEventSchema = BaseEventSchema.extend({ }), }); +const ExperimentalStartHookSchema = z.object({ + token: z.string().min(1), + ttlSeconds: z.number().int().positive(), +}); + // ============================================================================= // Run lifecycle events // ============================================================================= @@ -293,6 +298,7 @@ const RunCreatedEventSchema = BaseEventSchema.extend({ executionContext: z.record(z.string(), z.any()).optional(), attributes: z.record(z.string(), z.string()).optional(), allowReservedAttributes: z.literal(true).optional(), + experimentalStartHook: ExperimentalStartHookSchema.optional(), }), }); @@ -315,6 +321,7 @@ const RunStartedEventSchema = BaseEventSchema.extend({ executionContext: z.record(z.string(), z.any()).optional(), attributes: z.record(z.string(), z.string()).optional(), allowReservedAttributes: z.literal(true).optional(), + experimentalStartHook: ExperimentalStartHookSchema.optional(), }) .optional(), }); diff --git a/packages/world/src/interfaces.ts b/packages/world/src/interfaces.ts index a30dae2ab9..496c389b5a 100644 --- a/packages/world/src/interfaces.ts +++ b/packages/world/src/interfaces.ts @@ -285,6 +285,12 @@ export interface World extends Queue, Streamer, Storage { */ specVersion?: number; + /** + * Whether this World atomically reserves `run_created.eventData.experimentalStartHook` + * tokens with run admission. + */ + supportsExperimentalStartHook?: true; + /** * Whether calling `process.exit(1)` from a queue handler is observed by * the World as a delivery failure that will be retried. diff --git a/packages/world/src/queue.ts b/packages/world/src/queue.ts index f60ae034a7..2108745270 100644 --- a/packages/world/src/queue.ts +++ b/packages/world/src/queue.ts @@ -115,6 +115,13 @@ export const RunInputSchema = z.object({ executionContext: z.record(z.string(), z.any()).optional(), /** Initial plaintext run attributes, for resilient run creation. */ attributes: z.record(z.string(), z.string()).optional(), + /** Start-hook claim data, for resilient run creation. */ + experimentalStartHook: z + .object({ + token: z.string(), + ttlSeconds: z.number().int().positive(), + }) + .optional(), /** * Permits reserved `$`-prefixed keys in `attributes`, mirrored from the * `start()` option so resilient run creation validates the same way as diff --git a/workbench/example/workflows/99_e2e.ts b/workbench/example/workflows/99_e2e.ts index 304578eebb..d7c1287fc0 100644 --- a/workbench/example/workflows/99_e2e.ts +++ b/workbench/example/workflows/99_e2e.ts @@ -778,6 +778,26 @@ export async function hookClaimOnlyMutexWorkflow( }; } +export async function experimentalStartHookWorkflow(token: string) { + 'use workflow'; + + using hook = createHook<{ message: string }>({ token }); + + const conflict = await hook.getConflict(); + if (conflict) { + return { + role: 'duplicate' as const, + conflictRunId: conflict.runId, + }; + } + + const payload = await hook; + return { + role: 'owner' as const, + message: payload.message, + }; +} + /** * "Adopt the owner's result" strategy: the duplicate run waits for the * active owner to finish and returns the owner's result, so callers From f7cede77db30b39509e40e0ebbec5a06318f81b3 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Sat, 27 Jun 2026 12:45:06 -0700 Subject: [PATCH 2/4] Simplify Postgres start-hook run inserts --- packages/world-postgres/src/storage.ts | 59 ++++++++------------------ 1 file changed, 18 insertions(+), 41 deletions(-) diff --git a/packages/world-postgres/src/storage.ts b/packages/world-postgres/src/storage.ts index f7ffb080a0..6b878a1829 100644 --- a/packages/world-postgres/src/storage.ts +++ b/packages/world-postgres/src/storage.ts @@ -969,9 +969,22 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { allowReservedAttributes: eventData.allowReservedAttributes === true, } ); - const runValues = eventData.experimentalStartHook + const pendingRun = { + runId: effectiveRunId, + deploymentId: eventData.deploymentId, + workflowName: eventData.workflowName, + // Propagate specVersion from the event to the run entity + specVersion: effectiveSpecVersion, + input: eventData.input as SerializedContent, + executionContext: eventData.executionContext as + | SerializedContent + | undefined, + attributes: eventData.attributes, + status: 'pending' as const, + }; + const startHook = eventData.experimentalStartHook; + const runValues = startHook ? await (async () => { - const startHook = eventData.experimentalStartHook!; const [existingHook] = await getHookByToken.execute({ token: startHook.token, }); @@ -1003,19 +1016,7 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { const insertedRuns = await drizzle .insert(Schema.runs) - .values({ - runId: effectiveRunId, - deploymentId: eventData.deploymentId, - workflowName: eventData.workflowName, - // Propagate specVersion from the event to the run entity - specVersion: effectiveSpecVersion, - input: eventData.input as SerializedContent, - executionContext: eventData.executionContext as - | SerializedContent - | undefined, - attributes: eventData.attributes, - status: 'pending', - }) + .values(pendingRun) .onConflictDoNothing() .returning(); if (!insertedRuns[0]) { @@ -1056,19 +1057,7 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { const insertedRuns = await tx .insert(Schema.runs) - .values({ - runId: effectiveRunId, - deploymentId: eventData.deploymentId, - workflowName: eventData.workflowName, - // Propagate specVersion from the event to the run entity - specVersion: effectiveSpecVersion, - input: eventData.input as SerializedContent, - executionContext: eventData.executionContext as - | SerializedContent - | undefined, - attributes: eventData.attributes, - status: 'pending', - }) + .values(pendingRun) .onConflictDoNothing() .returning(); if (!insertedRuns[0]) { @@ -1094,19 +1083,7 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { })() : await drizzle .insert(Schema.runs) - .values({ - runId: effectiveRunId, - deploymentId: eventData.deploymentId, - workflowName: eventData.workflowName, - // Propagate specVersion from the event to the run entity - specVersion: effectiveSpecVersion, - input: eventData.input as SerializedContent, - executionContext: eventData.executionContext as - | SerializedContent - | undefined, - attributes: eventData.attributes, - status: 'pending', - }) + .values(pendingRun) .onConflictDoNothing() .returning(); const [runValue] = runValues; From 96664fe6002e6a1175ab7692ba25f63ba5159cd9 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Wed, 1 Jul 2026 16:20:23 -0700 Subject: [PATCH 3/4] Rework start-hook idempotency: align API shape, fix claim races, drop phase state - Rename start() option to experimentalStartHook with a single experimental_ttl key (the decided API shape) - Release unmaterialized claims on run_cancelled so cancel-then-retry (incl. start()'s own queue-failure cleanup) can reuse the token - Postgres: drop the derivable phase column; set-based claim retention; same-run admission retry no longer self-conflicts; single-transaction hook_created/hook_disposed; reclaim leaked ttl-0 claims of dead runs - Local: canonical run_created eventId convergence for duplicate deliveries; atomic stale-lock breaking; shared claim schema/reader; spurious self hook_conflict after lost materialize fixed - Core: collapse queue-first validation guards; simplify event-first branch to create-then-queue with cancel-on-failure --- .changeset/idempotent-start-hooks.md | 13 + .../v5/api-reference/workflow-api/start.mdx | 26 +- .../api-reference/workflow-errors/index.mdx | 3 + .../workflow-errors/workflow-start-error.mdx | 74 ++ .../docs/v5/foundations/idempotency.mdx | 71 +- packages/cli/src/lib/inspect/hydration.ts | 1 + packages/core/e2e/e2e.test.ts | 6 +- packages/core/src/capabilities.test.ts | 26 + packages/core/src/capabilities.ts | 19 +- packages/core/src/runtime.test.ts | 72 +- packages/core/src/runtime.ts | 16 + packages/core/src/runtime/start.test.ts | 374 ++++++++- packages/core/src/runtime/start.ts | 305 ++++--- packages/core/src/serialization.test.ts | 49 +- .../core/src/serialization/reducers/common.ts | 51 ++ packages/core/src/serialization/types.ts | 14 + packages/errors/src/index.ts | 58 +- packages/web-shared/src/lib/hydration.ts | 26 +- packages/web-shared/test/hydration.test.ts | 35 +- .../test/serializable-revivers.test.ts | 21 +- packages/workflow/src/internal/errors.ts | 1 + packages/world-local/src/index.ts | 25 +- packages/world-local/src/storage.test.ts | 112 ++- .../world-local/src/storage/events-storage.ts | 408 +++++----- packages/world-local/src/storage/helpers.ts | 67 +- .../world-local/src/storage/hooks-storage.ts | 53 +- .../migrations/0015_add_hook_claims.sql | 1 - packages/world-postgres/src/drizzle/schema.ts | 17 +- packages/world-postgres/src/index.ts | 2 +- packages/world-postgres/src/storage.ts | 754 ++++++++---------- packages/world-postgres/test/storage.test.ts | 96 ++- packages/world-vercel/src/events-v4.test.ts | 96 ++- packages/world-vercel/src/events-v4.ts | 23 +- packages/world-vercel/src/events.test.ts | 28 + packages/world-vercel/src/events.ts | 24 +- packages/world-vercel/src/index.ts | 8 + packages/world/src/interfaces.ts | 6 +- 37 files changed, 2035 insertions(+), 946 deletions(-) create mode 100644 .changeset/idempotent-start-hooks.md create mode 100644 docs/content/docs/v5/api-reference/workflow-errors/workflow-start-error.mdx diff --git a/.changeset/idempotent-start-hooks.md b/.changeset/idempotent-start-hooks.md new file mode 100644 index 0000000000..acafc4dea8 --- /dev/null +++ b/.changeset/idempotent-start-hooks.md @@ -0,0 +1,13 @@ +--- +"@workflow/core": minor +"workflow": minor +"@workflow/errors": minor +"@workflow/world": minor +"@workflow/world-local": minor +"@workflow/world-postgres": minor +"@workflow/world-vercel": minor +"@workflow/cli": patch +"@workflow/web-shared": patch +--- + +Add queue-first experimental start-hook idempotency for Vercel worlds. diff --git a/docs/content/docs/v5/api-reference/workflow-api/start.mdx b/docs/content/docs/v5/api-reference/workflow-api/start.mdx index 2291fc16a3..a3aded8da8 100644 --- a/docs/content/docs/v5/api-reference/workflow-api/start.mdx +++ b/docs/content/docs/v5/api-reference/workflow-api/start.mdx @@ -56,10 +56,11 @@ Learn more about [`WorkflowReadableStreamOptions`](/docs/api-reference/workflow- * In v5, `start()` can also be called directly from a workflow function to spawn a child run or continue work in a new run. See [Workflow Composition](/cookbook/common-patterns/workflow-composition) and [Versioning](/docs/foundations/versioning). * This is different from calling workflow functions directly, which is the typical pattern in Next.js applications. * The function returns immediately after enqueuing the workflow - it doesn't wait for the workflow to complete. -* Each call to `start()` creates a new workflow run. If retried requests must route to one active workflow, have the workflow create a deterministic hook token and use [`getHookByToken()`](/docs/api-reference/workflow-api/get-hook-by-token) to reuse an already-registered active hook. The lookup is not atomic with `start()`, so concurrent callers can still create extra runs before the hook is registered; handle that race inside the workflow by checking `await hook.getConflict()` before duplicate-sensitive work — on a conflict it resolves with the run that owns the token, so the duplicate can return the active owner to the caller. If duplicates must be rejected before a workflow body runs, keep a durable request record until native atomic start-and-hook registration exists. See [Idempotency](/docs/foundations/idempotency#run-idempotency). +* Each call to `start()` creates a new workflow run unless you opt into an idempotency mechanism. The stable pattern is to have the workflow create a deterministic hook token and use [`getHookByToken()`](/docs/api-reference/workflow-api/get-hook-by-token) to reuse an already-registered active hook. The lookup is not atomic with `start()`, so concurrent callers can still create extra runs before the hook is registered; handle that race inside the workflow by checking `await hook.getConflict()` before duplicate-sensitive work. If duplicates must be rejected before duplicate workflow code can run, use `experimentalStartHook` in a supporting World. See [Idempotency](/docs/foundations/idempotency#run-idempotency). * All arguments must be [serializable](/docs/foundations/serialization). * When `deploymentId` is provided, the argument types and return type become `unknown` since there is no guarantee the workflow function's types will be consistent across different deployments. * `attributes` seeds plaintext run metadata as part of creation and requires a World implementing spec version 4 or later. Keys that start with `$` are reserved for framework and library code; framework-level callers can pass `allowReservedAttributes: true` to seed reserved keys, with the same semantics as the [`experimental_setAttributes`](/docs/api-reference/workflow/experimental-set-attributes) option of the same name. +* `experimentalStartHook` reserves a deterministic hook token as part of start admission in Worlds that support it. On Vercel, this uses queue-first admission, requires the target deployment to support queued start-hook handling, disables turbo mode for the first workflow invocation, rejects TTLs longer than 30 days or tokens larger than 255 bytes, and may throw [`HookConflictError`](/docs/api-reference/workflow-errors/hook-conflict-error) or [`WorkflowStartError`](/docs/api-reference/workflow-errors/workflow-start-error). See [Run idempotency](/docs/foundations/idempotency#atomic-start-hook-idempotency). If `start()` throws `'start' received an invalid workflow function. Ensure the Workflow Development Kit is configured correctly and the function includes a 'use workflow' directive.`, the passed function was not transformed as a workflow. The two most common causes are a missing `"use workflow"` directive or missing framework integration. See [start-invalid-workflow-function](/docs/errors/start-invalid-workflow-function). @@ -88,6 +89,29 @@ const run = await start(myWorkflow, ["arg1", "arg2"], { // [!code highlight] }); // [!code highlight] ``` +### With `experimentalStartHook` + +Use `experimentalStartHook` when duplicate starts must be rejected before duplicate workflow code can run. The workflow should still create the same hook token and check `hook.getConflict()` so the queued invocation can materialize the reserved hook and so non-supporting Worlds have the same in-workflow guard. + +```typescript +import { start } from "workflow/api"; +import { processOrder } from "./workflows/process-order"; + +try { + const run = await start(processOrder, ["order_123"], { + experimentalStartHook: { + token: "order:order_123", + experimental_ttl: "30 days", + }, + }); + console.log("Started run", run.runId); +} catch (error) { + // HookConflictError means another active or retained run owns the token. + // WorkflowStartError means the run may already be queued; inspect error.runId. + throw error; +} +``` + ### Using `deploymentId: "latest"` Set `deploymentId` to `"latest"` to automatically resolve the most recent deployment for the current environment. This is useful when you want to ensure a workflow run targets the latest deployed version of your application rather than the deployment that initiated the call. For when to use this and how it fits with default run pinning, see [Versioning](/docs/foundations/versioning). diff --git a/docs/content/docs/v5/api-reference/workflow-errors/index.mdx b/docs/content/docs/v5/api-reference/workflow-errors/index.mdx index eee43c82cf..ea3b45585d 100644 --- a/docs/content/docs/v5/api-reference/workflow-errors/index.mdx +++ b/docs/content/docs/v5/api-reference/workflow-errors/index.mdx @@ -51,6 +51,9 @@ All errors extend [`WorkflowError`](/docs/api-reference/workflow-errors/workflow Thrown when the workflow runtime encounters an execution error, such as serialization failures or timeouts. + + Thrown when start() cannot synchronously confirm queueing or start-hook admission. + Thrown when a workflow run has expired and can no longer be operated on. diff --git a/docs/content/docs/v5/api-reference/workflow-errors/workflow-start-error.mdx b/docs/content/docs/v5/api-reference/workflow-errors/workflow-start-error.mdx new file mode 100644 index 0000000000..873d222a53 --- /dev/null +++ b/docs/content/docs/v5/api-reference/workflow-errors/workflow-start-error.mdx @@ -0,0 +1,74 @@ +--- +title: WorkflowStartError +description: Thrown when start() cannot synchronously confirm queueing or start-hook admission. +type: reference +summary: Catch WorkflowStartError when a start call may have queued a run but could not confirm admission. +related: + - /docs/api-reference/workflow-api/start + - /docs/foundations/idempotency +--- + +`WorkflowStartError` is thrown by [`start()`](/docs/api-reference/workflow-api/start) when the SDK cannot synchronously confirm a run started successfully. It carries the generated `runId` so callers can inspect the run, poll later, or retry the start call. + +This is most important for `experimentalStartHook` on Vercel, where the queue is contacted before start-hook admission. If the queue accepts the message but admission cannot be confirmed, the original run may still start later. Retrying the same request can then return a [`HookConflictError`](/docs/api-reference/workflow-errors/hook-conflict-error) if the original run wins the token claim. + +```typescript lineNumbers +import { start } from "workflow/api"; +import { WorkflowStartError } from "workflow/errors"; +import { processOrder } from "./workflows/process-order"; + +try { + await start(processOrder, ["order_123"], { + experimentalStartHook: { + token: "order:order_123", + experimental_ttl: "30 days", + }, + }); +} catch (error) { + if (WorkflowStartError.is(error)) { // [!code highlight] + console.log("Run may already be queued:", error.runId); + } +} +``` + +## API Signature + +### Properties + + + +### Static Methods + +#### `WorkflowStartError.is(value)` + +Type-safe check for `WorkflowStartError` instances. + +```typescript +import { WorkflowStartError } from "workflow/errors" +declare const error: unknown; // @setup + +if (WorkflowStartError.is(error)) { + // error is typed as WorkflowStartError +} +``` diff --git a/docs/content/docs/v5/foundations/idempotency.mdx b/docs/content/docs/v5/foundations/idempotency.mdx index 190b5c9f81..8ab4bf7001 100644 --- a/docs/content/docs/v5/foundations/idempotency.mdx +++ b/docs/content/docs/v5/foundations/idempotency.mdx @@ -146,11 +146,80 @@ export async function POST(request: Request) { ``` -This avoids creating a new run only after the first run has registered its hook. Because `start()` returns before the run body executes and calls `createHook()`, two concurrent requests can both observe "no hook yet" and each call `start()`. The race is resolved inside the workflow body, where the losing run observes `getConflict()` resolving with the active owner and returns without doing duplicate-sensitive work — and the route detects it by comparing the resumed hook's `runId` against the run it just started, without waiting for either run to finish. A native API for atomically starting a run and registering a hook is in the works. Until then, model recovery inside the workflow by checking `hook.getConflict()`. +This avoids creating a new run only after the first run has registered its hook. Because `start()` returns before the run body executes and calls `createHook()`, two concurrent requests can both observe "no hook yet" and each call `start()`. The race is resolved inside the workflow body, where the losing run observes `getConflict()` resolving with the active owner and returns without doing duplicate-sensitive work — and the route detects it by comparing the resumed hook's `runId` against the run it just started, without waiting for either run to finish. If duplicates must be rejected before duplicate workflow code can run, use experimental start-hook admission in a supporting World. This is active-run coordination. When the workflow completes and disposes the hook, the token can be used again. If a duplicate request after completion must return the original result instead of starting fresh work, persist that completed result under the same domain key. +### Atomic start-hook idempotency + +`start()` can reserve the same hook token at admission time with the experimental `experimentalStartHook` option. In supporting Worlds, the run and hook-token claim are admitted atomically. On Vercel, the queue message is accepted first, turbo mode is disabled for that start, and the workflow-server claim decides whether this run owns the token. + +```typescript lineNumbers +import { start } from "workflow/api"; +import { HookConflictError, WorkflowStartError } from "workflow/errors"; +import { processOrder } from "./workflows/process-order"; + +export async function POST(request: Request) { + const { orderId } = await request.json(); + const token = `order:${orderId}`; + + try { + const run = await start(processOrder, [orderId], { + experimentalStartHook: { + token, + experimental_ttl: "30 days", + }, + }); + + return Response.json({ runId: run.runId, reused: false }); + } catch (error) { + if (HookConflictError.is(error)) { + return Response.json({ + runId: error.conflictingRunId, + reused: true, + }); + } + + if (WorkflowStartError.is(error)) { + // The queue may already have accepted this run. Keep error.runId so + // callers can poll it or retry the start call. + return Response.json( + { runId: error.runId, queued: error.queued }, + { status: 202 } + ); + } + + throw error; + } +} +``` + +The workflow should still create the same hook token and check `hook.getConflict()`. That materializes the reserved hook for the owning run and keeps the workflow body correct in Worlds that do not support atomic admission. + +```typescript lineNumbers +import { createHook } from "workflow"; + +type OrderRequest = { confirmed: boolean }; + +export async function processOrder(orderId: string) { + "use workflow"; + + using request = createHook({ + token: `order:${orderId}`, + }); + + const conflict = await request.getConflict(); + if (conflict) { + return { status: "duplicate" as const, runId: conflict.runId }; + } + + // Continue with the owning run. +} +``` + +On Vercel, `experimental_ttl` is capped at 30 days and hook tokens are capped at 255 bytes; values above either cap fail with a world contract error before queueing. Cross-deployment starts also fail before queueing unless the target deployment reports support for queued start-hook handling. The token claim is retained until at least the later of hook creation plus TTL and run completion, so duplicates can be rejected while the run is active and for the configured retention window. Cancelling a run before it has created its hook releases the claim immediately, so an explicit cancel-then-retry can reuse the token without waiting out the TTL. + ### Conflict-handling strategies Some workflow systems resolve duplicate IDs with a fixed, pre-declared policy — typically a static choice between rejecting the new execution, deferring to the existing one, or terminating it. Workflow has no policy enum. `hook.getConflict()` hands the duplicate run the conflicting `Run` itself, and the policy is ordinary code — including policies that inspect state before deciding, which static configuration can't express. diff --git a/packages/cli/src/lib/inspect/hydration.ts b/packages/cli/src/lib/inspect/hydration.ts index 499edaaac7..8a0162c2d4 100644 --- a/packages/cli/src/lib/inspect/hydration.ts +++ b/packages/cli/src/lib/inspect/hydration.ts @@ -159,6 +159,7 @@ const ERROR_REVIVER_KEYS = [ 'SyntaxError', 'TypeError', 'URIError', + 'WorkflowStartError', ] as const; /** diff --git a/packages/core/e2e/e2e.test.ts b/packages/core/e2e/e2e.test.ts index cbbd57fc40..e86f111079 100644 --- a/packages/core/e2e/e2e.test.ts +++ b/packages/core/e2e/e2e.test.ts @@ -1791,15 +1791,15 @@ describe('e2e', () => { ); test.skipIf(!!process.env.WORKFLOW_VERCEL_ENV)( - 'experimentalWithHook rejects duplicate starts before hook materialization', + 'experimentalStartHook rejects duplicate starts before hook materialization', { timeout: 60_000 }, async () => { const token = Math.random().toString(36).slice(2); const workflow = await e2e('experimentalStartHookWorkflow'); const startOptions = { - experimentalWithHook: { + experimentalStartHook: { token, - experimentalTtl: 60_000, + experimental_ttl: 60_000, }, }; diff --git a/packages/core/src/capabilities.test.ts b/packages/core/src/capabilities.test.ts index ad62b61331..8159671a14 100644 --- a/packages/core/src/capabilities.test.ts +++ b/packages/core/src/capabilities.test.ts @@ -103,4 +103,30 @@ describe('getRunCapabilities', () => { expect(getRunCapabilities(version).framedByteStreams).toBe(true); }); }); + + describe('experimentalStartHookAdmission', () => { + it('is false when version is missing or invalid', () => { + expect(getRunCapabilities(undefined).experimentalStartHookAdmission).toBe( + false + ); + expect( + getRunCapabilities('not-a-version').experimentalStartHookAdmission + ).toBe(false); + }); + + it('is false before the start-hook admission cutoff', () => { + expect( + getRunCapabilities('5.0.0-beta.25').experimentalStartHookAdmission + ).toBe(false); + }); + + it('is true at and after the start-hook admission cutoff', () => { + expect( + getRunCapabilities('5.0.0-beta.26').experimentalStartHookAdmission + ).toBe(true); + expect(getRunCapabilities('5.0.0').experimentalStartHookAdmission).toBe( + true + ); + }); + }); }); diff --git a/packages/core/src/capabilities.ts b/packages/core/src/capabilities.ts index 4ef6b5a78e..1789c07d48 100644 --- a/packages/core/src/capabilities.ts +++ b/packages/core/src/capabilities.ts @@ -31,6 +31,7 @@ * - `gzip` (gzip payload compression): added in `5.0.0-beta.18` * - `zstd` (zstd payload compression, preferred codec): added in `5.0.0-beta.18` * alongside gzip — they co-ship, so any run that can read one can read both. + * - `experimentalStartHookAdmission` (queued start-hook loser handling): added in `5.0.0-beta.26` */ import semver from 'semver'; @@ -59,6 +60,13 @@ export interface RunCapabilities { * raw bytes (the legacy format) for compatibility with older runs. */ framedByteStreams: boolean; + + /** + * Whether the target workflow handler can safely process queue-first + * experimental start-hook runs: turbo is disabled and `HookConflictError` + * during `run_started` is acknowledged without running user workflow code. + */ + experimentalStartHookAdmission: boolean; } /** @@ -96,7 +104,14 @@ const CAPABILITY_VERSION_TABLE: ReadonlyArray<{ // to the next beta. A too-low cutoff makes new producers write framed bytes to // consumers that cannot unframe them (silent corruption); too-high merely // delays the optimization (safe). -}> = [{ capability: 'framedByteStreams', minVersion: '5.0.0-beta.15' }]; +}> = [ + { capability: 'framedByteStreams', minVersion: '5.0.0-beta.15' }, + // TODO(release): bump if this change ships after beta.26. + { + capability: 'experimentalStartHookAdmission', + minVersion: '5.0.0-beta.26', + }, +]; /** * The set of formats supported by all specVersion 2 runs, regardless of @@ -123,6 +138,7 @@ export function getRunCapabilities( return { supportedFormats: BASELINE_FORMATS, framedByteStreams: false, + experimentalStartHookAdmission: false, }; } @@ -137,6 +153,7 @@ export function getRunCapabilities( const result: RunCapabilities = { supportedFormats: formats, framedByteStreams: false, + experimentalStartHookAdmission: false, }; for (const { capability, minVersion } of CAPABILITY_VERSION_TABLE) { diff --git a/packages/core/src/runtime.test.ts b/packages/core/src/runtime.test.ts index ea4dcdbd2d..f2cc244c7b 100644 --- a/packages/core/src/runtime.test.ts +++ b/packages/core/src/runtime.test.ts @@ -1,4 +1,5 @@ import { + HookConflictError, RUN_ERROR_CODES, ThrottleError, WorkflowWorldError, @@ -1381,13 +1382,14 @@ describe('workflowEntrypoint turbo mode', () => { return r; }${xform('workflow')}`; - async function makeRunInput(runId: string) { + async function makeRunInput(runId: string, extra?: Record) { return { input: await dehydrateWorkflowArguments([], runId, undefined, []), deploymentId: 'test-deployment', workflowName: 'workflow', specVersion: SPEC_VERSION_CURRENT, executionContext: {}, + ...extra, }; } @@ -1401,7 +1403,9 @@ describe('workflowEntrypoint turbo mode', () => { runId: string; attempt: number; source: string; + runInput?: Awaited>; runStartedGate?: Promise; + runStartedError?: unknown; }) { const { runId, attempt, source } = opts; const order = turboOrder; @@ -1432,6 +1436,7 @@ describe('workflowEntrypoint turbo mode', () => { const eventsCreate = vi.fn(async (_runId: string, data: any) => { if (data.eventType === 'run_started') { if (opts.runStartedGate) await opts.runStartedGate; + if (opts.runStartedError) throw opts.runStartedError; order.push('run_started_resolved'); return { run: runEntity, events: [] as Event[] }; } @@ -1475,7 +1480,7 @@ describe('workflowEntrypoint turbo mode', () => { { runId, requestedAt: new Date('2024-01-01T00:00:00.000Z'), - runInput: await makeRunInput(runId), + runInput: opts.runInput ?? (await makeRunInput(runId)), }, { requestId: 'req_turbo', @@ -1576,6 +1581,69 @@ describe('workflowEntrypoint turbo mode', () => { ); }); + it('does not turbo when runInput carries an experimental start hook', async () => { + let release!: () => void; + const gate = new Promise((r) => { + release = r; + }); + const runId = 'wrun_start_hook_no_turbo'; + const { handlerPromise, order, eventsCreate } = await driveTurbo({ + runId, + attempt: 1, + source: oneStepWorkflow, + runInput: await makeRunInput(runId, { + experimentalStartHook: { token: 'order:123', ttlSeconds: 60 }, + }), + runStartedGate: gate, + }); + + await vi.waitFor( + () => + expect( + eventsCreate.mock.calls.some( + (c) => (c[1] as any).eventType === 'run_started' + ) + ).toBe(true), + { timeout: 15_000 } + ); + expect(order).not.toContain('body'); + expect(order).not.toContain('run_started_resolved'); + + release(); + const res = await handlerPromise; + expect(res.status).toBe(204); + expect(order.indexOf('run_started_resolved')).toBeLessThan( + order.indexOf('body') + ); + const runStarted = eventsCreate.mock.calls.find( + (c) => (c[1] as any).eventType === 'run_started' + ); + expect((runStarted?.[2] as any)?.skipPreload).toBeUndefined(); + }); + + it('acknowledges queued start-hook losers without running workflow code', async () => { + const runId = 'wrun_start_hook_loser'; + const { handlerPromise, order, eventsCreate } = await driveTurbo({ + runId, + attempt: 1, + source: oneStepWorkflow, + runInput: await makeRunInput(runId, { + experimentalStartHook: { token: 'order:123', ttlSeconds: 60 }, + }), + runStartedError: new HookConflictError('order:123', 'wrun_winner'), + }); + + const res = await handlerPromise; + expect(res.status).toBe(204); + expect(order).not.toContain('body'); + expect(order).not.toContain('step_started_called'); + expect( + eventsCreate.mock.calls.some( + (c) => (c[1] as any).eventType === 'run_failed' + ) + ).toBe(false); + }); + it('asks the World to skip the run_started preload only under turbo', async () => { // The backgrounded run_started is used purely as a write barrier and its // preloaded events are never read (preloadedEvents is forced to []), so diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 29490bd5e3..c32e145b78 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -3,6 +3,7 @@ import { CorruptedEventLogError, EntityConflictError, FatalError, + HookConflictError, ReplayDivergenceError, RUN_ERROR_CODES, type RunErrorCode, @@ -535,6 +536,7 @@ export function workflowEntrypoint( const turbo = isTurboEnabled() && runInput !== undefined && + runInput.experimentalStartHook === undefined && metadata.attempt === 1 && incomingStepId === undefined && !replayDivergence; @@ -878,6 +880,20 @@ export function workflowEntrypoint( ); } } catch (err) { + if ( + runInput?.experimentalStartHook && + HookConflictError.is(err) + ) { + runtimeLogger.info( + 'Start-hook claim lost during setup, skipping queued run', + { + workflowRunId: runId, + token: runInput.experimentalStartHook.token, + message: err.message, + } + ); + return; + } // Run was concurrently completed/failed/cancelled if ( EntityConflictError.is(err) || diff --git a/packages/core/src/runtime/start.test.ts b/packages/core/src/runtime/start.test.ts index df244cc419..45c8bf9f5e 100644 --- a/packages/core/src/runtime/start.test.ts +++ b/packages/core/src/runtime/start.test.ts @@ -1,7 +1,10 @@ +import { waitUntil } from '@vercel/functions'; import { EntityConflictError, HookConflictError, + RUN_ERROR_CODES, WorkflowRuntimeError, + WorkflowStartError, WorkflowWorldError, } from '@workflow/errors'; import { @@ -94,6 +97,11 @@ describe('start', () => { describe('specVersion', () => { let mockEventsCreate: ReturnType; let mockQueue: ReturnType; + const queueFirstStartHookAdmission = { + mode: 'queue-first' as const, + maxTtlSeconds: 30 * 24 * 60 * 60, + maxTokenBytes: 255, + }; beforeEach(() => { mockEventsCreate = vi.fn().mockImplementation((runId) => { @@ -116,6 +124,16 @@ describe('start', () => { vi.clearAllMocks(); }); + function setQueueFirstWorld(specVersion = SPEC_VERSION_CURRENT) { + setWorld({ + specVersion, + experimentalStartHookAdmission: queueFirstStartHookAdmission, + getDeploymentId: vi.fn().mockResolvedValue('deploy_123'), + events: { create: mockEventsCreate }, + queue: mockQueue, + } as any); + } + it('should use world.specVersion when available, falling back to SPEC_VERSION_SUPPORTS_EVENT_SOURCING', async () => { const validWorkflow = Object.assign(() => Promise.resolve('result'), { workflowId: 'test-workflow', @@ -140,7 +158,6 @@ describe('start', () => { // Mock world with specVersion 3 → uses it setWorld({ specVersion: SPEC_VERSION_CURRENT, - supportsExperimentalStartHook: true, getDeploymentId: vi.fn().mockResolvedValue('deploy_123'), events: { create: mockEventsCreate }, queue: mockQueue, @@ -242,14 +259,14 @@ describe('start', () => { }); setWorld({ specVersion: SPEC_VERSION_CURRENT, - supportsExperimentalStartHook: true, + experimentalStartHookAdmission: { mode: 'event-first' }, getDeploymentId: vi.fn().mockResolvedValue('deploy_123'), events: { create: mockEventsCreate }, queue: mockQueue, } as any); await start(validWorkflow, [], { - experimentalWithHook: { + experimentalStartHook: { token: 'order:123', experimental_ttl: '30 days', }, @@ -289,7 +306,7 @@ describe('start', () => { ); setWorld({ specVersion: SPEC_VERSION_CURRENT, - supportsExperimentalStartHook: true, + experimentalStartHookAdmission: { mode: 'event-first' }, getDeploymentId: vi.fn().mockResolvedValue('deploy_123'), events: { create: mockEventsCreate }, queue: mockQueue, @@ -297,9 +314,9 @@ describe('start', () => { await expect( start(validWorkflow, [], { - experimentalWithHook: { + experimentalStartHook: { token: 'order:123', - experimentalTtl: '30 days', + experimental_ttl: '30 days', }, }) ).rejects.toThrow(HookConflictError); @@ -313,9 +330,9 @@ describe('start', () => { await expect( start(validWorkflow, [], { - experimentalWithHook: { + experimentalStartHook: { token: 123, - experimentalTtl: '30 days', + experimental_ttl: '30 days', } as any, }) ).rejects.toThrow(/token must be a non-empty string/); @@ -331,7 +348,7 @@ describe('start', () => { mockQueue.mockRejectedValueOnce(queueError); setWorld({ specVersion: SPEC_VERSION_CURRENT, - supportsExperimentalStartHook: true, + experimentalStartHookAdmission: { mode: 'event-first' }, getDeploymentId: vi.fn().mockResolvedValue('deploy_123'), events: { create: mockEventsCreate }, queue: mockQueue, @@ -339,9 +356,9 @@ describe('start', () => { await expect( start(validWorkflow, [], { - experimentalWithHook: { + experimentalStartHook: { token: 'order:123', - experimentalTtl: '30 days', + experimental_ttl: '30 days', }, }) ).rejects.toThrow(queueError); @@ -354,19 +371,17 @@ describe('start', () => { }); }); - it('does not queue when experimental start hook admission is unconfirmed after retry', async () => { + it('does not queue when experimental start hook admission fails', async () => { const validWorkflow = Object.assign(() => Promise.resolve('result'), { workflowId: 'test-workflow', }); const admissionError = new WorkflowWorldError('response lost', { status: 500, }); - mockEventsCreate - .mockRejectedValueOnce(admissionError) - .mockRejectedValueOnce(admissionError); + mockEventsCreate.mockRejectedValueOnce(admissionError); setWorld({ specVersion: SPEC_VERSION_CURRENT, - supportsExperimentalStartHook: true, + experimentalStartHookAdmission: { mode: 'event-first' }, getDeploymentId: vi.fn().mockResolvedValue('deploy_123'), events: { create: mockEventsCreate }, queue: mockQueue, @@ -374,54 +389,325 @@ describe('start', () => { await expect( start(validWorkflow, [], { - experimentalWithHook: { + experimentalStartHook: { token: 'order:123', - experimentalTtl: '30 days', + experimental_ttl: '30 days', }, }) ).rejects.toThrow(admissionError); - expect(mockEventsCreate).toHaveBeenCalledTimes(2); + expect(mockEventsCreate).toHaveBeenCalledTimes(1); expect(mockQueue).not.toHaveBeenCalled(); }); - it('queues a confirmed experimental start hook run after a retryable admission response', async () => { + it('queues before admitting experimental start hooks in queue-first worlds', async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-01-01T00:00:00.000Z')); + + const validWorkflow = Object.assign(() => Promise.resolve('result'), { + workflowId: 'test-workflow', + }); + setQueueFirstWorld(); + + const run = await start(validWorkflow, [], { + experimentalStartHook: { + token: 'order:123', + experimental_ttl: '30 days', + }, + }); + + expect(run.runId).toBe(mockEventsCreate.mock.calls[0]?.[0]); + expect(mockQueue).toHaveBeenCalledTimes(1); + expect(mockEventsCreate).toHaveBeenCalledTimes(1); + expect( + mockQueue.mock.calls[0]?.[1].runInput.experimentalStartHook + ).toEqual({ + token: 'order:123', + ttlSeconds: 30 * 24 * 60 * 60, + }); + expect(mockEventsCreate.mock.calls[0]?.[1].eventData).toEqual( + expect.objectContaining({ + experimentalStartHook: { + token: 'order:123', + ttlSeconds: 30 * 24 * 60 * 60, + }, + }) + ); + expect(mockQueue.mock.calls[0]?.[2]).not.toHaveProperty('idempotencyKey'); + expect(mockQueue.mock.invocationCallOrder[0]).toBeLessThan( + mockEventsCreate.mock.invocationCallOrder[0] + ); + }); + + it('throws WorkflowStartError without admission when queue-first enqueue fails', async () => { + const validWorkflow = Object.assign(() => Promise.resolve('result'), { + workflowId: 'test-workflow', + }); + const queueError = new WorkflowWorldError('queue unavailable', { + status: 503, + }); + mockQueue.mockRejectedValueOnce(queueError); + setQueueFirstWorld(); + + const startPromise = start(validWorkflow, [], { + experimentalStartHook: { + token: 'order:123', + experimental_ttl: '30 days', + }, + }); + + await expect(startPromise).rejects.toBeInstanceOf(WorkflowStartError); + await expect(startPromise).rejects.toMatchObject({ + name: 'WorkflowStartError', + stage: 'queue', + queued: 'unknown', + retryable: true, + status: 503, + cause: queueError, + }); + expect(mockEventsCreate).not.toHaveBeenCalled(); + }); + + it('throws WorkflowStartError when queue-first admission cannot be confirmed', async () => { const validWorkflow = Object.assign(() => Promise.resolve('result'), { workflowId: 'test-workflow', }); const admissionError = new WorkflowWorldError('response lost', { status: 500, }); - const mockEventsList = vi.fn().mockResolvedValue({ - data: [{ eventType: 'run_created' }], + mockEventsCreate.mockRejectedValueOnce(admissionError); + setQueueFirstWorld(); + + await expect( + start(validWorkflow, [], { + experimentalStartHook: { + token: 'order:123', + experimental_ttl: '30 days', + }, + }) + ).rejects.toMatchObject({ + name: 'WorkflowStartError', + stage: 'admission', + queued: true, + retryable: true, + status: 500, + cause: admissionError, + }); + expect(mockQueue).toHaveBeenCalledTimes(1); + }); + + it('wraps transient queue-first admission world errors after queueing', async () => { + const validWorkflow = Object.assign(() => Promise.resolve('result'), { + workflowId: 'test-workflow', + }); + const admissionError = new WorkflowWorldError('request timed out', { + status: 408, + }); + mockEventsCreate.mockRejectedValueOnce(admissionError); + setQueueFirstWorld(); + + await expect( + start(validWorkflow, [], { + experimentalStartHook: { + token: 'order:123', + experimental_ttl: '30 days', + }, + }) + ).rejects.toMatchObject({ + name: 'WorkflowStartError', + stage: 'admission', + queued: true, + retryable: true, + status: 408, + cause: admissionError, + }); + expect(mockQueue).toHaveBeenCalledTimes(1); + }); + + it('wraps non-retryable queue-first admission world errors after queueing', async () => { + const validWorkflow = Object.assign(() => Promise.resolve('result'), { + workflowId: 'test-workflow', + }); + const admissionError = new WorkflowWorldError('forbidden', { + status: 403, + }); + mockEventsCreate.mockRejectedValueOnce(admissionError); + setQueueFirstWorld(); + + await expect( + start(validWorkflow, [], { + experimentalStartHook: { + token: 'order:123', + experimental_ttl: '30 days', + }, + }) + ).rejects.toMatchObject({ + name: 'WorkflowStartError', + stage: 'admission', + queued: true, + retryable: false, + status: 403, + cause: admissionError, + }); + expect(mockQueue).toHaveBeenCalledTimes(1); + }); + + it('wraps unknown queue-first admission failures after queueing', async () => { + const validWorkflow = Object.assign(() => Promise.resolve('result'), { + workflowId: 'test-workflow', + }); + const admissionError = new TypeError('socket closed'); + mockEventsCreate.mockRejectedValueOnce(admissionError); + setQueueFirstWorld(); + const waitUntilCalls = vi.mocked(waitUntil).mock.calls.length; + + await expect( + start(validWorkflow, [], { + experimentalStartHook: { + token: 'order:123', + experimental_ttl: '30 days', + }, + }) + ).rejects.toMatchObject({ + name: 'WorkflowStartError', + stage: 'admission', + queued: true, + retryable: true, + cause: admissionError, + }); + await vi.waitFor(() => + expect(vi.mocked(waitUntil).mock.calls.length).toBeGreaterThan( + waitUntilCalls + ) + ); + }); + + it('still surfaces HookConflictError after queue-first enqueue', async () => { + const validWorkflow = Object.assign(() => Promise.resolve('result'), { + workflowId: 'test-workflow', + }); + mockEventsCreate.mockRejectedValueOnce( + new HookConflictError('order:123', 'wrun_conflicting') + ); + setQueueFirstWorld(); + + await expect( + start(validWorkflow, [], { + experimentalStartHook: { + token: 'order:123', + experimental_ttl: '30 days', + }, + }) + ).rejects.toThrow(HookConflictError); + expect(mockQueue).toHaveBeenCalledTimes(1); + expect(mockQueue.mock.invocationCallOrder[0]).toBeLessThan( + mockEventsCreate.mock.invocationCallOrder[0] + ); + }); + + it('rejects experimental start hook TTLs above the world cap before queueing', async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-01-01T00:00:00.000Z')); + + const validWorkflow = Object.assign(() => Promise.resolve('result'), { + workflowId: 'test-workflow', + }); + setQueueFirstWorld(); + + await expect( + start(validWorkflow, [], { + experimentalStartHook: { + token: 'order:123', + experimental_ttl: '31 days', + }, + }) + ).rejects.toMatchObject({ + name: 'WorkflowWorldError', + code: RUN_ERROR_CODES.WORLD_CONTRACT_ERROR, + }); + expect(mockEventsCreate).not.toHaveBeenCalled(); + expect(mockQueue).not.toHaveBeenCalled(); + }); + + it('rejects queue-first start hooks with oversized tokens before queueing', async () => { + const validWorkflow = Object.assign(() => Promise.resolve('result'), { + workflowId: 'test-workflow', + }); + setQueueFirstWorld(); + + await expect( + start(validWorkflow, [], { + experimentalStartHook: { + token: 'x'.repeat(256), + experimental_ttl: '30 days', + }, + }) + ).rejects.toMatchObject({ + name: 'WorkflowWorldError', + code: RUN_ERROR_CODES.WORLD_CONTRACT_ERROR, + }); + expect(mockEventsCreate).not.toHaveBeenCalled(); + expect(mockQueue).not.toHaveBeenCalled(); + }); + + it('rejects queue-first start hooks without runInput queue transport before queueing', async () => { + const validWorkflow = Object.assign(() => Promise.resolve('result'), { + workflowId: 'test-workflow', + }); + setQueueFirstWorld(SPEC_VERSION_SUPPORTS_EVENT_SOURCING); + + await expect( + start(validWorkflow, [], { + experimentalStartHook: { + token: 'order:123', + experimental_ttl: '30 days', + }, + }) + ).rejects.toMatchObject({ + name: 'WorkflowWorldError', + code: RUN_ERROR_CODES.WORLD_CONTRACT_ERROR, + }); + expect(mockEventsCreate).not.toHaveBeenCalled(); + expect(mockQueue).not.toHaveBeenCalled(); + }); + + it('rejects queue-first start hooks when the target deployment lacks runtime support', async () => { + const validWorkflow = Object.assign(() => Promise.resolve('result'), { + workflowId: 'test-workflow', }); - mockEventsCreate - .mockRejectedValueOnce(admissionError) - .mockRejectedValueOnce(new EntityConflictError('Run already exists')); setWorld({ specVersion: SPEC_VERSION_CURRENT, - supportsExperimentalStartHook: true, - getDeploymentId: vi.fn().mockResolvedValue('deploy_123'), - events: { create: mockEventsCreate, list: mockEventsList }, + experimentalStartHookAdmission: queueFirstStartHookAdmission, + getDeploymentId: vi.fn().mockResolvedValue('deploy_current'), + events: { create: mockEventsCreate }, queue: mockQueue, - } as any); - - const run = await start(validWorkflow, [], { - experimentalWithHook: { - token: 'order:123', - experimentalTtl: '30 days', + streams: { + get: vi.fn().mockResolvedValue( + new Response( + JSON.stringify({ + healthy: true, + workflowCoreVersion: '5.0.0-beta.25', + }) + ).body + ), }, - }); + } as any); - expect(run.runId).toBe(mockEventsCreate.mock.calls[0]?.[0]); - expect(mockEventsCreate).toHaveBeenCalledTimes(2); - expect(mockEventsList).toHaveBeenCalledWith({ - runId: run.runId, - pagination: { limit: 1, sortOrder: 'asc' }, - resolveData: 'none', + await expect( + start(validWorkflow, [], { + deploymentId: 'deploy_old', + experimentalStartHook: { + token: 'order:123', + experimental_ttl: '30 days', + }, + }) + ).rejects.toMatchObject({ + name: 'WorkflowWorldError', + code: RUN_ERROR_CODES.WORLD_CONTRACT_ERROR, }); + expect(mockEventsCreate).not.toHaveBeenCalled(); expect(mockQueue).toHaveBeenCalledTimes(1); - expect(mockQueue.mock.calls[0]?.[1].runId).toBe(run.runId); + expect(mockQueue.mock.calls[0]?.[0]).toContain('health_check'); }); it('rejects experimental start hooks when the world has not opted in', async () => { @@ -437,9 +723,9 @@ describe('start', () => { await expect( start(validWorkflow, [], { - experimentalWithHook: { + experimentalStartHook: { token: 'order:123', - experimentalTtl: '30 days', + experimental_ttl: '30 days', }, }) ).rejects.toThrow(/supports experimental start-hook admission/); diff --git a/packages/core/src/runtime/start.ts b/packages/core/src/runtime/start.ts index 151edebf79..8b0e89847a 100644 --- a/packages/core/src/runtime/start.ts +++ b/packages/core/src/runtime/start.ts @@ -1,7 +1,10 @@ import { EntityConflictError, + HookConflictError, + RUN_ERROR_CODES, ThrottleError, WorkflowRuntimeError, + WorkflowStartError, WorkflowWorldError, } from '@workflow/errors'; import { parseDurationToDate } from '@workflow/utils'; @@ -43,6 +46,7 @@ import { safeWaitUntil, waitedUntil } from './wait-until.js'; * deployments don't recognize the health check at all. */ const CROSS_DEPLOYMENT_CAPABILITY_PROBE_TIMEOUT_MS = 2_000; +const textEncoder = new TextEncoder(); /** ULID generator for client-side runId generation */ const ulid = monotonicFactory(); @@ -63,12 +67,10 @@ export function _resetLatestNoOpWarnForTests(): void { hasWarnedLatestNoOp = false; } -type ExperimentalStartHookOptions = { +interface ExperimentalStartHookOptions { token: string; -} & ( - | { experimentalTtl: StringValue | number; experimental_ttl?: never } - | { experimental_ttl: StringValue | number; experimentalTtl?: never } -); + experimental_ttl: StringValue | number; +} function normalizeExperimentalStartHook(hook: ExperimentalStartHookOptions): { token: string; @@ -76,30 +78,21 @@ function normalizeExperimentalStartHook(hook: ExperimentalStartHookOptions): { } { if (typeof hook.token !== 'string' || hook.token.length === 0) { throw new WorkflowRuntimeError( - 'experimentalWithHook.token must be a non-empty string.' - ); - } - - if ( - hook.experimentalTtl !== undefined && - hook.experimental_ttl !== undefined - ) { - throw new WorkflowRuntimeError( - 'experimentalWithHook accepts either experimentalTtl or experimental_ttl, not both.' + 'experimentalStartHook.token must be a non-empty string.' ); } - const ttl = hook.experimentalTtl ?? hook.experimental_ttl; - if (ttl === undefined) { + if (hook.experimental_ttl === undefined) { throw new WorkflowRuntimeError( - 'experimentalWithHook.experimentalTtl must be provided.' + 'experimentalStartHook.experimental_ttl must be provided.' ); } - const ttlMilliseconds = parseDurationToDate(ttl).getTime() - Date.now(); + const ttlMilliseconds = + parseDurationToDate(hook.experimental_ttl).getTime() - Date.now(); if (!Number.isFinite(ttlMilliseconds) || ttlMilliseconds <= 0) { throw new WorkflowRuntimeError( - 'experimentalWithHook.experimentalTtl must be a positive duration.' + 'experimentalStartHook.experimental_ttl must be a positive duration.' ); } @@ -121,14 +114,15 @@ async function hasDurableRunCreatedEvent( function getRunCreatedResult( runId: string, - result: Awaited> + result: Awaited>, + verifyRunId = true ) { if (!result.run) { throw new WorkflowRuntimeError( "Missing 'run' in server response for 'run_created' event" ); } - if (result.run.runId !== runId) { + if (verifyRunId && result.run.runId !== runId) { throw new WorkflowRuntimeError( `Server returned different runId than requested: expected ${runId}, got ${result.run.runId}` ); @@ -136,6 +130,29 @@ function getRunCreatedResult( return result.run; } +function isWorkflowWorldError(error: unknown): error is WorkflowWorldError { + return error instanceof WorkflowWorldError || WorkflowWorldError.is(error); +} + +function getWorkflowWorldErrorDetails(error: unknown) { + if (!isWorkflowWorldError(error)) return {}; + return { + status: error.status, + url: error.url, + code: error.code, + retryAfter: error.retryAfter, + }; +} + +function isRetryableQueueError(error: unknown): boolean { + if (!isWorkflowWorldError(error)) return true; + if (error.status === undefined) return true; + if (error.status === 408 || error.status === 425 || error.status === 429) { + return true; + } + return error.status >= 500; +} + export interface StartOptionsBase { /** * The world to use for the workflow run creation, @@ -173,9 +190,11 @@ export interface StartOptionsBase { * EXPERIMENTAL: atomically reserve a hook token as part of run admission. * * If another active or retained run already owns the token, `start()` throws - * `HookConflictError` and does not queue a new workflow run. + * `HookConflictError`. On Vercel the queue may already have accepted the + * losing run before the conflict is observed; that queued invocation exits + * without running user workflow code. */ - experimentalWithHook?: ExperimentalStartHookOptions; + experimentalStartHook?: ExperimentalStartHookOptions; } export interface StartOptionsWithDeploymentId extends StartOptionsBase { @@ -336,12 +355,15 @@ export async function start( // mocks) can't service health checks, so we skip the probe for them. let framedByteStreams: boolean; let targetSupportsCompression: boolean; + let targetSupportsStartHookAdmission: boolean; if (deploymentId === currentDeploymentId) { framedByteStreams = true; targetSupportsCompression = true; + targetSupportsStartHookAdmission = true; } else if (typeof world.streams?.get !== 'function') { framedByteStreams = false; targetSupportsCompression = false; + targetSupportsStartHookAdmission = false; } else { const probe = await healthCheck(world, 'workflow', { deploymentId, @@ -352,9 +374,29 @@ export async function start( targetSupportsCompression = capabilities.supportedFormats.has( SerializationFormat.GZIP ); + targetSupportsStartHookAdmission = + capabilities.experimentalStartHookAdmission; } const ops: Promise[] = []; + let opsFlushScheduled = false; + const scheduleOpsFlush = () => { + if (opsFlushScheduled) return; + opsFlushScheduled = true; + // These argument-stream ops are flushed in the background; the promise + // handed to waitUntil must never reject (an unconsumed waitUntil + // rejection crashes the process as unhandledRejection), so unexpected + // failures are logged instead. + safeWaitUntil(Promise.all(ops), (err) => { + runtimeLogger.warn( + 'Background flush of workflow argument streams failed', + { + workflowRunId: runId, + error: err instanceof Error ? err.message : String(err), + } + ); + }); + }; // Generate runId client-side so we have it before serialization // (required for future E2E encryption where runId is part of the encryption context) @@ -372,21 +414,52 @@ export async function start( world.specVersion ?? SPEC_VERSION_SUPPORTS_EVENT_SOURCING; const v1Compat = isLegacySpecVersion(specVersion); - const experimentalStartHook = opts.experimentalWithHook - ? normalizeExperimentalStartHook(opts.experimentalWithHook) + const experimentalStartHook = opts.experimentalStartHook + ? normalizeExperimentalStartHook(opts.experimentalStartHook) : undefined; - if (experimentalStartHook && v1Compat) { - throw new WorkflowRuntimeError( - 'experimentalWithHook requires an event-sourced World.' - ); - } - if ( - experimentalStartHook && - world.supportsExperimentalStartHook !== true - ) { - throw new WorkflowRuntimeError( - 'experimentalWithHook requires a World that supports experimental start-hook admission.' - ); + const startHookAdmission = world.experimentalStartHookAdmission; + if (experimentalStartHook) { + if (v1Compat) { + throw new WorkflowRuntimeError( + 'experimentalStartHook requires an event-sourced World.' + ); + } + if (startHookAdmission === undefined) { + throw new WorkflowRuntimeError( + 'experimentalStartHook requires a World that supports experimental start-hook admission.' + ); + } + if (startHookAdmission.mode === 'queue-first') { + if (!targetSupportsStartHookAdmission) { + throw new WorkflowWorldError( + 'experimentalStartHook requires a target deployment that supports queue-first start-hook admission.', + { code: RUN_ERROR_CODES.WORLD_CONTRACT_ERROR } + ); + } + if ( + experimentalStartHook.ttlSeconds > startHookAdmission.maxTtlSeconds + ) { + throw new WorkflowWorldError( + `experimentalStartHook.experimental_ttl exceeds this World's maximum of ${startHookAdmission.maxTtlSeconds} seconds.`, + { code: RUN_ERROR_CODES.WORLD_CONTRACT_ERROR } + ); + } + if ( + textEncoder.encode(experimentalStartHook.token).byteLength > + startHookAdmission.maxTokenBytes + ) { + throw new WorkflowWorldError( + `experimentalStartHook.token exceeds this World's maximum of ${startHookAdmission.maxTokenBytes} bytes.`, + { code: RUN_ERROR_CODES.WORLD_CONTRACT_ERROR } + ); + } + if (specVersion < SPEC_VERSION_SUPPORTS_CBOR_QUEUE_TRANSPORT) { + throw new WorkflowWorldError( + 'experimentalStartHook requires a spec version that supports runInput queue transport in queue-first Worlds.', + { code: RUN_ERROR_CODES.WORLD_CONTRACT_ERROR } + ); + } + } } const allowReservedAttributes = opts.allowReservedAttributes === true; let attributes: Record | undefined; @@ -503,64 +576,106 @@ export async function start( | NonNullable>['run']> | undefined; if (experimentalStartHook) { - try { - const result = await world.events.create(runId, runCreatedEvent, { - v1Compat, - }); - createdRunForSpan = getRunCreatedResult(runId, result); - } catch (error) { - if (!isRetryableStartError(error)) throw error; + if (startHookAdmission?.mode === 'queue-first') { + try { + await world.queue( + getWorkflowQueueName(workflowName), + queuePayload, + { + deploymentId, + specVersion, + } + ); + } catch (error) { + scheduleOpsFlush(); + throw new WorkflowStartError( + `Workflow start failed while enqueueing run "${runId}". The queue may have accepted the message; inspect this run ID or retry the start call.`, + { + runId, + stage: 'queue', + queued: 'unknown', + retryable: isRetryableQueueError(error), + cause: error, + ...getWorkflowWorldErrorDetails(error), + } + ); + } + try { const result = await world.events.create(runId, runCreatedEvent, { v1Compat, }); createdRunForSpan = getRunCreatedResult(runId, result); - } catch (retryError) { - if ( - !EntityConflictError.is(retryError) && - !isRetryableStartError(retryError) - ) { - throw retryError; - } - try { + } catch (error) { + scheduleOpsFlush(); + if (EntityConflictError.is(error)) { if (!(await hasDurableRunCreatedEvent(world, runId))) { throw error; } - } catch { + resilientStart = true; + runtimeLogger.warn( + 'Run creation conflicted after start-hook queue acceptance, but the run_created event is durable.', + { workflowRunId: runId, error: error.message } + ); + } else if (!HookConflictError.is(error)) { + throw new WorkflowStartError( + `Workflow run "${runId}" was queued, but start-hook admission could not be confirmed. Inspect this run ID or retry the start call; a retry may conflict if the queued run starts successfully.`, + { + runId, + stage: 'admission', + queued: true, + retryable: isRetryableQueueError(error), + cause: error, + ...getWorkflowWorldErrorDetails(error), + } + ); + } else { throw error; } } - runtimeLogger.warn( - 'Run creation response failed after start-hook admission, but the run_created event is durable. Queueing the confirmed run.', - { workflowRunId: runId, error: (error as Error).message } - ); - } - - try { - await world.queue(getWorkflowQueueName(workflowName), queuePayload, { - deploymentId, - specVersion, + } else { + // Event-first: admission (run creation + token claim) must succeed + // before the run is queued, so a losing start never enqueues work. + // Any creation failure — including HookConflictError — fails the + // start; nothing was queued, so the caller can simply retry. + const result = await world.events.create(runId, runCreatedEvent, { + v1Compat, }); - } catch (error) { + createdRunForSpan = getRunCreatedResult(runId, result); + try { - await world.events.create( - runId, + await world.queue( + getWorkflowQueueName(workflowName), + queuePayload, { - eventType: 'run_cancelled', + deploymentId, specVersion, - }, - { v1Compat } - ); - } catch (cleanupError) { - runtimeLogger.warn( - 'Run queueing failed after start-hook admission, and cleanup also failed.', - { - workflowRunId: runId, - error: (cleanupError as Error).message, } ); + } catch (error) { + // Undo admission so the token is not fenced by a run that will + // never execute. Cancellation releases unmaterialized start-hook + // claims in event-first Worlds. + try { + await world.events.create( + runId, + { + eventType: 'run_cancelled', + specVersion, + }, + { v1Compat } + ); + } catch (cleanupError) { + runtimeLogger.warn( + 'Run queueing failed after start-hook admission, and cleanup also failed.', + { + workflowRunId: runId, + error: (cleanupError as Error).message, + } + ); + } + throw error; } - throw error; } } else { // Call events.create (run_created) and queue in parallel. @@ -601,37 +716,15 @@ export async function start( throw err; } } else { - const result = runCreatedResult.value; - // Assert that the run was created - if (!result.run) { - throw new WorkflowRuntimeError( - "Missing 'run' in server response for 'run_created' event" - ); - } - - // Verify server accepted our runId - if (!v1Compat && result.run.runId !== runId) { - throw new WorkflowRuntimeError( - `Server returned different runId than requested: expected ${runId}, got ${result.run.runId}` - ); - } - createdRunForSpan = result.run; + createdRunForSpan = getRunCreatedResult( + runId, + runCreatedResult.value, + !v1Compat + ); } } - // These argument-stream ops are flushed in the background; the promise - // handed to waitUntil must never reject (an unconsumed waitUntil - // rejection crashes the process as unhandledRejection), so unexpected - // failures are logged instead. - safeWaitUntil(Promise.all(ops), (err) => { - runtimeLogger.warn( - 'Background flush of workflow argument streams failed', - { - workflowRunId: runId, - error: err instanceof Error ? err.message : String(err), - } - ); - }); + scheduleOpsFlush(); span?.setAttributes({ ...Attribute.WorkflowRunId(runId), diff --git a/packages/core/src/serialization.test.ts b/packages/core/src/serialization.test.ts index 5c21c55c3c..54f78520ae 100644 --- a/packages/core/src/serialization.test.ts +++ b/packages/core/src/serialization.test.ts @@ -5,6 +5,7 @@ import { HookConflictError, RetryableError, RuntimeDecryptionError, + WorkflowStartError, } from '@workflow/errors'; import { WORKFLOW_DESERIALIZE, WORKFLOW_SERIALIZE } from '@workflow/serde'; import { beforeAll, describe, expect, it, vi } from 'vitest'; @@ -2667,7 +2668,7 @@ describe('step function serialization', () => { const stepId = 'step//workflows/test.ts//addNumbers'; // Create a VM context like the workflow runner does - const { context, globalThis: vmGlobalThis } = createContext({ + const { globalThis: vmGlobalThis } = createContext({ seed: 'test', fixedTimestamp: 1714857600000, }); @@ -2729,7 +2730,7 @@ describe('step function serialization', () => { const stepId = 'step//workflows/test.ts//missingUseStep'; // Create a VM context WITHOUT setting up WORKFLOW_USE_STEP - const { context, globalThis: vmGlobalThis } = createContext({ + const { globalThis: vmGlobalThis } = createContext({ seed: 'test', fixedTimestamp: 1714857600000, }); @@ -3984,6 +3985,48 @@ describe('Workflow error serialization', () => { expect(str).not.toContain('Instance'); }); + it('should round-trip WorkflowStartError preserving recovery fields', async () => { + const error = new WorkflowStartError('queued but not admitted', { + runId: 'wrun_queued', + stage: 'admission', + queued: true, + retryable: true, + status: 408, + retryAfter: 2, + }); + const hydrated = (await roundTrip(error)) as WorkflowStartError; + + expect(hydrated).toBeInstanceOf(WorkflowStartError); + expect(WorkflowStartError.is(hydrated)).toBe(true); + expect(hydrated.runId).toBe('wrun_queued'); + expect(hydrated.stage).toBe('admission'); + expect(hydrated.queued).toBe(true); + expect(hydrated.retryable).toBe(true); + expect(hydrated.status).toBe(408); + expect(hydrated.retryAfter).toBe(2); + }); + + it('should serialize WorkflowStartError using its dedicated reducer key', async () => { + const error = new WorkflowStartError('queue response lost', { + runId: 'wrun_unknown', + stage: 'queue', + queued: 'unknown', + retryable: true, + }); + const serialized = await dehydrateStepReturnValue( + error, + mockRunId, + noEncryptionKey + ); + const str = new TextDecoder().decode( + (serialized as Uint8Array).subarray(4) + ); + + expect(str).toContain('["WorkflowStartError",'); + expect(str).not.toContain('["Error",'); + expect(str).not.toContain('Instance'); + }); + it('should preserve cause on FatalError when present', async () => { const cause = new Error('underlying issue'); const error = new FatalError('fatal with cause'); @@ -5561,7 +5604,7 @@ describe('isEncrypted', () => { // ============================================================================ describe('AbortController serialization', () => { - const { context, globalThis: vmGlobalThis } = createContext({ + const { globalThis: vmGlobalThis } = createContext({ seed: 'test-abort-serde', fixedTimestamp: 1714857600000, }); diff --git a/packages/core/src/serialization/reducers/common.ts b/packages/core/src/serialization/reducers/common.ts index 6ad8b773ff..59d16e8dde 100644 --- a/packages/core/src/serialization/reducers/common.ts +++ b/packages/core/src/serialization/reducers/common.ts @@ -15,6 +15,7 @@ import { HookConflictError, RetryableError, RuntimeDecryptionError, + WorkflowStartError, } from '@workflow/errors'; import type { Reducers, Revivers, SerializableSpecial } from '../types.js'; @@ -228,6 +229,27 @@ export function getCommonReducers( } return reduced; }, + WorkflowStartError: (value) => { + const base = reduceNamedErrorSubclassBase('WorkflowStartError', value); + if (!base) return false; + const error = value as WorkflowStartError; + const details = { + ...base, + runId: error.runId, + retryable: error.retryable, + }; + const reduced: SerializableSpecial['WorkflowStartError'] = + error.stage === 'queue' + ? { ...details, stage: 'queue', queued: 'unknown' } + : { ...details, stage: 'admission', queued: true }; + if (error.status !== undefined) reduced.status = error.status; + if (error.url !== undefined) reduced.url = error.url; + if (error.code !== undefined) reduced.code = error.code; + if (error.retryAfter !== undefined) { + reduced.retryAfter = error.retryAfter; + } + return reduced; + }, RangeError: makeErrorSubclassReducer('RangeError'), ReferenceError: makeErrorSubclassReducer('ReferenceError'), // RetryableError carries an extra `retryAfter` Date that we serialize as @@ -409,6 +431,35 @@ export function getCommonRevivers( } return error; }, + WorkflowStartError: (value) => { + const Ctor = + ((global as Record)[ + Symbol.for('@workflow/errors//WorkflowStartError') + ] as typeof WorkflowStartError | undefined) ?? WorkflowStartError; + const details = { + runId: value.runId, + retryable: value.retryable, + status: value.status, + url: value.url, + code: value.code, + retryAfter: value.retryAfter, + ...('cause' in value ? { cause: value.cause } : {}), + }; + const error = + value.stage === 'queue' + ? new Ctor(value.message, { + ...details, + stage: 'queue', + queued: 'unknown', + }) + : new Ctor(value.message, { + ...details, + stage: 'admission', + queued: true, + }); + if (value.stack !== undefined) error.stack = value.stack; + return error; + }, RangeError: makeErrorSubclassReviver(global, 'RangeError'), ReferenceError: makeErrorSubclassReviver(global, 'ReferenceError'), RetryableError: (value) => { diff --git a/packages/core/src/serialization/types.ts b/packages/core/src/serialization/types.ts index 3668c05148..d7898cb464 100644 --- a/packages/core/src/serialization/types.ts +++ b/packages/core/src/serialization/types.ts @@ -83,6 +83,20 @@ export interface SerializableSpecial { // TODO: Make this required when HookConflictError.conflictingRunId is required. conflictingRunId?: string; }; + WorkflowStartError: { + message: string; + stack?: string; + cause?: unknown; + runId: string; + retryable: boolean; + status?: number; + url?: string; + code?: string; + retryAfter?: number; + } & ( + | { stage: 'queue'; queued: 'unknown' } + | { stage: 'admission'; queued: true } + ); Int8Array: string; // base64 string Int16Array: string; // base64 string Int32Array: string; // base64 string diff --git a/packages/errors/src/index.ts b/packages/errors/src/index.ts index 1be4c371b4..9098102766 100644 --- a/packages/errors/src/index.ts +++ b/packages/errors/src/index.ts @@ -54,9 +54,9 @@ function appendFramedDetails( const head = isLast ? '╰▶ ' : '├▶ '; const cont = isLast ? ' ' : '│ '; const text = `${detail.label}: ${detail.value}`; - text - .split('\n') - .forEach((line, i) => lines.push(`${i === 0 ? head : cont}${line}`)); + text.split('\n').forEach((line, i) => { + lines.push(`${i === 0 ? head : cont}${line}`); + }); }); return lines.join('\n'); } @@ -187,6 +187,44 @@ export class WorkflowWorldError extends WorkflowError { } } +type WorkflowStartErrorOptions = { + runId: string; + retryable: boolean; + status?: number; + url?: string; + code?: string; + retryAfter?: number; + cause?: unknown; +} & ( + | { stage: 'queue'; queued: 'unknown' } + | { stage: 'admission'; queued: true } +); + +/** + * Thrown when `start()` cannot synchronously confirm whether a workflow run was + * fully admitted. The error carries the client-generated run ID so callers can + * inspect or retry the run when the queue may already have accepted it. + */ +export class WorkflowStartError extends WorkflowWorldError { + readonly runId: string; + readonly stage: WorkflowStartErrorOptions['stage']; + readonly queued: WorkflowStartErrorOptions['queued']; + readonly retryable: boolean; + + constructor(message: string, options: WorkflowStartErrorOptions) { + super(message, options); + this.name = 'WorkflowStartError'; + this.runId = options.runId; + this.stage = options.stage; + this.queued = options.queued; + this.retryable = options.retryable; + } + + static is(value: unknown): value is WorkflowStartError { + return isError(value) && value.name === 'WorkflowStartError'; + } +} + /** * Thrown when a workflow run fails during execution. * @@ -914,7 +952,8 @@ export { RUN_ERROR_CODES, type RunErrorCode } from './error-codes.js'; // Cross-realm class registration // --------------------------------------------------------------------------- // -// `FatalError`, `RetryableError`, and `HookConflictError` are not built-ins, so different realms +// `FatalError`, `RetryableError`, `HookConflictError`, and +// `WorkflowStartError` are not built-ins, so different realms // (e.g. the workflow VM context vs. the host context that runs the queue // handler) bundle and load their own copies of this module — meaning each // realm has its own distinct class identity. Cross-realm `instanceof` fails @@ -933,6 +972,9 @@ const RETRYABLE_ERROR_KEY = Symbol.for('@workflow/errors//RetryableError'); const HOOK_CONFLICT_ERROR_KEY = Symbol.for( '@workflow/errors//HookConflictError' ); +const WORKFLOW_START_ERROR_KEY = Symbol.for( + '@workflow/errors//WorkflowStartError' +); const RUNTIME_DECRYPTION_ERROR_KEY = Symbol.for( '@workflow/errors//RuntimeDecryptionError' ); @@ -962,6 +1004,14 @@ if (typeof globalThis !== 'undefined') { configurable: false, }); } + if (!Object.hasOwn(globalThis, WORKFLOW_START_ERROR_KEY)) { + Object.defineProperty(globalThis, WORKFLOW_START_ERROR_KEY, { + value: WorkflowStartError, + writable: false, + enumerable: false, + configurable: false, + }); + } if (!Object.hasOwn(globalThis, RUNTIME_DECRYPTION_ERROR_KEY)) { Object.defineProperty(globalThis, RUNTIME_DECRYPTION_ERROR_KEY, { value: RuntimeDecryptionError, diff --git a/packages/web-shared/src/lib/hydration.ts b/packages/web-shared/src/lib/hydration.ts index b7472021b9..452963db9c 100644 --- a/packages/web-shared/src/lib/hydration.ts +++ b/packages/web-shared/src/lib/hydration.ts @@ -136,7 +136,7 @@ export function getWebRevivers(): Revivers { // `packages/core/src/serialization/reducers/common.ts`) emits a tagged // entry for each built-in Error subclass plus the workflow-specific // `FatalError` / `RetryableError` / `HookConflictError` / - // `RuntimeDecryptionError` and `AggregateError`. Without + // `WorkflowStartError` / `RuntimeDecryptionError` and `AggregateError`. Without // matching revivers here, `devalue.unflatten` throws "Unknown type X" // — which surfaces in the web o11y UI as "Failed to load resource // details: Unknown type FatalError". @@ -198,6 +198,30 @@ export function getWebRevivers(): Revivers { if (value.stack !== undefined) error.stack = value.stack; return error; }, + WorkflowStartError: (value) => { + const opts = 'cause' in value ? { cause: value.cause } : undefined; + const error = new Error(value.message, opts) as Error & { + runId?: string; + stage?: string; + queued?: true | 'unknown'; + retryable?: boolean; + status?: number; + url?: string; + code?: string; + retryAfter?: number; + }; + error.name = 'WorkflowStartError'; + error.runId = value.runId; + error.stage = value.stage; + error.queued = value.queued; + error.retryable = value.retryable; + if (value.status !== undefined) error.status = value.status; + if (value.url !== undefined) error.url = value.url; + if (value.code !== undefined) error.code = value.code; + if (value.retryAfter !== undefined) error.retryAfter = value.retryAfter; + if (value.stack !== undefined) error.stack = value.stack; + return error; + }, RetryableError: (value) => { const opts = 'cause' in value ? { cause: value.cause } : undefined; const error = new Error(value.message, opts) as Error & { diff --git a/packages/web-shared/test/hydration.test.ts b/packages/web-shared/test/hydration.test.ts index 576c170622..1c26f3c08e 100644 --- a/packages/web-shared/test/hydration.test.ts +++ b/packages/web-shared/test/hydration.test.ts @@ -1,6 +1,10 @@ import { dehydrateStepError } from '@workflow/core/serialization'; import { hydrateData } from '@workflow/core/serialization-format'; -import { FatalError, RetryableError } from '@workflow/errors'; +import { + FatalError, + RetryableError, + WorkflowStartError, +} from '@workflow/errors'; import { describe, expect, it } from 'vitest'; import { getWebRevivers } from '../src/lib/hydration.js'; @@ -111,6 +115,35 @@ describe('getWebRevivers — error family', () => { expect(revived.conflictingRunId).toBe('wrun_conflicting'); }); + it('hydrates a WorkflowStartError with recovery details preserved', async () => { + const revived = await roundTrip< + Error & { + runId?: string; + stage?: string; + queued?: true | 'unknown'; + retryable?: boolean; + status?: number; + } + >( + new WorkflowStartError('admission uncertain', { + runId: 'wrun_queued', + stage: 'admission', + queued: true, + retryable: false, + status: 403, + }) + ); + + expect(revived).toBeInstanceOf(Error); + expect(revived.name).toBe('WorkflowStartError'); + expect(revived.message).toBe('admission uncertain'); + expect(revived.runId).toBe('wrun_queued'); + expect(revived.stage).toBe('admission'); + expect(revived.queued).toBe(true); + expect(revived.retryable).toBe(false); + expect(revived.status).toBe(403); + }); + it('hydrates a RetryableError with retryAfter as a Date', async () => { const retryAt = new Date('2025-01-01T00:00:00.000Z'); const revived = await roundTrip( diff --git a/packages/web-shared/test/serializable-revivers.test.ts b/packages/web-shared/test/serializable-revivers.test.ts index c5163bea7a..07c9c66318 100644 --- a/packages/web-shared/test/serializable-revivers.test.ts +++ b/packages/web-shared/test/serializable-revivers.test.ts @@ -1,10 +1,10 @@ import { readFileSync } from 'node:fs'; import { fileURLToPath } from 'node:url'; import { hydrateData } from '@workflow/core/serialization-format'; -import { getCLIRevivers } from '../../cli/src/lib/inspect/hydration.js'; -import { getWebRevivers } from '../src/lib/hydration.js'; import ts from 'typescript'; import { describe, expect, it } from 'vitest'; +import { getCLIRevivers } from '../../cli/src/lib/inspect/hydration.js'; +import { getWebRevivers } from '../src/lib/hydration.js'; const SERIALIZABLE_TYPES_PATH = fileURLToPath( new URL('../../core/src/serialization/types.ts', import.meta.url) @@ -100,6 +100,23 @@ const SERIALIZABLE_PAYLOADS: Record = { 'approval-token', 'wrun_conflicting', ], + WorkflowStartError: [ + ['WorkflowStartError', 1], + { + message: 2, + runId: 3, + stage: 4, + queued: 5, + retryable: 6, + status: 7, + }, + 'Workflow run "wrun_queued" was queued, but start-hook admission could not be confirmed.', + 'wrun_queued', + 'admission', + true, + false, + 403, + ], Instance: [ ['Instance', 1], { classId: 2, data: 3 }, diff --git a/packages/workflow/src/internal/errors.ts b/packages/workflow/src/internal/errors.ts index 97c941947e..d2c1578185 100644 --- a/packages/workflow/src/internal/errors.ts +++ b/packages/workflow/src/internal/errors.ts @@ -14,5 +14,6 @@ export { WorkflowRunNotCompletedError, WorkflowRunNotFoundError, WorkflowRuntimeError, + WorkflowStartError, WorkflowWorldError, } from '@workflow/errors'; diff --git a/packages/world-local/src/index.ts b/packages/world-local/src/index.ts index dacda103b2..75695a0407 100644 --- a/packages/world-local/src/index.ts +++ b/packages/world-local/src/index.ts @@ -17,7 +17,11 @@ import { import { initDataDir } from './init.js'; import { instrumentObject } from './instrumentObject.js'; import { createQueue, type DirectHandler } from './queue.js'; -import { hashToken, hookRecoveryMarkerPath } from './storage/helpers.js'; +import { + hashToken, + hookRecoveryMarkerPath, + readHookTokenClaim, +} from './storage/helpers.js'; import { createStorage } from './storage.js'; import { createStreamer } from './streamer.js'; @@ -66,7 +70,7 @@ export function createLocalWorld(args?: Partial): LocalWorld { const recoverActiveRuns = mergedConfig.recoverActiveRuns ?? true; return { specVersion: SPEC_VERSION_CURRENT, - supportsExperimentalStartHook: true, + experimentalStartHookAdmission: { mode: 'event-first' }, ...queue, ...storage, ...instrumentObject('world.streams', { @@ -145,22 +149,11 @@ export function createLocalWorld(args?: Partial): LocalWorld { .filter((tokenFile) => tokenFile.endsWith('.json')) .map(async (tokenFile) => { const claimPath = path.join(tokensDir, tokenFile); - const claim = JSON.parse( - await fs.readFile(claimPath, 'utf8') - ) as { - tag?: unknown; - token?: unknown; - runId?: unknown; - hookId?: unknown; - }; - if (claim.tag !== tag) return; + const claim = await readHookTokenClaim(claimPath); + if (claim?.tag !== tag) return; await deleteJSON(claimPath); - if ( - typeof claim.token === 'string' && - typeof claim.runId === 'string' && - typeof claim.hookId === 'string' - ) { + if (claim.token && claim.hookId) { await deleteJSON( hookRecoveryMarkerPath( basedir, diff --git a/packages/world-local/src/storage.test.ts b/packages/world-local/src/storage.test.ts index 895be20981..6a0f3323dd 100644 --- a/packages/world-local/src/storage.test.ts +++ b/packages/world-local/src/storage.test.ts @@ -2134,7 +2134,7 @@ describe('Storage', () => { expect(result.hook?.hookId).toBe('hook_stale_materialize'); }); - it('should clean up start hook claims when run creation loses', async () => { + it('should not adopt or duplicate run_created when losing to a pre-existing hookless run', async () => { const token = 'failed-run-create-start-claim-token'; const run = await createRun(storage, { deploymentId: 'deployment-123', @@ -2142,6 +2142,10 @@ describe('Storage', () => { input: new Uint8Array(), }); + // A hook-carrying run_created for a runId that already exists + // (created without a start hook) is a divergent duplicate: it must + // conflict, and it must NOT append a second run_created event. Its + // claim stays behind and expires via TTL once the run is terminal. await expect( storage.events.create(run.runId, { eventType: 'run_created', @@ -2155,21 +2159,63 @@ describe('Storage', () => { }) ).rejects.toThrow(EntityConflictError); + const runEvents = await storage.events.list({ + runId: run.runId, + pagination: {}, + }); + expect( + runEvents.data.filter((e) => e.eventType === 'run_created') + ).toHaveLength(1); + }); + + it('should converge duplicate same-run hook-carrying run_created deliveries on one event', async () => { + const token = 'duplicate-run-create-start-claim-token'; + const request = { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }; + const started = await storage.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: request, + }); + const runId = started.run?.runId; + if (!runId) throw new Error('Expected run to be created'); + + // A duplicate delivery of the same run_created converges on the + // canonical event path and dedups instead of appending a second + // run_created or deleting the live claim. + await expect( + storage.events.create(runId, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: request, + }) + ).rejects.toThrow(EntityConflictError); + + const runEvents = await storage.events.list({ + runId, + pagination: {}, + }); + expect( + runEvents.data.filter((e) => e.eventType === 'run_created') + ).toHaveLength(1); + + // The claim survived the duplicate: another run still conflicts. await expect( storage.events.create(null, { eventType: 'run_created', specVersion: SPEC_VERSION_CURRENT, eventData: { - deploymentId: 'deployment-789', - workflowName: 'test-workflow-3', + deploymentId: 'deployment-456', + workflowName: 'test-workflow-2', input: new Uint8Array(), experimentalStartHook: { token, ttlSeconds: 60 }, }, }) - ).resolves.toMatchObject({ - event: { eventType: 'run_created' }, - run: { workflowName: 'test-workflow-3' }, - }); + ).rejects.toThrow(HookConflictError); }); it('should retain a disposed start hook claim until ttl expiry', async () => { @@ -2248,8 +2294,8 @@ describe('Storage', () => { expect(result.hook).toBeUndefined(); }); - it('should retain an unmaterialized start hook claim until ttl expiry', async () => { - const token = 'cancelled-unmaterialized-start-claim-token'; + it('should retain an unmaterialized start hook claim after run failure until ttl expiry', async () => { + const token = 'failed-unmaterialized-start-claim-token'; const started = await storage.events.create(null, { eventType: 'run_created', specVersion: SPEC_VERSION_CURRENT, @@ -2264,7 +2310,11 @@ describe('Storage', () => { if (!run) throw new Error('Expected run to be created'); await storage.events.create(run.runId, { - eventType: 'run_cancelled', + eventType: 'run_started', + }); + await storage.events.create(run.runId, { + eventType: 'run_failed', + eventData: { error: new Uint8Array() }, }); await expect( @@ -2289,7 +2339,6 @@ describe('Storage', () => { ttlSeconds: 60, createdAt: past, expiresAt: past, - phase: 'retained', }, { overwrite: true } ); @@ -2311,6 +2360,45 @@ describe('Storage', () => { }); }); + it('should release an unmaterialized start hook claim on cancellation', async () => { + const token = 'cancelled-unmaterialized-start-claim-token'; + const started = await storage.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }, + }); + const run = started.run; + if (!run) throw new Error('Expected run to be created'); + + // Cancel the claim-owning run before it materialized a hook: the + // token must be immediately reusable (cancel-then-retry, including + // start()'s own cleanup when queueing fails after admission). + await storage.events.create(run.runId, { + eventType: 'run_cancelled', + }); + + await expect( + storage.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'deployment-456', + workflowName: 'test-workflow-2', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }, + }) + ).resolves.toMatchObject({ + event: { eventType: 'run_created' }, + run: { workflowName: 'test-workflow-2' }, + }); + }); + it('should keep an expired disposed start hook claim while the run is active', async () => { const token = 'active-expired-start-claim-token'; const started = await storage.events.create(null, { @@ -2395,7 +2483,6 @@ describe('Storage', () => { runId: 'orphaned-start-run', ttlSeconds: 1, createdAt: past, - phase: 'start_claim', }, { overwrite: true } ); @@ -2456,7 +2543,6 @@ describe('Storage', () => { ttlSeconds: 1, createdAt: past, expiresAt: past, - phase: 'retained', }, { overwrite: true } ); diff --git a/packages/world-local/src/storage/events-storage.ts b/packages/world-local/src/storage/events-storage.ts index 500e379276..f9674a6a7c 100644 --- a/packages/world-local/src/storage/events-storage.ts +++ b/packages/world-local/src/storage/events-storage.ts @@ -52,11 +52,14 @@ import { import { stripEventDataRefs } from './filters.js'; import { getObjectCreatedAt, + type HookTokenClaim, hashToken, hookRecoveryMarkerPath, + hookTokenClaimPath, monotonicUlid, + readHookTokenClaim, } from './helpers.js'; -import { deleteAllHooksForRun } from './hooks-storage.js'; +import { deleteAllHooksForRun, retainStartHookClaim } from './hooks-storage.js'; import { handleLegacyEvent } from './legacy.js'; import { withRunFileLock } from './runs-storage.js'; @@ -83,34 +86,7 @@ import { withRunFileLock } from './runs-storage.js'; // but a shared filesystem), exactly matching the cross-process // semantics without spawning subprocesses. -const HookTokenClaimSchema = z.object({ - token: z.string().optional(), - // The token-claim writer below has always persisted `hookId`, but - // this read schema previously omitted it, which is the bug fixed - // by https://github.com/vercel/workflow/issues/2283. `optional()` - // is defensive: any claim file that somehow lacks the field still - // parses (yielding `undefined`) and falls through to the cross- - // hook conflict branch, matching pre-fix behavior. - hookId: z.string().optional(), - runId: z.string(), - // `eventId` is the canonical hook_created event ID the claiming - // worker committed to publishing. Persisting it here turns the - // claim file into a durable convergence key for cross-worker / - // cross-process retries (see comment on the hook_created branch). - // `optional()` for backward compatibility: a legacy claim file - // written before this field existed falls through to the recovery- - // marker upgrade path, which atomically pins a canonical eventId - // via a sidecar marker (also a `writeExclusive`). - eventId: z.string().optional(), - ttlSeconds: z.number().int().positive().optional(), - startEventId: z.string().optional(), - createdAt: z.coerce.date().optional(), - expiresAt: z.coerce.date().optional(), - phase: z.enum(['start_claim', 'materialized', 'retained']).optional(), - tag: z.string().optional(), -}); - -const START_HOOK_MATERIALIZE_LOCK_STALE_MS = 30_000; +const START_HOOK_LOCK_STALE_MS = 30_000; /** * Sidecar recovery marker that pins a canonical `hook_created` @@ -133,49 +109,61 @@ const HookRecoveryMarkerSchema = z.object({ eventId: z.string(), }); -async function readHookTokenClaim( - constraintPath: string -): Promise | null> { - try { - return await readJSON(constraintPath, HookTokenClaimSchema); - } catch (error) { - if (error instanceof SyntaxError || error instanceof z.ZodError) { - return null; - } - throw error; - } -} - -function hookTokenClaimPath(basedir: string, token: string): string { - return path.join(basedir, 'hooks', 'tokens', `${hashToken(token)}.json`); -} - -function hookTokenReclaimLockPath( +/** + * Runs `fn` under an exclusive cross-process file lock scoped to a hook + * token. Returns `undefined` (without running `fn`) when the lock is + * contended. Locks abandoned by a crashed holder (older than + * START_HOOK_LOCK_STALE_MS) are broken atomically via rename, so two + * breakers can never both enter, and the holder only removes a lock it + * still owns, so a stalled holder cannot free a breaker's lock. + */ +async function withTokenClaimLock( basedir: string, token: string, - tag: string | undefined -): string { - const lockName = tag - ? `${hashToken(token)}.reclaim.${tag}` - : `${hashToken(token)}.reclaim`; - return resolveWithinBase(basedir, '.locks', 'hooks', lockName); -} + purpose: 'reclaim' | 'materialize', + fn: () => Promise +): Promise { + const lockPath = resolveWithinBase( + basedir, + '.locks', + 'hooks', + `${hashToken(token)}.${purpose}` + ); + const owner = monotonicUlid(); + let locked = await writeExclusive(lockPath, owner); + if (!locked) { + const stat = await fs.stat(lockPath).catch(() => undefined); + if (stat && Date.now() - stat.mtimeMs > START_HOOK_LOCK_STALE_MS) { + const broken = await fs.rename(lockPath, `${lockPath}.${owner}`).then( + () => true, + () => false + ); + if (broken) { + await fs.unlink(`${lockPath}.${owner}`).catch(() => {}); + locked = await writeExclusive(lockPath, owner); + } + } + } + if (!locked) return undefined; -function hookTokenMaterializeLockPath( - basedir: string, - token: string, - tag: string | undefined -): string { - const lockName = tag - ? `${hashToken(token)}.materialize.${tag}` - : `${hashToken(token)}.materialize`; - return resolveWithinBase(basedir, '.locks', 'hooks', lockName); + try { + return await fn(); + } finally { + const current = await fs.readFile(lockPath, 'utf8').catch(() => undefined); + if (current === owner) { + await fs.unlink(lockPath).catch(() => {}); + } + } } +/** + * An expired claim may be replaced only when its owning run is gone or + * terminal — an active run keeps its token fenced even past the TTL. + */ async function canReuseExpiredStartClaim( basedir: string, tag: string | undefined, - claim: z.infer + claim: HookTokenClaim ): Promise { const expiresAt = claim.expiresAt ?? @@ -199,136 +187,107 @@ async function replaceExpiredStartClaim( basedir: string, tag: string | undefined, token: string, - nextClaim: z.infer + nextClaim: HookTokenClaim ): Promise { - const lockPath = hookTokenReclaimLockPath(basedir, token, undefined); - let locked = await writeExclusive(lockPath, ''); - if (!locked) { - const stat = await fs.stat(lockPath).catch(() => undefined); - if ( - stat && - Date.now() - stat.mtimeMs > START_HOOK_MATERIALIZE_LOCK_STALE_MS - ) { - await fs.unlink(lockPath).catch(() => {}); - locked = await writeExclusive(lockPath, ''); - } - } - if (!locked) return false; - - try { - const constraintPath = hookTokenClaimPath(basedir, token); - const latestClaim = await readHookTokenClaim(constraintPath); - if ( - !latestClaim || - !(await canReuseExpiredStartClaim(basedir, tag, latestClaim)) - ) { - return false; + const replaced = await withTokenClaimLock( + basedir, + token, + 'reclaim', + async () => { + const constraintPath = hookTokenClaimPath(basedir, token); + const latestClaim = await readHookTokenClaim(constraintPath); + if ( + !latestClaim || + !(await canReuseExpiredStartClaim(basedir, tag, latestClaim)) + ) { + return false; + } + await writeJSON(constraintPath, nextClaim, { overwrite: true }); + return true; } - await writeJSON(constraintPath, nextClaim, { overwrite: true }); - return true; - } finally { - await fs.unlink(lockPath).catch(() => {}); - } + ); + return replaced ?? false; } +/** + * Binds an unmaterialized same-run start claim to a hook. Fails (returns + * false) when the claim is gone, owned by another run, already bound to a + * hook, or retained. + */ async function materializeStartHookClaim( basedir: string, - tag: string | undefined, token: string, - nextClaim: z.infer + nextClaim: HookTokenClaim ): Promise { - const lockPath = hookTokenMaterializeLockPath(basedir, token, tag); - let locked = await writeExclusive(lockPath, ''); - if (!locked) { - const stat = await fs.stat(lockPath).catch(() => undefined); - if ( - stat && - Date.now() - stat.mtimeMs > START_HOOK_MATERIALIZE_LOCK_STALE_MS - ) { - await fs.unlink(lockPath).catch(() => {}); - locked = await writeExclusive(lockPath, ''); - } - } - if (!locked) return false; - - try { - const constraintPath = hookTokenClaimPath(basedir, token); - const latestClaim = await readHookTokenClaim(constraintPath); - if ( - !latestClaim || - latestClaim.runId !== nextClaim.runId || - latestClaim.hookId !== undefined || - latestClaim.phase === 'retained' - ) { - return false; + const materialized = await withTokenClaimLock( + basedir, + token, + 'materialize', + async () => { + const constraintPath = hookTokenClaimPath(basedir, token); + const latestClaim = await readHookTokenClaim(constraintPath); + if ( + !latestClaim || + latestClaim.runId !== nextClaim.runId || + latestClaim.hookId !== undefined || + latestClaim.expiresAt !== undefined + ) { + return false; + } + await writeJSON(constraintPath, nextClaim, { overwrite: true }); + return true; } - await writeJSON(constraintPath, nextClaim, { overwrite: true }); - return true; - } finally { - await fs.unlink(lockPath).catch(() => {}); - } + ); + return materialized ?? false; } +/** + * Claims `token` for `runId`'s run_created event. Returns the canonical + * run_created eventId for the run: the caller's `eventId` when this call + * created (or reclaimed) the token, or the eventId recorded by a previous + * same-run attempt — every duplicate delivery converges on one event path, + * so the exclusive event publish downstream dedups instead of appending a + * second run_created. Throws HookConflictError when another run holds a + * live claim. + */ async function claimStartHookToken( basedir: string, runId: string, eventId: string, startHook: { token: string; ttlSeconds: number }, tag?: string -): Promise { +): Promise { const constraintPath = hookTokenClaimPath(basedir, startHook.token); - const createdAt = new Date(); + const nextClaim: HookTokenClaim = { + token: startHook.token, + runId, + ttlSeconds: startHook.ttlSeconds, + startEventId: eventId, + createdAt: new Date(), + tag, + }; const claimed = await writeExclusive( constraintPath, - JSON.stringify({ - token: startHook.token, - runId, - ttlSeconds: startHook.ttlSeconds, - startEventId: eventId, - createdAt, - phase: 'start_claim', - tag, - }) + JSON.stringify(nextClaim) ); - if (claimed) return true; + if (claimed) return eventId; const existingClaim = await readHookTokenClaim(constraintPath); - if (existingClaim?.runId === runId) return false; + if (existingClaim?.runId === runId) { + return existingClaim.startEventId ?? eventId; + } if ( existingClaim && - (await replaceExpiredStartClaim(basedir, tag, startHook.token, { - token: startHook.token, - runId, - ttlSeconds: startHook.ttlSeconds, - startEventId: eventId, - createdAt, - phase: 'start_claim', - tag, - })) + (await replaceExpiredStartClaim(basedir, tag, startHook.token, nextClaim)) ) { - return true; + return eventId; } const currentClaim = await readHookTokenClaim(constraintPath); - throw new HookConflictError(startHook.token, currentClaim?.runId); -} - -async function deleteOwnedStartHookClaim( - basedir: string, - token: string, - runId: string, - eventId: string -): Promise { - const constraintPath = hookTokenClaimPath(basedir, token); - const latestClaim = await readHookTokenClaim(constraintPath); - if ( - latestClaim?.runId === runId && - latestClaim.startEventId === eventId && - latestClaim.phase === 'start_claim' && - !latestClaim.hookId - ) { - await deleteJSON(constraintPath); + if (currentClaim?.runId === runId) { + return currentClaim.startEventId ?? eventId; } + throw new HookConflictError(startHook.token, currentClaim?.runId); } async function readHookRecoveryMarker( @@ -782,10 +741,13 @@ export function createEventsStorage( createdAt: now, updatedAt: now, }; - const runCreatedEventId = `evnt_${monotonicUlid()}`; - let ownsStartHookClaim = false; + // For start-hook runs, claimStartHookToken returns the + // canonical run_created eventId — duplicate deliveries of the + // same runId converge on one event path so the exclusive + // event write below dedups. + let runCreatedEventId = `evnt_${monotonicUlid()}`; if (runInputData.experimentalStartHook) { - ownsStartHookClaim = await claimStartHookToken( + runCreatedEventId = await claimStartHookToken( basedir, effectiveRunId, runCreatedEventId, @@ -819,22 +781,17 @@ export function createEventsStorage( }, }; const createdCompositeKey = `${effectiveRunId}-${runCreatedEventId}`; - await writeJSON( + await writeExclusive( taggedPath(basedir, 'events', createdCompositeKey, tag), - runCreatedEvent + JSON.stringify(runCreatedEvent, jsonReplacer, 2) ); currentRun = createdRun; } else { - if (ownsStartHookClaim && runInputData.experimentalStartHook) { - await deleteOwnedStartHookClaim( - basedir, - runInputData.experimentalStartHook.token, - effectiveRunId, - runCreatedEventId - ); - } // Run already exists (concurrent run_created won the - // race). Re-read it so downstream logic sees the real state. + // race). Re-read it so downstream logic sees the real + // state. Any start-hook claim stays: same-run attempts + // share the canonical event path, and a claim orphaned by + // a truly divergent writer expires via its TTL. currentRun = await readJSONWithFallback( basedir, 'runs', @@ -1131,15 +1088,23 @@ export function createEventsStorage( allowReservedAttributes: runData.allowReservedAttributes === true, } ); + // For start-hook runs, claimStartHookToken returns the canonical + // run_created eventId — duplicate deliveries converge on one + // event path so the exclusive event publish downstream dedups. + // `ownsStartHookClaim` records whether this call created the + // claim (canonical id === ours) or adopted a previous same-run + // attempt's. let ownsStartHookClaim = false; if (runData.experimentalStartHook) { - ownsStartHookClaim = await claimStartHookToken( + const canonicalEventId = await claimStartHookToken( basedir, effectiveRunId, eventId, runData.experimentalStartHook, tag ); + ownsStartHookClaim = canonicalEventId === eventId; + eventId = canonicalEventId; } run = { runId: effectiveRunId, @@ -1168,21 +1133,23 @@ export function createEventsStorage( JSON.stringify(run, jsonReplacer, 2) ); if (!created) { - const startHookClaim = runData.experimentalStartHook + // Lost the run write to a concurrent same-run creation. For + // start-hook runs whose claim we share, converge on the winner: + // reuse the existing run and the canonical eventId, and let the + // exclusive event publish downstream dedup. The claim is left + // alone — same-run attempts share it, and a claim orphaned by a + // truly divergent writer expires via its TTL. + const startHook = runData.experimentalStartHook; + const startHookClaim = startHook ? await readHookTokenClaim( - hookTokenClaimPath( - basedir, - runData.experimentalStartHook.token - ) + hookTokenClaimPath(basedir, startHook.token) ) : null; - const startEventId = startHookClaim?.startEventId; const existingRun = - runData.experimentalStartHook && + startHook && !ownsStartHookClaim && startHookClaim?.runId === effectiveRunId && - startHookClaim.phase === 'start_claim' && - startEventId + startHookClaim.startEventId ? await readJSONWithFallback( basedir, 'runs', @@ -1192,13 +1159,8 @@ export function createEventsStorage( ) : null; - if ( - existingRun && - runData.experimentalStartHook && - startHookClaim && - startEventId - ) { - eventId = startEventId; + if (existingRun && startHook && startHookClaim?.startEventId) { + eventId = startHookClaim.startEventId; event = { eventType: 'run_created', runId: effectiveRunId, @@ -1215,23 +1177,14 @@ export function createEventsStorage( ? { allowReservedAttributes: true as const } : {}), experimentalStartHook: { - token: runData.experimentalStartHook.token, + token: startHook.token, ttlSeconds: - startHookClaim.ttlSeconds ?? - runData.experimentalStartHook.ttlSeconds, + startHookClaim.ttlSeconds ?? startHook.ttlSeconds, }, }, }; run = existingRun; } else { - if (ownsStartHookClaim && runData.experimentalStartHook) { - await deleteOwnedStartHookClaim( - basedir, - runData.experimentalStartHook.token, - effectiveRunId, - eventId - ); - } throw new EntityConflictError( `Workflow run "${effectiveRunId}" already exists` ); @@ -1367,7 +1320,9 @@ export function createEventsStorage( } ); await Promise.all([ - deleteAllHooksForRun(basedir, effectiveRunId), + deleteAllHooksForRun(basedir, effectiveRunId, { + releaseUnmaterializedClaims: true, + }), deleteAllWaitsForRun(basedir, effectiveRunId), ]); } @@ -1801,16 +1756,16 @@ export function createEventsStorage( let writeHookEntityWithOverwrite = false; if (!tokenClaimed) { - const existingClaim = await readHookTokenClaim(constraintPath); + let existingClaim = await readHookTokenClaim(constraintPath); if ( existingClaim?.runId === effectiveRunId && existingClaim.hookId === undefined && - existingClaim.phase !== 'retained' + existingClaim.expiresAt === undefined ) { + // Unmaterialized same-run start claim: bind it to this hook. tokenClaimed = await materializeStartHookClaim( basedir, - tag, hookData.token, { token: hookData.token, @@ -1820,14 +1775,21 @@ export function createEventsStorage( ttlSeconds: existingClaim.ttlSeconds, startEventId: existingClaim.startEventId, createdAt: existingClaim.createdAt, - phase: 'materialized', tag: existingClaim.tag, } ); + if (!tokenClaimed) { + // Lost a concurrent materialize. Re-read so the dedup + // branch below sees the winner's claim — if the winner was + // a duplicate of this very (runId, hookId), this converges + // on its canonical eventId instead of emitting a spurious + // self-conflict. + existingClaim = await readHookTokenClaim(constraintPath); + } } if ( !tokenClaimed && - existingClaim?.phase === 'retained' && + existingClaim?.expiresAt !== undefined && (await replaceExpiredStartClaim(basedir, tag, hookData.token, { token: hookData.token, hookId: data.correlationId, @@ -1842,7 +1804,7 @@ export function createEventsStorage( !tokenClaimed && existingClaim?.runId === effectiveRunId && existingClaim.hookId === data.correlationId && - existingClaim.phase !== 'retained' + existingClaim.expiresAt === undefined ) { // Adopt a canonical eventId for the recovery write. The // outer event publish (`writeExclusive(eventPath)`) @@ -2068,27 +2030,15 @@ export function createEventsStorage( ); const claim = await readHookTokenClaim(disposedConstraintPath); if (claim?.runId === existingHook.runId && claim.ttlSeconds) { - await writeJSON( + await retainStartHookClaim( disposedConstraintPath, { + ...claim, token: existingHook.token, hookId: existingHook.hookId, - runId: existingHook.runId, - eventId: claim.eventId, - ttlSeconds: claim.ttlSeconds, - startEventId: claim.startEventId, - createdAt: claim.createdAt, - phase: 'retained', - tag: claim.tag, - expiresAt: new Date( - Math.max( - existingHook.createdAt.getTime() + - claim.ttlSeconds * 1000, - Date.now() - ) - ), }, - { overwrite: true } + now, + existingHook.createdAt ); } else { await deleteJSON(disposedConstraintPath); diff --git a/packages/world-local/src/storage/helpers.ts b/packages/world-local/src/storage/helpers.ts index 23156df992..6751125db1 100644 --- a/packages/world-local/src/storage/helpers.ts +++ b/packages/world-local/src/storage/helpers.ts @@ -1,7 +1,8 @@ import { createHash } from 'node:crypto'; import path from 'node:path'; import { monotonicFactory } from 'ulid'; -import { stripTag, ulidToDate } from '../fs.js'; +import { z } from 'zod'; +import { readJSON, stripTag, ulidToDate } from '../fs.js'; /** * Hash a hook token to produce a filesystem-safe constraint filename. @@ -10,6 +11,70 @@ export function hashToken(token: string): string { return createHash('sha256').update(token).digest('hex'); } +/** + * The token-claim constraint file: the single-owner record behind a hook + * token. A claim's lifecycle state is derived, not stored: + * - unmaterialized start claim: `startEventId` set, no `hookId`, no + * `expiresAt` (reserved by `start()`, no hook entity yet) + * - materialized: `hookId` set, no `expiresAt` (live hook entity) + * - retained: `expiresAt` set (hook disposed / run finished; the token + * stays fenced until `expiresAt`) + * + * Claims without `ttlSeconds` are plain `createHook` token guards with no + * retention window — they are deleted when the hook is disposed or the run + * reaches a terminal state. + */ +export const HookTokenClaimSchema = z.object({ + token: z.string().optional(), + // The token-claim writer has always persisted `hookId`, but an older + // read schema omitted it, which is the bug fixed by + // https://github.com/vercel/workflow/issues/2283. `optional()` is + // defensive: any claim file that somehow lacks the field still parses + // (yielding `undefined`) and falls through to the cross-hook conflict + // branch, matching pre-fix behavior. + hookId: z.string().optional(), + runId: z.string(), + // `eventId` is the canonical hook_created event ID the claiming worker + // committed to publishing. Persisting it here turns the claim file into + // a durable convergence key for cross-worker / cross-process retries + // (see the hook_created branch in events-storage.ts). `optional()` for + // backward compatibility: a legacy claim file written before this field + // existed falls through to the recovery-marker upgrade path, which + // atomically pins a canonical eventId via a sidecar marker. + eventId: z.string().optional(), + ttlSeconds: z.number().int().positive().optional(), + // `startEventId` is the canonical run_created event ID for claims + // reserved by `start()` — the same convergence-key role `eventId` plays + // for hook_created. + startEventId: z.string().optional(), + createdAt: z.coerce.date().optional(), + expiresAt: z.coerce.date().optional(), + tag: z.string().optional(), +}); + +export type HookTokenClaim = z.infer; + +export function hookTokenClaimPath(basedir: string, token: string): string { + return path.join(basedir, 'hooks', 'tokens', `${hashToken(token)}.json`); +} + +/** + * Tolerant claim reader: malformed or partially-written claim files parse + * to `null` (treated as "no claim") instead of failing the caller. + */ +export async function readHookTokenClaim( + constraintPath: string +): Promise { + try { + return await readJSON(constraintPath, HookTokenClaimSchema); + } catch (error) { + if (error instanceof SyntaxError || error instanceof z.ZodError) { + return null; + } + throw error; + } +} + /** * Compute the path of the recovery-marker sidecar for a specific * `(token, runId, hookId)` triple. Identity is encoded in the diff --git a/packages/world-local/src/storage/hooks-storage.ts b/packages/world-local/src/storage/hooks-storage.ts index 30104b128c..e14fd939a3 100644 --- a/packages/world-local/src/storage/hooks-storage.ts +++ b/packages/world-local/src/storage/hooks-storage.ts @@ -8,7 +8,6 @@ import type { Storage, } from '@workflow/world'; import { HookSchema } from '@workflow/world'; -import { z } from 'zod'; import { DEFAULT_RESOLVE_DATA_OPTION } from '../config.js'; import { assertSafeEntityId, @@ -20,24 +19,22 @@ import { writeJSON, } from '../fs.js'; import { filterHookData } from './filters.js'; -import { hashToken, hookRecoveryMarkerPath } from './helpers.js'; - -const HookTokenClaimSchema = z.object({ - token: z.string().optional(), - hookId: z.string().optional(), - runId: z.string(), - eventId: z.string().optional(), - ttlSeconds: z.number().int().positive().optional(), - startEventId: z.string().optional(), - createdAt: z.coerce.date().optional(), - expiresAt: z.coerce.date().optional(), - phase: z.enum(['start_claim', 'materialized', 'retained']).optional(), - tag: z.string().optional(), -}); +import { + type HookTokenClaim, + hashToken, + hookRecoveryMarkerPath, + readHookTokenClaim, +} from './helpers.js'; -async function retainStartHookClaim( +/** + * Transitions a TTL-carrying claim to retained: the token stays fenced + * until at least `hookCreatedAt + ttl` (falling back to the claim's own + * createdAt when no hook was ever created), never shrinking an existing + * retention window and never expiring before `now`. + */ +export async function retainStartHookClaim( constraintPath: string, - claim: z.infer, + claim: HookTokenClaim, now: Date, hookCreatedAt?: Date ): Promise { @@ -62,7 +59,6 @@ async function retainStartHookClaim( ttlSeconds: claim.ttlSeconds, startEventId: claim.startEventId, createdAt: claim.createdAt, - phase: 'retained', tag: claim.tag, expiresAt, }, @@ -163,10 +159,16 @@ export function createHooksStorage( /** * Helper function to delete all hooks associated with a workflow run. * Called when a run reaches a terminal state. + * + * `releaseUnmaterializedClaims` (used for cancellation) additionally deletes + * start-hook claims the workflow never materialized into a hook, so + * cancel-then-retry — including `start()`'s own cleanup when queueing fails + * after admission — can reuse the token immediately. */ export async function deleteAllHooksForRun( basedir: string, - runId: string + runId: string, + opts?: { releaseUnmaterializedClaims?: boolean } ): Promise { const hooksDir = path.join(basedir, 'hooks'); const files = await listJSONFiles(hooksDir); @@ -188,7 +190,7 @@ export async function deleteAllHooksForRun( 'tokens', `${hashToken(hook.token)}.json` ); - const claim = await readJSON(constraintPath, HookTokenClaimSchema); + const claim = await readHookTokenClaim(constraintPath); if (claim?.runId === runId && claim.ttlSeconds) { await retainStartHookClaim( constraintPath, @@ -209,8 +211,15 @@ export async function deleteAllHooksForRun( for (const file of await listJSONFiles(tokensDir)) { if (file.endsWith('.recovery')) continue; const constraintPath = path.join(tokensDir, `${file}.json`); - const claim = await readJSON(constraintPath, HookTokenClaimSchema); - if (claim?.runId === runId && claim.ttlSeconds) { + const claim = await readHookTokenClaim(constraintPath); + if (claim?.runId !== runId || !claim.ttlSeconds) continue; + if ( + opts?.releaseUnmaterializedClaims && + claim.hookId === undefined && + claim.expiresAt === undefined + ) { + await deleteJSON(constraintPath); + } else { await retainStartHookClaim(constraintPath, claim, now); } } diff --git a/packages/world-postgres/src/drizzle/migrations/0015_add_hook_claims.sql b/packages/world-postgres/src/drizzle/migrations/0015_add_hook_claims.sql index 040135ae45..03e9245477 100644 --- a/packages/world-postgres/src/drizzle/migrations/0015_add_hook_claims.sql +++ b/packages/world-postgres/src/drizzle/migrations/0015_add_hook_claims.sql @@ -2,7 +2,6 @@ CREATE TABLE IF NOT EXISTS "workflow"."workflow_hook_claims" ( "token" varchar PRIMARY KEY NOT NULL, "run_id" varchar NOT NULL, "hook_id" varchar, - "phase" varchar NOT NULL, "ttl_seconds" integer NOT NULL, "expires_at" timestamp, "created_at" timestamp DEFAULT now() NOT NULL diff --git a/packages/world-postgres/src/drizzle/schema.ts b/packages/world-postgres/src/drizzle/schema.ts index 6710cdd0df..b572e6e24d 100644 --- a/packages/world-postgres/src/drizzle/schema.ts +++ b/packages/world-postgres/src/drizzle/schema.ts @@ -219,15 +219,26 @@ export const hooks = schema.table( (tb) => [index().on(tb.runId), index().on(tb.token)] ); +/** + * Hook-token claims: the single-owner constraint behind hook tokens. + * + * A claim's lifecycle state is derived, not stored: + * - unmaterialized start claim: `hookId IS NULL` (reserved by `start()`, + * no hook entity yet) + * - materialized: `hookId` set, `expiresAt IS NULL` (live hook entity) + * - retained: `expiresAt` set (hook disposed / run finished; the token + * stays fenced until `expiresAt`) + * + * `ttlSeconds <= 0` marks a plain `createHook` token guard with no + * retention window — it is released when the hook is disposed or the run + * reaches a terminal state. + */ export const hookClaims = schema.table( 'workflow_hook_claims', { token: varchar('token').primaryKey(), runId: varchar('run_id').notNull(), hookId: varchar('hook_id'), - phase: varchar('phase') - .$type<'start_claim' | 'materialized' | 'retained'>() - .notNull(), ttlSeconds: integer('ttl_seconds').notNull(), expiresAt: timestamp('expires_at'), createdAt: timestamp('created_at').defaultNow().notNull(), diff --git a/packages/world-postgres/src/index.ts b/packages/world-postgres/src/index.ts index 8a26583607..3aa53def60 100644 --- a/packages/world-postgres/src/index.ts +++ b/packages/world-postgres/src/index.ts @@ -58,7 +58,7 @@ export function createWorld( return { specVersion: SPEC_VERSION_CURRENT, - supportsExperimentalStartHook: true, + experimentalStartHookAdmission: { mode: 'event-first' }, ...storage, ...streamer, ...queue, diff --git a/packages/world-postgres/src/storage.ts b/packages/world-postgres/src/storage.ts index 6b878a1829..fca995e0fd 100644 --- a/packages/world-postgres/src/storage.ts +++ b/packages/world-postgres/src/storage.ts @@ -402,29 +402,43 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { .limit(1) .prepare('events_get_wait_for_validation'); + /** + * Reads the claim for a token, lazily reclaiming dead claims. A claim is + * live while its expiry (explicit `expiresAt`, or `createdAt + ttl` for + * never-retained start claims) is in the future. Claims past expiry — and + * plain `createHook` guards (`ttlSeconds <= 0`), which have no expiry — + * additionally survive while their owning run is still active; otherwise + * they are debris from a crashed cleanup and are deleted here. + */ async function getLiveHookClaim(token: string): Promise<{ runId: string; - phase: 'start_claim' | 'materialized' | 'retained'; + hookId: string | null; expiresAt: Date | null; } | null> { - const [claim] = await drizzle - .select({ - runId: Schema.hookClaims.runId, - phase: Schema.hookClaims.phase, - expiresAt: Schema.hookClaims.expiresAt, - ttlSeconds: Schema.hookClaims.ttlSeconds, - createdAt: Schema.hookClaims.createdAt, - }) - .from(Schema.hookClaims) - .where(eq(Schema.hookClaims.token, token)) - .limit(1); - if (!claim) return null; - const expiresAt = - claim.expiresAt ?? - (claim.ttlSeconds > 0 - ? new Date(claim.createdAt.getTime() + claim.ttlSeconds * 1000) - : null); - if (expiresAt && expiresAt <= new Date()) { + // Bounded loop instead of recursion: the guarded DELETE below can miss + // when a concurrent writer replaces the row between the read and the + // delete; re-read and re-evaluate. + for (let attempt = 0; attempt < 3; attempt++) { + const [claim] = await drizzle + .select({ + runId: Schema.hookClaims.runId, + hookId: Schema.hookClaims.hookId, + expiresAt: Schema.hookClaims.expiresAt, + ttlSeconds: Schema.hookClaims.ttlSeconds, + createdAt: Schema.hookClaims.createdAt, + }) + .from(Schema.hookClaims) + .where(eq(Schema.hookClaims.token, token)) + .limit(1); + if (!claim) return null; + + const expiresAt = + claim.expiresAt ?? + (claim.ttlSeconds > 0 + ? new Date(claim.createdAt.getTime() + claim.ttlSeconds * 1000) + : null); + if (expiresAt && expiresAt > new Date()) return claim; + const [run] = await drizzle .select({ status: Schema.runs.status }) .from(Schema.runs) @@ -433,85 +447,89 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { if (run && !['completed', 'failed', 'cancelled'].includes(run.status)) { return claim; } + const [deleted] = await drizzle .delete(Schema.hookClaims) .where( and( eq(Schema.hookClaims.token, token), eq(Schema.hookClaims.runId, claim.runId), - eq(Schema.hookClaims.phase, claim.phase), + claim.hookId + ? eq(Schema.hookClaims.hookId, claim.hookId) + : isNull(Schema.hookClaims.hookId), claim.expiresAt ? eq(Schema.hookClaims.expiresAt, claim.expiresAt) : isNull(Schema.hookClaims.expiresAt) ) ) .returning({ token: Schema.hookClaims.token }); - return deleted ? null : getLiveHookClaim(token); + if (deleted) return null; } - return claim; + // The row kept churning under us; treat whatever is there as live — the + // caller reports a conflict, which is the safe default. + const [claim] = await drizzle + .select({ + runId: Schema.hookClaims.runId, + hookId: Schema.hookClaims.hookId, + expiresAt: Schema.hookClaims.expiresAt, + }) + .from(Schema.hookClaims) + .where(eq(Schema.hookClaims.token, token)) + .limit(1); + return claim ?? null; } + /** + * Transitions a finishing run's hook-token claims. Called before the + * run's hook entities are deleted, so the retention window can anchor on + * the hook's `createdAt`. Set-based statements so a concurrent + * materialize cannot interleave with a read-modify-write. + */ async function retainHookClaimsForRun( runId: string, - completedAt: Date + completedAt: Date, + opts?: { releaseUnmaterializedClaims?: boolean } ): Promise { - const hooksForRun = await drizzle - .select({ - token: Schema.hooks.token, - createdAt: Schema.hooks.createdAt, - }) - .from(Schema.hooks) - .where(eq(Schema.hooks.runId, runId)); - const hookCreatedAtByToken = new Map( - hooksForRun.map((hook) => [hook.token, hook.createdAt]) - ); - - const claimsForRun = await drizzle - .select() - .from(Schema.hookClaims) - .where(eq(Schema.hookClaims.runId, runId)); - - for (const claim of claimsForRun) { - const hookCreatedAt = hookCreatedAtByToken.get(claim.token); - if (claim.ttlSeconds <= 0) { - await drizzle - .delete(Schema.hookClaims) - .where( - and( - eq(Schema.hookClaims.token, claim.token), - eq(Schema.hookClaims.runId, runId), - eq(Schema.hookClaims.phase, claim.phase) - ) - ); - continue; - } - - const ttlBase = hookCreatedAt ?? claim.createdAt; - const expiresAt = new Date( - Math.max( - ttlBase.getTime() + claim.ttlSeconds * 1000, - claim.expiresAt?.getTime() ?? 0, - completedAt.getTime() - ) - ); - - await drizzle + await drizzle.transaction(async (tx) => { + // Plain createHook token guards (ttlSeconds <= 0) are released with + // the run. Cancellation additionally releases start-hook claims the + // workflow never materialized into a hook: a cancelled run did no + // fenced work, and cancel-then-retry (including start()'s own cleanup + // when queueing fails after admission) must be able to reuse the + // token immediately. + await tx + .delete(Schema.hookClaims) + .where( + and( + eq(Schema.hookClaims.runId, runId), + opts?.releaseUnmaterializedClaims + ? sql`(${Schema.hookClaims.ttlSeconds} <= 0 OR ${Schema.hookClaims.hookId} IS NULL)` + : sql`${Schema.hookClaims.ttlSeconds} <= 0` + ) + ); + // Retain the rest until at least `hookCreatedAt + ttl` (falling back + // to the claim's own createdAt when no hook entity exists), never + // shrinking an existing retention window and never expiring before + // run completion. + await tx .update(Schema.hookClaims) .set({ - expiresAt, - phase: 'retained', + expiresAt: sql`GREATEST( + COALESCE( + (SELECT h.created_at FROM ${Schema.hooks} h WHERE h.hook_id = ${Schema.hookClaims.hookId}), + ${Schema.hookClaims.createdAt} + ) + make_interval(secs => ${Schema.hookClaims.ttlSeconds}), + COALESCE(${Schema.hookClaims.expiresAt}, ${completedAt}), + ${completedAt} + )`, }) .where( and( - eq(Schema.hookClaims.token, claim.token), eq(Schema.hookClaims.runId, runId), - eq(Schema.hookClaims.phase, claim.phase), - claim.hookId - ? eq(Schema.hookClaims.hookId, claim.hookId) - : isNull(Schema.hookClaims.hookId) + gt(Schema.hookClaims.ttlSeconds, 0) ) ); - } + }); } return { @@ -675,7 +693,6 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { .values({ token: startHook.token, runId: effectiveRunId, - phase: 'start_claim', ttlSeconds: startHook.ttlSeconds, }) .onConflictDoNothing() @@ -985,6 +1002,12 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { const startHook = eventData.experimentalStartHook; const runValues = startHook ? await (async () => { + // Pre-checks outside the transaction: reject tokens held by a + // legacy hook entity without a claim row, and let + // getLiveHookClaim lazily reclaim dead claims so the insert + // below can succeed. Claim + run + event are committed in one + // transaction, so a live same-run claim means this exact + // run_created was already admitted — a duplicate delivery. const [existingHook] = await getHookByToken.execute({ token: startHook.token, }); @@ -995,53 +1018,23 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { ); } const existingClaim = await getLiveHookClaim(startHook.token); - if (existingClaim) { - if (existingClaim.runId === effectiveRunId) { - const [existingEvent] = await drizzle - .select({ eventId: events.eventId }) - .from(events) - .where( - and( - eq(events.runId, effectiveRunId), - eq(events.eventType, 'run_created') - ) - ) - .orderBy(asc(events.eventId)) - .limit(1); - if (existingEvent) { - throw new EntityConflictError( - `Run "${effectiveRunId}" already exists` - ); - } - - const insertedRuns = await drizzle - .insert(Schema.runs) - .values(pendingRun) - .onConflictDoNothing() - .returning(); - if (!insertedRuns[0]) { - const [existingRun] = await drizzle - .select() - .from(Schema.runs) - .where(eq(Schema.runs.runId, effectiveRunId)) - .limit(1); - if (existingRun) return [existingRun]; - throw new WorkflowRunNotFoundError(effectiveRunId); - } - return insertedRuns; - } + if (existingClaim && existingClaim.runId !== effectiveRunId) { throw new HookConflictError( startHook.token, existingClaim.runId ); } + if (existingClaim) { + throw new EntityConflictError( + `Run "${effectiveRunId}" already exists` + ); + } const result = await drizzle.transaction(async (tx) => { const [createdClaim] = await tx .insert(Schema.hookClaims) .values({ token: startHook.token, runId: effectiveRunId, - phase: 'start_claim', ttlSeconds: startHook.ttlSeconds, }) .onConflictDoNothing() @@ -1052,7 +1045,15 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { .from(Schema.hookClaims) .where(eq(Schema.hookClaims.token, startHook.token)) .limit(1); - throw new HookConflictError(startHook.token, existing?.runId); + // A same-run claim raced in (duplicate delivery of this + // run_created): fall through — the run insert below + // conflicts, yielding EntityConflictError. + if (existing && existing.runId !== effectiveRunId) { + throw new HookConflictError( + startHook.token, + existing.runId + ); + } } const insertedRuns = await tx @@ -1264,8 +1265,13 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { ); } } - // Close active hooks/waits; retained start-hook claims expire separately. - await retainHookClaimsForRun(effectiveRunId, now); + // Close active hooks/waits. Cancellation releases start-hook claims + // the workflow never materialized into a hook, so cancel-then-retry + // (including start()'s own cleanup when queueing fails after + // admission) can reuse the token immediately. + await retainHookClaimsForRun(effectiveRunId, now, { + releaseUnmaterializedClaims: true, + }); await Promise.all([ drizzle .delete(Schema.hooks) @@ -1643,8 +1649,8 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { } } - // Handle hook_created event: create hook entity - // Uses prepared statement for token uniqueness check (performance optimization) + // Handle hook_created event: create the hook entity and take (or + // materialize) the token claim. if (data.eventType === 'hook_created') { const eventData = (data as any).eventData as { token: string; @@ -1653,133 +1659,126 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { isSystem?: boolean; }; - // Check for duplicate token using prepared statement + // Emit a hook_conflict event instead of throwing 409 — this lets + // the workflow continue and fail gracefully when the hook is + // awaited. + const emitHookConflict = async (conflictingRunId?: string) => { + const conflictEventData = { + token: eventData.token, + ...(conflictingRunId ? { conflictingRunId } : {}), + }; + const [conflictValue] = await drizzle + .insert(events) + .values({ + runId: effectiveRunId, + eventId, + correlationId: data.correlationId, + eventType: 'hook_conflict', + eventData: conflictEventData, + specVersion: effectiveSpecVersion, + }) + .returning({ createdAt: events.createdAt }); + if (!conflictValue) { + throw new EntityConflictError( + `Event ${eventId} could not be created` + ); + } + const parsedConflict = EventSchema.parse({ + eventType: 'hook_conflict' as const, + correlationId: data.correlationId, + eventData: conflictEventData, + ...conflictValue, + runId: effectiveRunId, + eventId, + specVersion: effectiveSpecVersion, + }); + const resolveData = params?.resolveData ?? 'all'; + return { + event: stripEventDataRefs(parsedConflict, resolveData), + run, + step, + hook: undefined, + }; + }; + + // Snapshot current token ownership. getLiveHookClaim lazily + // reclaims dead claims, so a `null` claim means the token is free. const [existingHook] = await getHookByToken.execute({ token: eventData.token, }); const existingClaim = await getLiveHookClaim(eventData.token); - const materializesStartClaim = - !existingHook && - existingClaim?.runId === effectiveRunId && - existingClaim.phase === 'start_claim'; - if (existingHook || existingClaim?.runId !== undefined) { - // Idempotency: if the existing hook is the *same* (runId, hookId) - // we are trying to create, this is either a duplicate / replayed - // processing of the same hook_created (not a real conflict), or - // an orphaned hook row from a prior crashed attempt (the hook - // INSERT below landed but the events INSERT below didn't — - // these writes are not in one transaction). Distinguish by - // checking whether the `hook_created` event actually exists in - // the event log: - // - exists → real duplicate: throw EntityConflictError so the - // runtime's concurrent-replay catch path (matching the - // step_created path) swallows it, instead of producing a - // self-conflict in the event log that would later replay - // as HookConflictError. - // See https://github.com/vercel/workflow/issues/2283. - // - missing → orphaned hook row (crash between hook INSERT - // and events INSERT): skip the hook insert (the existing - // row already has the desired state) and fall through to - // the events INSERT below, completing the partial write. - if ( - existingHook?.runId === effectiveRunId && - existingHook.hookId === data.correlationId - ) { - const [existingEvent] = await getHookCreatedEvent.execute({ - runId: effectiveRunId, - correlationId: data.correlationId, - eventType: 'hook_created', - }); - if (existingEvent) { - throw new EntityConflictError( - `Hook "${data.correlationId}" already created` - ); - } - if (existingClaim?.phase === 'start_claim') { - await drizzle - .update(Schema.hookClaims) - .set({ - hookId: data.correlationId!, - phase: 'materialized', - }) - .where( - and( - eq(Schema.hookClaims.token, eventData.token), - eq(Schema.hookClaims.runId, effectiveRunId), - eq(Schema.hookClaims.phase, 'start_claim') - ) - ); - } - // Orphaned hook row: hook row exists but no hook_created - // event in the log. Skip the hook insert below (the row - // already exists with our (runId, hookId)) and let the - // outer code path emit the hook_created event, completing - // the partial write. We also re-fetch the existing hook - // row so the EventResult carries the actual persisted - // entity rather than `undefined`. - const [recoveredHookValue] = await drizzle - .select() - .from(Schema.hooks) - .where(eq(Schema.hooks.hookId, data.correlationId!)) - .limit(1); - if (recoveredHookValue) { - recoveredHookValue.metadata ||= recoveredHookValue.metadataJson; - hook = HookSchema.parse(compact(recoveredHookValue)); - } - } else { - // Cross-hook / cross-run conflict: a different - // (runId, hookId) holds this token. Create a hook_conflict - // event instead of throwing 409 — this lets the workflow - // continue and fail gracefully when the hook is awaited. - const conflictingRunId = - existingHook?.runId ?? existingClaim!.runId; - if (materializesStartClaim) { - // This run reserved the token during start(); materialize the - // actual workflow hook below. - } else { - const conflictEventData = { - token: eventData.token, - conflictingRunId, - }; - const [conflictValue] = await drizzle - .insert(events) - .values({ - runId: effectiveRunId, - eventId, - correlationId: data.correlationId, - eventType: 'hook_conflict', - eventData: conflictEventData, - specVersion: effectiveSpecVersion, - }) - .returning({ createdAt: events.createdAt }); - - if (!conflictValue) { - throw new EntityConflictError( - `Event ${eventId} could not be created` - ); - } - - const conflictResult = { - eventType: 'hook_conflict' as const, - correlationId: data.correlationId, - eventData: conflictEventData, - ...conflictValue, - runId: effectiveRunId, - eventId, - }; - const parsedConflict = EventSchema.parse(conflictResult); - const resolveData = params?.resolveData ?? 'all'; - return { - event: stripEventDataRefs(parsedConflict, resolveData), - run, - step, - hook: undefined, - }; - } + const isOwnHookRow = (row: typeof existingHook) => + row?.runId === effectiveRunId && row.hookId === data.correlationId; + + // Adopt our own already-persisted hook row: either a replayed + // duplicate of this hook_created (the hook_created event exists → + // throw EntityConflictError so the runtime's dedup catch path + // swallows it instead of recording a self-conflict, see + // https://github.com/vercel/workflow/issues/2283), or an orphaned + // row from a crash between the entity write and the events INSERT + // (event missing → reuse the row and fall through to the events + // INSERT below, completing the partial write). + const adoptOwnHookRow = async () => { + const [existingEvent] = await getHookCreatedEvent.execute({ + runId: effectiveRunId, + correlationId: data.correlationId, + eventType: 'hook_created', + }); + if (existingEvent) { + throw new EntityConflictError( + `Hook "${data.correlationId}" already created` + ); } - } - if (!existingHook || existingClaim?.runId === effectiveRunId) { + // Repair the claim if it does not reflect the materialized hook + // yet (rows written before the claim update and hook insert were + // transactional). + await drizzle + .update(Schema.hookClaims) + .set({ hookId: data.correlationId! }) + .where( + and( + eq(Schema.hookClaims.token, eventData.token), + eq(Schema.hookClaims.runId, effectiveRunId), + isNull(Schema.hookClaims.hookId) + ) + ); + // Re-fetch the full row so the EventResult carries the actual + // persisted entity. + const [recoveredHookValue] = await drizzle + .select() + .from(Schema.hooks) + .where(eq(Schema.hooks.hookId, data.correlationId!)) + .limit(1); + if (recoveredHookValue) { + recoveredHookValue.metadata ||= recoveredHookValue.metadataJson; + hook = HookSchema.parse(compact(recoveredHookValue)); + } + }; + + if (isOwnHookRow(existingHook)) { + await adoptOwnHookRow(); + } else if (existingHook) { + // A different (runId, hookId) hook entity holds this token. + return await emitHookConflict(existingHook.runId); + } else if ( + existingClaim && + !( + existingClaim.runId === effectiveRunId && + existingClaim.hookId === null && + existingClaim.expiresAt === null + ) + ) { + // The token is held by a claim we cannot materialize: another + // run's claim, or our own claim already bound to a different hook + // or retained after disposal. + return await emitHookConflict(existingClaim.runId); + } else { + // The token is free, or reserved for this run by start(): create + // the hook row and take (or materialize) the claim in one + // transaction. ttlSeconds 0 marks a plain createHook guard with + // no retention window. + const materializesStartClaim = existingClaim !== null; const hookValues = { runId: effectiveRunId, hookId: data.correlationId!, @@ -1793,219 +1792,124 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { isWebhook: eventData.isWebhook, isSystem: eventData.isSystem ?? false, }; - const hookValue = - !existingHook && !existingClaim - ? await drizzle.transaction(async (tx) => { - const [createdClaim] = await tx - .insert(Schema.hookClaims) - .values({ - token: eventData.token, - runId: effectiveRunId, - hookId: data.correlationId!, - phase: 'materialized', - ttlSeconds: 0, - }) - .onConflictDoNothing() - .returning({ token: Schema.hookClaims.token }); - if (!createdClaim) return undefined; - const [createdHook] = await tx - .insert(Schema.hooks) - .values(hookValues) - .onConflictDoNothing() - .returning(); - if (!createdHook) { - throw new EntityConflictError( - `Hook "${data.correlationId}" already created` - ); - } - return createdHook; - }) - : materializesStartClaim - ? await drizzle.transaction(async (tx) => { - const [claimed] = await tx - .update(Schema.hookClaims) - .set({ - hookId: data.correlationId!, - phase: 'materialized', - }) - .where( - and( - eq(Schema.hookClaims.token, eventData.token), - eq(Schema.hookClaims.runId, effectiveRunId), - eq(Schema.hookClaims.phase, 'start_claim'), - isNull(Schema.hookClaims.hookId) - ) - ) - .returning({ token: Schema.hookClaims.token }); - if (!claimed) return undefined; - const [createdHook] = await tx - .insert(Schema.hooks) - .values(hookValues) - .onConflictDoNothing() - .returning(); - if (!createdHook) { - throw new EntityConflictError( - `Hook "${data.correlationId}" already created` - ); - } - return createdHook; + const hookValue = await drizzle.transaction(async (tx) => { + const [claimed] = materializesStartClaim + ? await tx + .update(Schema.hookClaims) + .set({ hookId: data.correlationId! }) + .where( + and( + eq(Schema.hookClaims.token, eventData.token), + eq(Schema.hookClaims.runId, effectiveRunId), + isNull(Schema.hookClaims.hookId), + isNull(Schema.hookClaims.expiresAt) + ) + ) + .returning({ token: Schema.hookClaims.token }) + : await tx + .insert(Schema.hookClaims) + .values({ + token: eventData.token, + runId: effectiveRunId, + hookId: data.correlationId!, + ttlSeconds: 0, }) - : ( - await drizzle - .insert(Schema.hooks) - .values(hookValues) - .onConflictDoNothing() - .returning() - )[0]; - if ( - !hookValue && - ((!existingHook && !existingClaim) || materializesStartClaim) - ) { - const racedClaim = await getLiveHookClaim(eventData.token); + .onConflictDoNothing() + .returning({ token: Schema.hookClaims.token }); + if (!claimed) return undefined; + const [createdHook] = await tx + .insert(Schema.hooks) + .values(hookValues) + .onConflictDoNothing() + .returning(); + if (!createdHook) { + throw new EntityConflictError( + `Hook "${data.correlationId}" already created` + ); + } + return createdHook; + }); + + if (hookValue) { + hookValue.metadata ||= hookValue.metadataJson; + hook = HookSchema.parse(compact(hookValue)); + } else { + // Lost the claim to a concurrent writer between the snapshot + // and the transaction. Re-read and converge: our own + // (runId, hookId) winning concurrently is a duplicate/orphan; + // anything else is a conflict. const [racedHook] = await getHookByToken.execute({ token: eventData.token, }); - if ( - racedHook?.runId === effectiveRunId && - racedHook.hookId === data.correlationId - ) { - const [existingEvent] = await getHookCreatedEvent.execute({ - runId: effectiveRunId, - correlationId: data.correlationId, - eventType: 'hook_created', - }); - if (existingEvent) { - throw new EntityConflictError( - `Hook "${data.correlationId}" already created` - ); - } - const [recoveredHookValue] = await drizzle - .select() - .from(Schema.hooks) - .where(eq(Schema.hooks.hookId, data.correlationId!)) - .limit(1); - if (recoveredHookValue) { - recoveredHookValue.metadata ||= recoveredHookValue.metadataJson; - hook = HookSchema.parse(compact(recoveredHookValue)); - } + const racedClaim = await getLiveHookClaim(eventData.token); + if (isOwnHookRow(racedHook)) { + await adoptOwnHookRow(); } else { - const conflictEventData = { - token: eventData.token, - ...((racedHook?.runId ?? racedClaim?.runId) - ? { - conflictingRunId: racedHook?.runId ?? racedClaim?.runId, - } - : {}), - }; - const [conflictValue] = await drizzle - .insert(events) - .values({ - runId: effectiveRunId, - eventId, - correlationId: data.correlationId, - eventType: 'hook_conflict', - eventData: conflictEventData, - specVersion: effectiveSpecVersion, - }) - .returning({ createdAt: events.createdAt }); - if (!conflictValue) { - throw new EntityConflictError( - `Event ${eventId} could not be created` - ); - } - const parsedConflict = EventSchema.parse({ - eventType: 'hook_conflict' as const, - correlationId: data.correlationId, - eventData: conflictEventData, - ...conflictValue, - runId: effectiveRunId, - eventId, - specVersion: effectiveSpecVersion, - }); - const resolveData = params?.resolveData ?? 'all'; - return { - event: stripEventDataRefs(parsedConflict, resolveData), - run, - step, - hook: undefined, - }; - } - } - if (hookValue) { - if ( - existingClaim?.phase === 'start_claim' && - !materializesStartClaim - ) { - await drizzle - .update(Schema.hookClaims) - .set({ - hookId: data.correlationId!, - phase: 'materialized', - }) - .where( - and( - eq(Schema.hookClaims.token, eventData.token), - eq(Schema.hookClaims.runId, effectiveRunId), - eq(Schema.hookClaims.phase, 'start_claim') - ) - ); + return await emitHookConflict( + racedHook?.runId ?? racedClaim?.runId + ); } - hookValue.metadata ||= hookValue.metadataJson; - hook = HookSchema.parse(compact(hookValue)); } } } - // Handle hook_disposed event: delete hook entity atomically. - // Uses DELETE ... RETURNING to ensure only one concurrent caller - // succeeds — if no rows are returned, the hook was already disposed. + // Handle hook_disposed event: delete the hook entity and settle its + // token claim atomically. The DELETE ... RETURNING ensures only one + // concurrent disposer succeeds (a losing tx rolls back its claim + // write); claims with a retention TTL stay behind as retained + // constraints so duplicate starts remain fenced, plain guards are + // released. if (data.eventType === 'hook_disposed' && data.correlationId) { const hookId = data.correlationId; - const [hookToDispose] = await drizzle - .select({ - hookId: Schema.hooks.hookId, - token: Schema.hooks.token, - createdAt: Schema.hooks.createdAt, - }) - .from(Schema.hooks) - .where(eq(Schema.hooks.hookId, hookId)) - .limit(1); - if (!hookToDispose) { - throw new EntityConflictError(`Hook "${hookId}" already disposed`); - } - const [claim] = await drizzle - .select() - .from(Schema.hookClaims) - .where(eq(Schema.hookClaims.token, hookToDispose.token)) - .limit(1); await drizzle.transaction(async (tx) => { - if (!claim || claim.ttlSeconds <= 0) { + const [disposed] = await tx + .delete(Schema.hooks) + .where(eq(Schema.hooks.hookId, hookId)) + .returning({ + runId: Schema.hooks.runId, + token: Schema.hooks.token, + createdAt: Schema.hooks.createdAt, + }); + if (!disposed) { + throw new EntityConflictError(`Hook "${hookId}" already disposed`); + } + const [claim] = await tx + .select({ + runId: Schema.hookClaims.runId, + ttlSeconds: Schema.hookClaims.ttlSeconds, + expiresAt: Schema.hookClaims.expiresAt, + }) + .from(Schema.hookClaims) + .where(eq(Schema.hookClaims.token, disposed.token)) + .limit(1); + // Only touch the claim if the disposed hook's run owns it. + if (!claim || claim.runId !== disposed.runId) return; + if (claim.ttlSeconds <= 0) { await tx .delete(Schema.hookClaims) - .where(eq(Schema.hookClaims.token, hookToDispose.token)); + .where( + and( + eq(Schema.hookClaims.token, disposed.token), + eq(Schema.hookClaims.runId, disposed.runId) + ) + ); } else { const expiresAt = new Date( Math.max( - hookToDispose.createdAt.getTime() + claim.ttlSeconds * 1000, + disposed.createdAt.getTime() + claim.ttlSeconds * 1000, + claim.expiresAt?.getTime() ?? 0, now.getTime() ) ); await tx .update(Schema.hookClaims) - .set({ - expiresAt, - hookId: hookToDispose.hookId, - phase: 'retained', - }) - .where(eq(Schema.hookClaims.token, hookToDispose.token)); - } - - const [deleted] = await tx - .delete(Schema.hooks) - .where(eq(Schema.hooks.hookId, hookId)) - .returning({ hookId: Schema.hooks.hookId }); - if (!deleted) { - throw new EntityConflictError(`Hook "${hookId}" already disposed`); + .set({ expiresAt, hookId }) + .where( + and( + eq(Schema.hookClaims.token, disposed.token), + eq(Schema.hookClaims.runId, disposed.runId) + ) + ); } }); } diff --git a/packages/world-postgres/test/storage.test.ts b/packages/world-postgres/test/storage.test.ts index 6a0a9f64e2..ff9574c5ee 100644 --- a/packages/world-postgres/test/storage.test.ts +++ b/packages/world-postgres/test/storage.test.ts @@ -1638,46 +1638,45 @@ describe('Storage (Postgres integration)', () => { ).rejects.toThrow(EntityConflictError); }); - it('should repair a same-run start hook claim when the run_created event is missing', async () => { - const token = 'postgres-missing-run-created-start-claim-token'; - const runId = `wrun_${monotonicFactory()()}`; - - await drizzle.insert(DrizzleSchema.runs).values({ - runId, + it('should not duplicate the run_created event on a concurrent same-run retry', async () => { + const token = 'postgres-same-run-concurrent-start-claim-token'; + const request = { deploymentId: 'deployment-123', workflowName: 'test-workflow', - status: 'pending', input: new Uint8Array(), - attributes: {}, - specVersion: SPEC_VERSION_CURRENT, - }); - await drizzle.insert(DrizzleSchema.hookClaims).values({ - token, - runId, - phase: 'start_claim', - ttlSeconds: 60, - }); + experimentalStartHook: { token, ttlSeconds: 60 }, + }; + const runId = `wrun_${monotonicFactory()()}`; - await expect( + // Two deliveries of the same run_created (same runId + token) must + // admit exactly once; the loser gets EntityConflictError — not a + // HookConflictError against its own run. + const results = await Promise.allSettled([ events.create(runId, { eventType: 'run_created' as const, - eventData: { - deploymentId: 'deployment-123', - workflowName: 'test-workflow', - input: new Uint8Array(), - experimentalStartHook: { token, ttlSeconds: 60 }, - }, - }) - ).resolves.toMatchObject({ - event: { eventType: 'run_created' }, - run: { runId }, + eventData: request, + }), + events.create(runId, { + eventType: 'run_created' as const, + eventData: request, + }), + ]); + + const fulfilled = results.filter((r) => r.status === 'fulfilled'); + const rejected = results.filter( + (r): r is PromiseRejectedResult => r.status === 'rejected' + ); + expect(fulfilled).toHaveLength(1); + expect(rejected).toHaveLength(1); + expect(rejected[0]?.reason).toMatchObject({ + name: 'EntityConflictError', }); - const repairedEvents = await drizzle + const createdEvents = await drizzle .select() .from(DrizzleSchema.events) .where(eq(DrizzleSchema.events.runId, runId)); - expect(repairedEvents).toHaveLength(1); + expect(createdEvents).toHaveLength(1); }); it('should retain a disposed start hook claim until ttl expiry', async () => { @@ -1779,7 +1778,7 @@ describe('Storage (Postgres integration)', () => { expect(materializedClaim).toMatchObject({ runId: first.runId, hookId, - phase: 'materialized', + expiresAt: null, }); await updateRun(events, first.runId, 'run_completed', { @@ -1794,7 +1793,6 @@ describe('Storage (Postgres integration)', () => { expect(retainedClaim).toMatchObject({ runId: first.runId, hookId, - phase: 'retained', }); expect(retainedClaim?.expiresAt?.getTime()).toBeGreaterThan(Date.now()); @@ -1808,8 +1806,8 @@ describe('Storage (Postgres integration)', () => { ).rejects.toThrow(HookConflictError); }); - it('should retain an unmaterialized start hook claim until ttl expiry', async () => { - const token = 'postgres-cancelled-unmaterialized-start-claim-token'; + it('should retain an unmaterialized start hook claim after run failure until ttl expiry', async () => { + const token = 'postgres-failed-unmaterialized-start-claim-token'; const first = await createRun(events, { deploymentId: 'deployment-123', workflowName: 'test-workflow', @@ -1817,8 +1815,8 @@ describe('Storage (Postgres integration)', () => { experimentalStartHook: { token, ttlSeconds: 60 }, }); - await events.create(first.runId, { - eventType: 'run_cancelled' as const, + await updateRun(events, first.runId, 'run_failed', { + error: new Uint8Array(), }); await expect( @@ -1848,6 +1846,34 @@ describe('Storage (Postgres integration)', () => { }); }); + it('should release an unmaterialized start hook claim on cancellation', async () => { + const token = 'postgres-cancelled-unmaterialized-start-claim-token'; + const first = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }); + + // Cancel the claim-owning run before it materialized a hook: the + // token must be immediately reusable (cancel-then-retry, including + // start()'s own cleanup when queueing fails after admission). + await events.create(first.runId, { + eventType: 'run_cancelled' as const, + }); + + await expect( + createRun(events, { + deploymentId: 'deployment-456', + workflowName: 'test-workflow-2', + input: new Uint8Array(), + experimentalStartHook: { token, ttlSeconds: 60 }, + }) + ).resolves.toMatchObject({ + workflowName: 'test-workflow-2', + }); + }); + it('should keep an expired disposed start hook claim while the run is active', async () => { const token = 'postgres-active-expired-start-claim-token'; const first = await createRun(events, { diff --git a/packages/world-vercel/src/events-v4.test.ts b/packages/world-vercel/src/events-v4.test.ts index 3a5a1a60da..a4972df304 100644 --- a/packages/world-vercel/src/events-v4.test.ts +++ b/packages/world-vercel/src/events-v4.test.ts @@ -1,5 +1,6 @@ import { EntityConflictError, + HookConflictError, RunExpiredError, ThrottleError, TooEarlyError, @@ -15,6 +16,19 @@ import { } from './events-v4.js'; import { encodeFrame, V4_FRAME_CONTENT_TYPE } from './frames.js'; +function decodePostedMeta(rawBody: unknown): Record { + const bytes = + typeof rawBody === 'string' + ? new TextEncoder().encode(rawBody) + : new Uint8Array(rawBody as ArrayBufferLike); + const metaLen = new DataView( + bytes.buffer, + bytes.byteOffset, + bytes.byteLength + ).getUint32(0, false); + return decode(bytes.subarray(4, 4 + metaLen)) as Record; +} + /** * The v4 client must preserve the typed-error contract of the v3 * `makeRequest` path — the workflow runtime branches on these types @@ -32,6 +46,25 @@ describe('throwForErrorResponse', () => { expect(() => call(409)).toThrowError(EntityConflictError); }); + it('maps hook-conflict 409s to HookConflictError', () => { + try { + call( + 409, + JSON.stringify({ + code: 'hook_conflict', + message: 'hook conflict', + token: 'order:123', + conflictingRunId: 'wrun_owner', + }) + ); + expect.unreachable(); + } catch (err) { + expect(HookConflictError.is(err)).toBe(true); + expect((err as HookConflictError).token).toBe('order:123'); + expect((err as HookConflictError).conflictingRunId).toBe('wrun_owner'); + } + }); + it('maps 410 to RunExpiredError (terminal run — runtime must not retry)', () => { expect(() => call(410)).toThrowError(RunExpiredError); }); @@ -328,24 +361,7 @@ describe('createWorkflowRunEventV4 over HTTP', () => { const agent = new MockAgent(); agent.disableNetConnect(); - // Decode the posted frame's CBOR meta block: - // u32_be(meta_len) || cbor_meta || u32_be(body_len) || body let capturedMeta: Record | undefined; - const captureMeta = (rawBody: unknown) => { - const bytes = - typeof rawBody === 'string' - ? new TextEncoder().encode(rawBody) - : new Uint8Array(rawBody as ArrayBufferLike); - const metaLen = new DataView( - bytes.buffer, - bytes.byteOffset, - bytes.byteLength - ).getUint32(0, false); - capturedMeta = decode(bytes.subarray(4, 4 + metaLen)) as Record< - string, - unknown - >; - }; agent .get(origin) @@ -356,7 +372,7 @@ describe('createWorkflowRunEventV4 over HTTP', () => { .reply( 200, (opts: { body?: unknown }) => { - captureMeta(opts.body); + capturedMeta = decodePostedMeta(opts.body); return encode({ run: { runId: 'wrun_1', status: 'running' } }); }, { @@ -383,6 +399,50 @@ describe('createWorkflowRunEventV4 over HTTP', () => { agent.assertNoPendingInterceptors(); }); + it('forwards experimentalStartHook in the frame meta', async () => { + const origin = 'https://vercel-workflow.com'; + const agent = new MockAgent(); + agent.disableNetConnect(); + + let capturedMeta: Record | undefined; + const startHook = { token: 'order:123', ttlSeconds: 60 }; + + agent + .get(origin) + .intercept({ + path: '/api/v4/runs/wrun_1/events/run_created', + method: 'POST', + }) + .reply( + 200, + (opts: { body?: unknown }) => { + capturedMeta = decodePostedMeta(opts.body); + return encode({ run: { runId: 'wrun_1', status: 'pending' } }); + }, + { + headers: { + 'x-wf-event-id': 'evnt_1', + 'x-wf-run-id': 'wrun_1', + 'x-wf-created-at': '2026-06-10T00:00:00.000Z', + }, + } + ); + + await createWorkflowRunEventV4( + { + runId: 'wrun_1', + eventType: 'run_created', + specVersion: 5, + experimentalStartHook: startHook, + }, + { token: 'test-token', dispatcher: agent } + ); + + expect(capturedMeta?.eventType).toBe('run_created'); + expect(capturedMeta?.experimentalStartHook).toEqual(startHook); + agent.assertNoPendingInterceptors(); + }); + it('omits skipPreload from the frame meta when not set (default / old SDK parity)', async () => { const origin = 'https://vercel-workflow.com'; const agent = new MockAgent(); diff --git a/packages/world-vercel/src/events-v4.ts b/packages/world-vercel/src/events-v4.ts index 52d5aacb78..d283f470d7 100644 --- a/packages/world-vercel/src/events-v4.ts +++ b/packages/world-vercel/src/events-v4.ts @@ -23,6 +23,7 @@ import { EntityConflictError, + HookConflictError, RunExpiredError, ThrottleError, TooEarlyError, @@ -136,6 +137,9 @@ export interface CreateEventV4Input { * resilient-start path). Validated server-side against the attribute * key/value/count caps. */ attributes?: Record; + /** Experimental start-hook token claim carried by run_created and the + * resilient-start run_started fallback. */ + experimentalStartHook?: { token: string; ttlSeconds: number }; /** attr_set's attribute change list ({key, value|null} entries). */ changes?: Array>; /** attr_set's writer provenance ({type:'workflow'} or @@ -219,6 +223,9 @@ function buildPostFrameMeta( meta.executionContext = input.executionContext; } if (input.attributes !== undefined) meta.attributes = input.attributes; + if (input.experimentalStartHook !== undefined) { + meta.experimentalStartHook = input.experimentalStartHook; + } if (input.changes !== undefined) meta.changes = input.changes; if (input.writer !== undefined) meta.writer = input.writer; if (input.allowReservedAttributes !== undefined) { @@ -253,10 +260,21 @@ export function throwForErrorResponse( ): never { let message = `v4 ${opName} failed: HTTP ${statusCode}`; let code: string | undefined; + let token: string | undefined; + let conflictingRunId: string | undefined; try { - const json = JSON.parse(errorBody) as { message?: string; code?: string }; + const json = JSON.parse(errorBody) as { + message?: string; + code?: string; + token?: string; + conflictingRunId?: string; + }; if (typeof json.message === 'string') message = json.message; if (typeof json.code === 'string') code = json.code; + if (typeof json.token === 'string') token = json.token; + if (typeof json.conflictingRunId === 'string') { + conflictingRunId = json.conflictingRunId; + } } catch { // body wasn't JSON — keep the default message, append raw text below if (errorBody) message += ` ${errorBody}`; @@ -270,6 +288,9 @@ export function throwForErrorResponse( if (!Number.isNaN(parsed)) retryAfter = parsed; } + if (statusCode === 409 && code === 'hook_conflict') { + throw new HookConflictError(token ?? '', conflictingRunId); + } if (statusCode === 409) throw new EntityConflictError(message); if (statusCode === 410) throw new RunExpiredError(message); if (statusCode === 425) throw new TooEarlyError(message, { retryAfter }); diff --git a/packages/world-vercel/src/events.test.ts b/packages/world-vercel/src/events.test.ts index 2faba17bd0..5d77a7fb08 100644 --- a/packages/world-vercel/src/events.test.ts +++ b/packages/world-vercel/src/events.test.ts @@ -177,6 +177,34 @@ describe('splitEventDataForV4 attribute fields', () => { expect(meta.attributes).toEqual({ sourceAtStart: 'api' }); }); + it('carries experimental start hooks on run_created and resilient-start run_started', () => { + const startHook = { token: 'order:123', ttlSeconds: 60 }; + + const created = splitEventDataForV4({ + eventType: 'run_created', + specVersion: 4, + eventData: { + deploymentId: 'dpl_1', + workflowName: 'wf', + input: new TextEncoder().encode('[]'), + experimentalStartHook: startHook, + }, + } as AnyEventRequest); + const started = splitEventDataForV4({ + eventType: 'run_started', + specVersion: 4, + eventData: { + input: new TextEncoder().encode('[]'), + deploymentId: 'dpl_1', + workflowName: 'wf', + experimentalStartHook: startHook, + }, + } as AnyEventRequest); + + expect(created.meta.experimentalStartHook).toEqual(startHook); + expect(started.meta.experimentalStartHook).toEqual(startHook); + }); + it('lifts workflowName into the frame meta on outcome events (step_completed/step_created), keeping the payload in the body', () => { // The backend keys payload refs by workflow name; carrying it in the // frame meta lets the v4 POST handler skip the per-step run lookup. diff --git a/packages/world-vercel/src/events.ts b/packages/world-vercel/src/events.ts index 8e3eb46c67..6583013c44 100644 --- a/packages/world-vercel/src/events.ts +++ b/packages/world-vercel/src/events.ts @@ -179,6 +179,8 @@ interface SplitEventData { executionContext?: Record; /** Initial run attributes (run_created / resilient-start run_started). */ attributes?: Record; + /** Experimental start-hook token claim (run_created / resilient start). */ + experimentalStartHook?: { token: string; ttlSeconds: number }; /** attr_set change list, included verbatim in frame meta. */ changes?: Array>; /** attr_set writer provenance, included verbatim in frame meta. */ @@ -210,16 +212,11 @@ type MetaSourceField = | 'errorCode' | 'executionContext' | 'attributes' + | 'experimentalStartHook' | 'changes' | 'writer' | 'allowReservedAttributes'; -// Vercel start-hook admission cannot be enabled until workflow-server and -// VQS enqueue are one recoverable backend operation. `createVercelWorld` -// does not advertise the capability, so core `start()` rejects before this -// field could reach the v4 event path. -type UnsupportedSourceField = 'experimentalStartHook'; - /** * Compile-time guard that the v4 `eventData` wire allowlist is exhaustive * against the @workflow/world event schemas. @@ -234,10 +231,7 @@ type UnsupportedSourceField = 'experimentalStartHook'; * the constraint '[never, never]'` — the historical "silently dropped" * footgun, now a build break that names the field. */ -type Unhandled = Exclude< - EventDataField, - PayloadField | MetaSourceField | UnsupportedSourceField ->; +type Unhandled = Exclude; type Stale = Exclude; function assertEventDataWireContractExhaustive< _Check extends [never, never], @@ -335,6 +329,16 @@ export function splitEventDataForV4(data: AnyEventRequest): SplitEventData { ) { meta.attributes = eventData.attributes as Record; } + if ( + eventData.experimentalStartHook !== undefined && + eventData.experimentalStartHook !== null && + typeof eventData.experimentalStartHook === 'object' + ) { + meta.experimentalStartHook = eventData.experimentalStartHook as { + token: string; + ttlSeconds: number; + }; + } if (Array.isArray(eventData.changes)) { meta.changes = eventData.changes as Array>; } diff --git a/packages/world-vercel/src/index.ts b/packages/world-vercel/src/index.ts index 0bdaa91ed6..9c1aead29c 100644 --- a/packages/world-vercel/src/index.ts +++ b/packages/world-vercel/src/index.ts @@ -8,6 +8,9 @@ import { createStorage } from './storage.js'; import { createStreamer } from './streamer.js'; import type { APIConfig } from './utils.js'; +const EXPERIMENTAL_START_HOOK_TTL_MAX_SECONDS = 30 * 24 * 60 * 60; +const EXPERIMENTAL_START_HOOK_TOKEN_MAX_BYTES = 255; + export { createGetEncryptionKeyForRun, deriveRunKey, @@ -39,6 +42,11 @@ export function createVercelWorld(config?: APIConfig): World { // `process.exit(1)` is an acceptable response to an exhausted replay // budget. processExitTriggersQueueRedelivery: true, + experimentalStartHookAdmission: { + mode: 'queue-first', + maxTtlSeconds: EXPERIMENTAL_START_HOOK_TTL_MAX_SECONDS, + maxTokenBytes: EXPERIMENTAL_START_HOOK_TOKEN_MAX_BYTES, + }, ...createQueue(config), ...createStorage(config), ...instrumentObject('world.streams', createStreamer(config)), diff --git a/packages/world/src/interfaces.ts b/packages/world/src/interfaces.ts index 496c389b5a..ae9b3bf53f 100644 --- a/packages/world/src/interfaces.ts +++ b/packages/world/src/interfaces.ts @@ -286,10 +286,12 @@ export interface World extends Queue, Streamer, Storage { specVersion?: number; /** - * Whether this World atomically reserves `run_created.eventData.experimentalStartHook` + * How this World atomically reserves `run_created.eventData.experimentalStartHook` * tokens with run admission. */ - supportsExperimentalStartHook?: true; + experimentalStartHookAdmission?: + | { mode: 'event-first' } + | { mode: 'queue-first'; maxTtlSeconds: number; maxTokenBytes: number }; /** * Whether calling `process.exit(1)` from a queue handler is observed by From f0dae3867a30c76c5a9afd0957e8c094a3028b82 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Wed, 1 Jul 2026 16:51:25 -0700 Subject: [PATCH 4/4] Simplify start-hook implementation after review pass - Bump experimentalStartHookLoserAck capability cutoff to 5.0.0-beta.27 (beta.26 was published from main without this change) and rename it away from colliding with World.experimentalStartHookAdmission - Derive WorkflowStartError.queued from stage (collapses the options union and its serialized/reducer/reviver/web-hydration copies) - Share ExperimentalStartHookSchema/type from @workflow/world; align the queue-side token validation (min(1)) with the event schema - Classify 409 hook_conflict inside the shared errorForResponse so v3 and v4 request paths map it identically - Postgres: extract getOwnStartHookClaim/claimStartHookTokenInTx (the two run-creation paths had drifted), parallelize hook/claim snapshot reads, settle disposed-hook claims with guarded statements instead of SELECT-then-branch, drop the retention transaction (statements touch disjoint rows), and GC expired claim debris of terminal runs - Local: one claim lock per token (reclaim and materialize now mutually exclude), expiry pre-check before taking the reclaim lock, shared settleClaimForDisposedHook, parallel tokens-dir sweep that skips already-settled claims and GCs expired debris - start(): hoist repeated enqueue/create calls, contractError helper, drop the positional verifyRunId flag; stop logging raw hook tokens --- packages/core/src/capabilities.test.ts | 17 +- packages/core/src/capabilities.ts | 22 +- packages/core/src/runtime.ts | 1 - packages/core/src/runtime/start.ts | 90 ++--- packages/core/src/serialization.test.ts | 2 - .../core/src/serialization/reducers/common.ts | 24 +- packages/core/src/serialization/types.ts | 8 +- packages/errors/src/index.ts | 20 +- packages/web-shared/src/lib/hydration.ts | 2 +- packages/web-shared/test/hydration.test.ts | 1 - .../test/serializable-revivers.test.ts | 6 +- packages/world-local/src/index.ts | 6 +- .../world-local/src/storage/events-storage.ts | 163 ++++---- .../world-local/src/storage/hooks-storage.ts | 117 +++--- packages/world-postgres/src/storage.ts | 358 +++++++++--------- packages/world-vercel/src/events-v4.ts | 12 +- packages/world-vercel/src/events.ts | 9 +- packages/world-vercel/src/http-core.ts | 11 + packages/world-vercel/src/utils.ts | 19 +- packages/world/src/events.ts | 11 +- packages/world/src/index.ts | 1 + packages/world/src/queue.ts | 6 +- 22 files changed, 449 insertions(+), 457 deletions(-) diff --git a/packages/core/src/capabilities.test.ts b/packages/core/src/capabilities.test.ts index 8159671a14..c87b2b42ea 100644 --- a/packages/core/src/capabilities.test.ts +++ b/packages/core/src/capabilities.test.ts @@ -104,27 +104,28 @@ describe('getRunCapabilities', () => { }); }); - describe('experimentalStartHookAdmission', () => { + describe('experimentalStartHookLoserAck', () => { it('is false when version is missing or invalid', () => { - expect(getRunCapabilities(undefined).experimentalStartHookAdmission).toBe( + expect(getRunCapabilities(undefined).experimentalStartHookLoserAck).toBe( false ); expect( - getRunCapabilities('not-a-version').experimentalStartHookAdmission + getRunCapabilities('not-a-version').experimentalStartHookLoserAck ).toBe(false); }); - it('is false before the start-hook admission cutoff', () => { + it('is false before the loser-ack cutoff', () => { + // beta.26 was published from main without loser-ack handling. expect( - getRunCapabilities('5.0.0-beta.25').experimentalStartHookAdmission + getRunCapabilities('5.0.0-beta.26').experimentalStartHookLoserAck ).toBe(false); }); - it('is true at and after the start-hook admission cutoff', () => { + it('is true at and after the loser-ack cutoff', () => { expect( - getRunCapabilities('5.0.0-beta.26').experimentalStartHookAdmission + getRunCapabilities('5.0.0-beta.27').experimentalStartHookLoserAck ).toBe(true); - expect(getRunCapabilities('5.0.0').experimentalStartHookAdmission).toBe( + expect(getRunCapabilities('5.0.0').experimentalStartHookLoserAck).toBe( true ); }); diff --git a/packages/core/src/capabilities.ts b/packages/core/src/capabilities.ts index 1789c07d48..394b79478a 100644 --- a/packages/core/src/capabilities.ts +++ b/packages/core/src/capabilities.ts @@ -31,7 +31,7 @@ * - `gzip` (gzip payload compression): added in `5.0.0-beta.18` * - `zstd` (zstd payload compression, preferred codec): added in `5.0.0-beta.18` * alongside gzip — they co-ship, so any run that can read one can read both. - * - `experimentalStartHookAdmission` (queued start-hook loser handling): added in `5.0.0-beta.26` + * - `experimentalStartHookLoserAck` (queued start-hook loser handling): added in `5.0.0-beta.27` */ import semver from 'semver'; @@ -62,11 +62,11 @@ export interface RunCapabilities { framedByteStreams: boolean; /** - * Whether the target workflow handler can safely process queue-first - * experimental start-hook runs: turbo is disabled and `HookConflictError` + * Whether the target workflow handler safely handles a queued start-hook + * run that LOST its token claim: turbo is disabled and `HookConflictError` * during `run_started` is acknowledged without running user workflow code. */ - experimentalStartHookAdmission: boolean; + experimentalStartHookLoserAck: boolean; } /** @@ -106,10 +106,14 @@ const CAPABILITY_VERSION_TABLE: ReadonlyArray<{ // delays the optimization (safe). }> = [ { capability: 'framedByteStreams', minVersion: '5.0.0-beta.15' }, - // TODO(release): bump if this change ships after beta.26. + // beta.26 was published from main WITHOUT this change, so the cutoff must + // point past it. TODO(release): bump again if another "Version Packages + // (beta)" PR merges before this change ships. A too-low cutoff runs user + // workflow code on a lost claim (unsafe); too-high merely rejects + // cross-deployment start-hook starts (safe). { - capability: 'experimentalStartHookAdmission', - minVersion: '5.0.0-beta.26', + capability: 'experimentalStartHookLoserAck', + minVersion: '5.0.0-beta.27', }, ]; @@ -138,7 +142,7 @@ export function getRunCapabilities( return { supportedFormats: BASELINE_FORMATS, framedByteStreams: false, - experimentalStartHookAdmission: false, + experimentalStartHookLoserAck: false, }; } @@ -153,7 +157,7 @@ export function getRunCapabilities( const result: RunCapabilities = { supportedFormats: formats, framedByteStreams: false, - experimentalStartHookAdmission: false, + experimentalStartHookLoserAck: false, }; for (const { capability, minVersion } of CAPABILITY_VERSION_TABLE) { diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 59440f8a80..9381642aed 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -892,7 +892,6 @@ export function workflowEntrypoint( 'Start-hook claim lost during setup, skipping queued run', { workflowRunId: runId, - token: runInput.experimentalStartHook.token, message: err.message, } ); diff --git a/packages/core/src/runtime/start.ts b/packages/core/src/runtime/start.ts index aef57b51b1..236ab3cd87 100644 --- a/packages/core/src/runtime/start.ts +++ b/packages/core/src/runtime/start.ts @@ -113,24 +113,26 @@ async function hasDurableRunCreatedEvent( } function getRunCreatedResult( - runId: string, result: Awaited>, - verifyRunId = true + expectedRunId?: string ) { if (!result.run) { throw new WorkflowRuntimeError( "Missing 'run' in server response for 'run_created' event" ); } - if (verifyRunId && result.run.runId !== runId) { + if (expectedRunId !== undefined && result.run.runId !== expectedRunId) { throw new WorkflowRuntimeError( - `Server returned different runId than requested: expected ${runId}, got ${result.run.runId}` + `Server returned different runId than requested: expected ${expectedRunId}, got ${result.run.runId}` ); } return result.run; } function isWorkflowWorldError(error: unknown): error is WorkflowWorldError { + // `.is()` alone misses subclasses (their `name` differs, e.g. + // ThrottleError); `instanceof` alone misses cross-realm base instances. + // World code runs in the host realm, so together they cover both. return error instanceof WorkflowWorldError || WorkflowWorldError.is(error); } @@ -367,7 +369,7 @@ export async function start( SerializationFormat.GZIP ); targetSupportsStartHookAdmission = - capabilities.experimentalStartHookAdmission; + capabilities.experimentalStartHookLoserAck; } const ops: Promise[] = []; @@ -417,33 +419,33 @@ export async function start( ); } if (startHookAdmission.mode === 'queue-first') { + const contractError = (message: string) => + new WorkflowWorldError(message, { + code: RUN_ERROR_CODES.WORLD_CONTRACT_ERROR, + }); if (!targetSupportsStartHookAdmission) { - throw new WorkflowWorldError( - 'experimentalStartHook requires a target deployment that supports queue-first start-hook admission.', - { code: RUN_ERROR_CODES.WORLD_CONTRACT_ERROR } + throw contractError( + 'experimentalStartHook requires a target deployment that supports queue-first start-hook admission.' ); } if ( experimentalStartHook.ttlSeconds > startHookAdmission.maxTtlSeconds ) { - throw new WorkflowWorldError( - `experimentalStartHook.experimental_ttl exceeds this World's maximum of ${startHookAdmission.maxTtlSeconds} seconds.`, - { code: RUN_ERROR_CODES.WORLD_CONTRACT_ERROR } + throw contractError( + `experimentalStartHook.experimental_ttl exceeds this World's maximum of ${startHookAdmission.maxTtlSeconds} seconds.` ); } if ( textEncoder.encode(experimentalStartHook.token).byteLength > startHookAdmission.maxTokenBytes ) { - throw new WorkflowWorldError( - `experimentalStartHook.token exceeds this World's maximum of ${startHookAdmission.maxTokenBytes} bytes.`, - { code: RUN_ERROR_CODES.WORLD_CONTRACT_ERROR } + throw contractError( + `experimentalStartHook.token exceeds this World's maximum of ${startHookAdmission.maxTokenBytes} bytes.` ); } if (specVersion < SPEC_VERSION_SUPPORTS_CBOR_QUEUE_TRANSPORT) { - throw new WorkflowWorldError( - 'experimentalStartHook requires a spec version that supports runInput queue transport in queue-first Worlds.', - { code: RUN_ERROR_CODES.WORLD_CONTRACT_ERROR } + throw contractError( + 'experimentalStartHook requires a spec version that supports runInput queue transport in queue-first Worlds.' ); } } @@ -558,6 +560,14 @@ export async function start( : {}), } satisfies WorkflowInvokePayload; + const createRunEvent = () => + world.events.create(runId, runCreatedEvent, { v1Compat }); + const enqueueRun = () => + world.queue(getWorkflowQueueName(workflowName), queuePayload, { + deploymentId, + specVersion, + }); + let resilientStart = false; let createdRunForSpan: | NonNullable>['run']> @@ -565,14 +575,7 @@ export async function start( if (experimentalStartHook) { if (startHookAdmission?.mode === 'queue-first') { try { - await world.queue( - getWorkflowQueueName(workflowName), - queuePayload, - { - deploymentId, - specVersion, - } - ); + await enqueueRun(); } catch (error) { scheduleOpsFlush(); throw new WorkflowStartError( @@ -580,7 +583,6 @@ export async function start( { runId, stage: 'queue', - queued: 'unknown', retryable: isRetryableWorldError(error), cause: error, ...getWorkflowWorldErrorDetails(error), @@ -589,10 +591,10 @@ export async function start( } try { - const result = await world.events.create(runId, runCreatedEvent, { - v1Compat, - }); - createdRunForSpan = getRunCreatedResult(runId, result); + createdRunForSpan = getRunCreatedResult( + await createRunEvent(), + runId + ); } catch (error) { scheduleOpsFlush(); if (EntityConflictError.is(error)) { @@ -610,7 +612,6 @@ export async function start( { runId, stage: 'admission', - queued: true, retryable: isRetryableWorldError(error), cause: error, ...getWorkflowWorldErrorDetails(error), @@ -625,20 +626,13 @@ export async function start( // before the run is queued, so a losing start never enqueues work. // Any creation failure — including HookConflictError — fails the // start; nothing was queued, so the caller can simply retry. - const result = await world.events.create(runId, runCreatedEvent, { - v1Compat, - }); - createdRunForSpan = getRunCreatedResult(runId, result); + createdRunForSpan = getRunCreatedResult( + await createRunEvent(), + runId + ); try { - await world.queue( - getWorkflowQueueName(workflowName), - queuePayload, - { - deploymentId, - specVersion, - } - ); + await enqueueRun(); } catch (error) { // Undo admission so the token is not fenced by a run that will // never execute. Cancellation releases unmaterialized start-hook @@ -669,11 +663,8 @@ export async function start( // If events.create fails with 429/5xx, the run was still accepted // via the queue and creation will be re-tried async by the runtime. const [runCreatedResult, queueResult] = await Promise.allSettled([ - world.events.create(runId, runCreatedEvent, { v1Compat }), - world.queue(getWorkflowQueueName(workflowName), queuePayload, { - deploymentId, - specVersion, - }), + createRunEvent(), + enqueueRun(), ]); // Queue failure is always fatal — the run was not enqueued @@ -705,9 +696,8 @@ export async function start( } } else { createdRunForSpan = getRunCreatedResult( - runId, runCreatedResult.value, - !v1Compat + v1Compat ? undefined : runId ); } } diff --git a/packages/core/src/serialization.test.ts b/packages/core/src/serialization.test.ts index 54f78520ae..aa78fd7629 100644 --- a/packages/core/src/serialization.test.ts +++ b/packages/core/src/serialization.test.ts @@ -3989,7 +3989,6 @@ describe('Workflow error serialization', () => { const error = new WorkflowStartError('queued but not admitted', { runId: 'wrun_queued', stage: 'admission', - queued: true, retryable: true, status: 408, retryAfter: 2, @@ -4010,7 +4009,6 @@ describe('Workflow error serialization', () => { const error = new WorkflowStartError('queue response lost', { runId: 'wrun_unknown', stage: 'queue', - queued: 'unknown', retryable: true, }); const serialized = await dehydrateStepReturnValue( diff --git a/packages/core/src/serialization/reducers/common.ts b/packages/core/src/serialization/reducers/common.ts index 59d16e8dde..e46dec7679 100644 --- a/packages/core/src/serialization/reducers/common.ts +++ b/packages/core/src/serialization/reducers/common.ts @@ -233,15 +233,12 @@ export function getCommonReducers( const base = reduceNamedErrorSubclassBase('WorkflowStartError', value); if (!base) return false; const error = value as WorkflowStartError; - const details = { + const reduced: SerializableSpecial['WorkflowStartError'] = { ...base, runId: error.runId, + stage: error.stage, retryable: error.retryable, }; - const reduced: SerializableSpecial['WorkflowStartError'] = - error.stage === 'queue' - ? { ...details, stage: 'queue', queued: 'unknown' } - : { ...details, stage: 'admission', queued: true }; if (error.status !== undefined) reduced.status = error.status; if (error.url !== undefined) reduced.url = error.url; if (error.code !== undefined) reduced.code = error.code; @@ -436,27 +433,16 @@ export function getCommonRevivers( ((global as Record)[ Symbol.for('@workflow/errors//WorkflowStartError') ] as typeof WorkflowStartError | undefined) ?? WorkflowStartError; - const details = { + const error = new Ctor(value.message, { runId: value.runId, + stage: value.stage, retryable: value.retryable, status: value.status, url: value.url, code: value.code, retryAfter: value.retryAfter, ...('cause' in value ? { cause: value.cause } : {}), - }; - const error = - value.stage === 'queue' - ? new Ctor(value.message, { - ...details, - stage: 'queue', - queued: 'unknown', - }) - : new Ctor(value.message, { - ...details, - stage: 'admission', - queued: true, - }); + }); if (value.stack !== undefined) error.stack = value.stack; return error; }, diff --git a/packages/core/src/serialization/types.ts b/packages/core/src/serialization/types.ts index d7898cb464..f4f6bd96f2 100644 --- a/packages/core/src/serialization/types.ts +++ b/packages/core/src/serialization/types.ts @@ -88,15 +88,15 @@ export interface SerializableSpecial { stack?: string; cause?: unknown; runId: string; + // `queued` is derived from `stage` by the constructor, so only `stage` + // is serialized. + stage: 'queue' | 'admission'; retryable: boolean; status?: number; url?: string; code?: string; retryAfter?: number; - } & ( - | { stage: 'queue'; queued: 'unknown' } - | { stage: 'admission'; queued: true } - ); + }; Int8Array: string; // base64 string Int16Array: string; // base64 string Int32Array: string; // base64 string diff --git a/packages/errors/src/index.ts b/packages/errors/src/index.ts index 6cd0b9ecff..097c24deb4 100644 --- a/packages/errors/src/index.ts +++ b/packages/errors/src/index.ts @@ -187,18 +187,21 @@ export class WorkflowWorldError extends WorkflowError { } } -type WorkflowStartErrorOptions = { +interface WorkflowStartErrorOptions { runId: string; + /** + * The phase that failed. `queue` means the enqueue call itself failed (the + * queue may or may not have accepted the message); `admission` means the + * message was queued but admission could not be confirmed. + */ + stage: 'queue' | 'admission'; retryable: boolean; status?: number; url?: string; code?: string; retryAfter?: number; cause?: unknown; -} & ( - | { stage: 'queue'; queued: 'unknown' } - | { stage: 'admission'; queued: true } -); +} /** * Thrown when `start()` cannot synchronously confirm whether a workflow run was @@ -207,8 +210,9 @@ type WorkflowStartErrorOptions = { */ export class WorkflowStartError extends WorkflowWorldError { readonly runId: string; - readonly stage: WorkflowStartErrorOptions['stage']; - readonly queued: WorkflowStartErrorOptions['queued']; + readonly stage: 'queue' | 'admission'; + /** Whether the queue accepted the message — derived from `stage`. */ + readonly queued: true | 'unknown'; readonly retryable: boolean; constructor(message: string, options: WorkflowStartErrorOptions) { @@ -216,7 +220,7 @@ export class WorkflowStartError extends WorkflowWorldError { this.name = 'WorkflowStartError'; this.runId = options.runId; this.stage = options.stage; - this.queued = options.queued; + this.queued = options.stage === 'queue' ? 'unknown' : true; this.retryable = options.retryable; } diff --git a/packages/web-shared/src/lib/hydration.ts b/packages/web-shared/src/lib/hydration.ts index a95e5f0430..94565cf1b6 100644 --- a/packages/web-shared/src/lib/hydration.ts +++ b/packages/web-shared/src/lib/hydration.ts @@ -227,7 +227,7 @@ export function getWebRevivers(): Revivers { error.name = 'WorkflowStartError'; error.runId = value.runId; error.stage = value.stage; - error.queued = value.queued; + error.queued = value.stage === 'queue' ? 'unknown' : true; error.retryable = value.retryable; if (value.status !== undefined) error.status = value.status; if (value.url !== undefined) error.url = value.url; diff --git a/packages/web-shared/test/hydration.test.ts b/packages/web-shared/test/hydration.test.ts index 6b2ff7b9f7..baa93bd100 100644 --- a/packages/web-shared/test/hydration.test.ts +++ b/packages/web-shared/test/hydration.test.ts @@ -140,7 +140,6 @@ describe('getWebRevivers — error family', () => { new WorkflowStartError('admission uncertain', { runId: 'wrun_queued', stage: 'admission', - queued: true, retryable: false, status: 403, }) diff --git a/packages/web-shared/test/serializable-revivers.test.ts b/packages/web-shared/test/serializable-revivers.test.ts index 07c9c66318..3d480a085f 100644 --- a/packages/web-shared/test/serializable-revivers.test.ts +++ b/packages/web-shared/test/serializable-revivers.test.ts @@ -106,14 +106,12 @@ const SERIALIZABLE_PAYLOADS: Record = { message: 2, runId: 3, stage: 4, - queued: 5, - retryable: 6, - status: 7, + retryable: 5, + status: 6, }, 'Workflow run "wrun_queued" was queued, but start-hook admission could not be confirmed.', 'wrun_queued', 'admission', - true, false, 403, ], diff --git a/packages/world-local/src/index.ts b/packages/world-local/src/index.ts index d9b4e2630a..79eebfeaae 100644 --- a/packages/world-local/src/index.ts +++ b/packages/world-local/src/index.ts @@ -18,8 +18,8 @@ import { initDataDir } from './init.js'; import { instrumentObject } from './instrumentObject.js'; import { createQueue, type DirectHandler } from './queue.js'; import { - hashToken, hookRecoveryMarkerPath, + hookTokenClaimPath, readHookTokenClaim, } from './storage/helpers.js'; import { createStorage } from './storage.js'; @@ -133,9 +133,7 @@ export function createLocalWorld(args?: Partial): LocalWorld { HookSchema ); if (hook?.token) { - await deleteJSON( - path.join(hooksDir, 'tokens', `${hashToken(hook.token)}.json`) - ); + await deleteJSON(hookTokenClaimPath(basedir, hook.token)); await deleteJSON( hookRecoveryMarkerPath( basedir, diff --git a/packages/world-local/src/storage/events-storage.ts b/packages/world-local/src/storage/events-storage.ts index 6a108c430b..fa14ef07d1 100644 --- a/packages/world-local/src/storage/events-storage.ts +++ b/packages/world-local/src/storage/events-storage.ts @@ -13,6 +13,7 @@ import { import type { Event, EventResult, + ExperimentalStartHook, Hook, SerializedData, Step, @@ -61,7 +62,10 @@ import { monotonicUlid, readHookTokenClaim, } from './helpers.js'; -import { deleteAllHooksForRun, retainStartHookClaim } from './hooks-storage.js'; +import { + deleteAllHooksForRun, + settleClaimForDisposedHook, +} from './hooks-storage.js'; import { handleLegacyEvent } from './legacy.js'; import { withRunFileLock } from './runs-storage.js'; @@ -90,6 +94,10 @@ import { withRunFileLock } from './runs-storage.js'; const START_HOOK_LOCK_STALE_MS = 30_000; +/** Whether a run status is terminal (no further lifecycle transitions). */ +const isRunTerminal = (status: string) => + ['completed', 'failed', 'cancelled'].includes(status); + /** * Sidecar recovery marker that pins a canonical `hook_created` * eventId for a legacy token claim — one written by a version of @@ -112,24 +120,25 @@ const HookRecoveryMarkerSchema = z.object({ }); /** - * Runs `fn` under an exclusive cross-process file lock scoped to a hook - * token. Returns `undefined` (without running `fn`) when the lock is - * contended. Locks abandoned by a crashed holder (older than - * START_HOOK_LOCK_STALE_MS) are broken atomically via rename, so two - * breakers can never both enter, and the holder only removes a lock it - * still owns, so a stalled holder cannot free a breaker's lock. + * Runs `fn` under THE exclusive cross-process file lock for a token's claim + * writes — one lock per token, shared by reclaim and materialize, so the two + * read-check-overwrite operations mutually exclude. Returns `undefined` + * (without running `fn`) when the lock is contended; callers treat that as + * a retryable failure and re-validate. Locks abandoned by a crashed holder + * (older than START_HOOK_LOCK_STALE_MS) are broken atomically via rename, + * so two breakers can never both enter, and the holder only removes a lock + * it still owns, so a stalled holder cannot free a breaker's lock. */ async function withTokenClaimLock( basedir: string, token: string, - purpose: 'reclaim' | 'materialize', fn: () => Promise ): Promise { const lockPath = resolveWithinBase( basedir, '.locks', 'hooks', - `${hashToken(token)}.${purpose}` + `${hashToken(token)}.claim` ); const owner = monotonicUlid(); let locked = await writeExclusive(lockPath, owner); @@ -182,32 +191,34 @@ async function canReuseExpiredStartClaim( claim.tag ?? tag ); if (!run) return true; - return ['completed', 'failed', 'cancelled'].includes(run.status); + return isRunTerminal(run.status); } async function replaceExpiredStartClaim( basedir: string, tag: string | undefined, token: string, + currentClaim: HookTokenClaim, nextClaim: HookTokenClaim ): Promise { - const replaced = await withTokenClaimLock( - basedir, - token, - 'reclaim', - async () => { - const constraintPath = hookTokenClaimPath(basedir, token); - const latestClaim = await readHookTokenClaim(constraintPath); - if ( - !latestClaim || - !(await canReuseExpiredStartClaim(basedir, tag, latestClaim)) - ) { - return false; - } - await writeJSON(constraintPath, nextClaim, { overwrite: true }); - return true; + // Cheap pre-check on the caller's snapshot: an unexpired claim can never + // be reclaimed, so skip the lock + re-reads entirely. The in-lock check + // below stays authoritative. + if (!(await canReuseExpiredStartClaim(basedir, tag, currentClaim))) { + return false; + } + const replaced = await withTokenClaimLock(basedir, token, async () => { + const constraintPath = hookTokenClaimPath(basedir, token); + const latestClaim = await readHookTokenClaim(constraintPath); + if ( + !latestClaim || + !(await canReuseExpiredStartClaim(basedir, tag, latestClaim)) + ) { + return false; } - ); + await writeJSON(constraintPath, nextClaim, { overwrite: true }); + return true; + }); return replaced ?? false; } @@ -221,25 +232,20 @@ async function materializeStartHookClaim( token: string, nextClaim: HookTokenClaim ): Promise { - const materialized = await withTokenClaimLock( - basedir, - token, - 'materialize', - async () => { - const constraintPath = hookTokenClaimPath(basedir, token); - const latestClaim = await readHookTokenClaim(constraintPath); - if ( - !latestClaim || - latestClaim.runId !== nextClaim.runId || - latestClaim.hookId !== undefined || - latestClaim.expiresAt !== undefined - ) { - return false; - } - await writeJSON(constraintPath, nextClaim, { overwrite: true }); - return true; + const materialized = await withTokenClaimLock(basedir, token, async () => { + const constraintPath = hookTokenClaimPath(basedir, token); + const latestClaim = await readHookTokenClaim(constraintPath); + if ( + !latestClaim || + latestClaim.runId !== nextClaim.runId || + latestClaim.hookId !== undefined || + latestClaim.expiresAt !== undefined + ) { + return false; } - ); + await writeJSON(constraintPath, nextClaim, { overwrite: true }); + return true; + }); return materialized ?? false; } @@ -256,7 +262,7 @@ async function claimStartHookToken( basedir: string, runId: string, eventId: string, - startHook: { token: string; ttlSeconds: number }, + startHook: ExperimentalStartHook, tag?: string ): Promise { const constraintPath = hookTokenClaimPath(basedir, startHook.token); @@ -280,11 +286,22 @@ async function claimStartHookToken( } if ( existingClaim && - (await replaceExpiredStartClaim(basedir, tag, startHook.token, nextClaim)) + (await replaceExpiredStartClaim( + basedir, + tag, + startHook.token, + existingClaim, + nextClaim + )) ) { return eventId; } + if (existingClaim) { + throw new HookConflictError(startHook.token, existingClaim.runId); + } + // The exclusive write failed but the claim file was gone by the time we + // read it (concurrently reclaimed) — re-read once to converge. const currentClaim = await readHookTokenClaim(constraintPath); if (currentClaim?.runId === runId) { return currentClaim.startEventId ?? eventId; @@ -780,10 +797,6 @@ export function createEventsStorage( // specVersion is always sent by the runtime, but we provide a fallback for safety const effectiveSpecVersion = data.specVersion ?? SPEC_VERSION_CURRENT; - // Helper to check if run is in terminal state - const isRunTerminal = (status: string) => - ['completed', 'failed', 'cancelled'].includes(status); - // Helper to check if step is in terminal state const isStepTerminal = (status: string) => ['completed', 'failed', 'cancelled'].includes(status); @@ -822,10 +835,7 @@ export function createEventsStorage( executionContext?: Record; attributes?: Record; allowReservedAttributes?: true; - experimentalStartHook?: { - token: string; - ttlSeconds: number; - }; + experimentalStartHook?: ExperimentalStartHook; }; if ( runInputData.deploymentId && @@ -1186,10 +1196,7 @@ export function createEventsStorage( executionContext?: Record; attributes?: Record; allowReservedAttributes?: true; - experimentalStartHook?: { - token: string; - ttlSeconds: number; - }; + experimentalStartHook?: ExperimentalStartHook; }; validateAttributeChanges( Object.entries(runData.attributes ?? {}).map(([key, value]) => ({ @@ -1902,12 +1909,18 @@ export function createEventsStorage( if ( !tokenClaimed && existingClaim?.expiresAt !== undefined && - (await replaceExpiredStartClaim(basedir, tag, hookData.token, { - token: hookData.token, - hookId: data.correlationId, - runId: effectiveRunId, - eventId, - })) + (await replaceExpiredStartClaim( + basedir, + tag, + hookData.token, + existingClaim, + { + token: hookData.token, + hookId: data.correlationId, + runId: effectiveRunId, + eventId, + } + )) ) { tokenClaimed = true; } @@ -2132,28 +2145,10 @@ export function createEventsStorage( tag ); if (existingHook) { - // Normal hook disposal frees the token. Start-hook claims carry + // Normal hook disposal frees the token; start-hook claims carry // ttlSeconds and are retained so duplicate starts remain fenced // after the active hook entity is gone. - const disposedConstraintPath = hookTokenClaimPath( - basedir, - existingHook.token - ); - const claim = await readHookTokenClaim(disposedConstraintPath); - if (claim?.runId === existingHook.runId && claim.ttlSeconds) { - await retainStartHookClaim( - disposedConstraintPath, - { - ...claim, - token: existingHook.token, - hookId: existingHook.hookId, - }, - now, - existingHook.createdAt - ); - } else { - await deleteJSON(disposedConstraintPath); - } + await settleClaimForDisposedHook(basedir, existingHook, now); await deleteJSON( hookRecoveryMarkerPath( basedir, diff --git a/packages/world-local/src/storage/hooks-storage.ts b/packages/world-local/src/storage/hooks-storage.ts index e14fd939a3..0c84bbc81a 100644 --- a/packages/world-local/src/storage/hooks-storage.ts +++ b/packages/world-local/src/storage/hooks-storage.ts @@ -21,8 +21,8 @@ import { import { filterHookData } from './filters.js'; import { type HookTokenClaim, - hashToken, hookRecoveryMarkerPath, + hookTokenClaimPath, readHookTokenClaim, } from './helpers.js'; @@ -49,21 +49,33 @@ export async function retainStartHookClaim( ) ); - await writeJSON( - constraintPath, - { - token: claim.token, - hookId: claim.hookId, - runId: claim.runId, - eventId: claim.eventId, - ttlSeconds: claim.ttlSeconds, - startEventId: claim.startEventId, - createdAt: claim.createdAt, - tag: claim.tag, - expiresAt, - }, - { overwrite: true } - ); + await writeJSON(constraintPath, { ...claim, expiresAt }, { overwrite: true }); +} + +/** + * Settles a disposed hook's token claim: TTL-carrying claims are retained + * (duplicate starts stay fenced past disposal), plain createHook guards are + * deleted to free the token. Shared by hook_disposed and terminal-run + * cleanup. + */ +export async function settleClaimForDisposedHook( + basedir: string, + hook: Pick, + now: Date +): Promise { + const constraintPath = hookTokenClaimPath(basedir, hook.token); + const claim = await readHookTokenClaim(constraintPath); + if (claim?.runId === hook.runId && claim.ttlSeconds) { + await retainStartHookClaim( + constraintPath, + { ...claim, token: hook.token }, + now, + hook.createdAt + ); + } else { + await deleteJSON(constraintPath); + } + return constraintPath; } /** @@ -175,32 +187,19 @@ export async function deleteAllHooksForRun( const tokensDir = path.join(hooksDir, 'tokens'); const now = new Date(); + // Settle claims through the run's hook entities (retain TTL claims, free + // plain guards), and delete the hooks + recovery markers. The marker's + // filename hash includes `(token, runId, hookId)` so a leaked marker can + // never corrupt a different lifetime — cleaning it up here keeps the + // tokens/ directory from accumulating recovered-hook sidecars over time. + const settledClaimPaths = new Set(); for (const file of files) { const hookPath = path.join(hooksDir, `${file}.json`); const hook = await readJSON(hookPath, HookSchema); if (hook && hook.runId === runId) { - // Delete the token constraint file to free up the token, and - // delete the recovery marker (if any) for disk hygiene. The - // marker's filename hash includes `(token, runId, hookId)` so - // a leaked marker can never corrupt a different lifetime — but - // cleaning it up here keeps the tokens/ directory from - // accumulating recovered-hook sidecars over time. - const constraintPath = path.join( - hooksDir, - 'tokens', - `${hashToken(hook.token)}.json` + settledClaimPaths.add( + await settleClaimForDisposedHook(basedir, hook, now) ); - const claim = await readHookTokenClaim(constraintPath); - if (claim?.runId === runId && claim.ttlSeconds) { - await retainStartHookClaim( - constraintPath, - { ...claim, token: hook.token }, - now, - hook.createdAt - ); - } else { - await deleteJSON(constraintPath); - } await deleteJSON( hookRecoveryMarkerPath(basedir, hook.token, hook.runId, hook.hookId) ); @@ -208,19 +207,33 @@ export async function deleteAllHooksForRun( } } - for (const file of await listJSONFiles(tokensDir)) { - if (file.endsWith('.recovery')) continue; - const constraintPath = path.join(tokensDir, `${file}.json`); - const claim = await readHookTokenClaim(constraintPath); - if (claim?.runId !== runId || !claim.ttlSeconds) continue; - if ( - opts?.releaseUnmaterializedClaims && - claim.hookId === undefined && - claim.expiresAt === undefined - ) { - await deleteJSON(constraintPath); - } else { - await retainStartHookClaim(constraintPath, claim, now); - } - } + // Sweep the tokens directory for claims with no hook entity: this run's + // unmaterialized start claims (retain or, on cancellation, release), plus + // opportunistic cleanup of anyone's expired retained-claim debris so the + // directory doesn't grow forever. + const tokenFiles = await listJSONFiles(tokensDir); + await Promise.all( + tokenFiles.map(async (file) => { + if (file.endsWith('.recovery')) return; + const constraintPath = path.join(tokensDir, `${file}.json`); + if (settledClaimPaths.has(constraintPath)) return; + const claim = await readHookTokenClaim(constraintPath); + if (!claim) return; + if (claim.expiresAt && claim.expiresAt <= now) { + // Retention window over — the claim is dead for every run. + await deleteJSON(constraintPath); + return; + } + if (claim.runId !== runId || !claim.ttlSeconds) return; + if ( + opts?.releaseUnmaterializedClaims && + claim.hookId === undefined && + claim.expiresAt === undefined + ) { + await deleteJSON(constraintPath); + } else { + await retainStartHookClaim(constraintPath, claim, now); + } + }) + ); } diff --git a/packages/world-postgres/src/storage.ts b/packages/world-postgres/src/storage.ts index fca995e0fd..3dbfdd0858 100644 --- a/packages/world-postgres/src/storage.ts +++ b/packages/world-postgres/src/storage.ts @@ -13,6 +13,7 @@ import type { Event, EventResult, ExperimentalSetAttributesResult, + ExperimentalStartHook, GetEventParams, Hook, ListEventsParams, @@ -50,6 +51,7 @@ import { isNull, lt, notInArray, + or, sql, } from 'drizzle-orm'; import { monotonicFactory } from 'ulid'; @@ -340,6 +342,10 @@ async function handleLegacyEventPostgres( export function createEventsStorage(drizzle: Drizzle): Storage['events'] { const ulid = monotonicFactory(); + + /** Whether a run status is terminal (no further lifecycle transitions). */ + const isRunTerminal = (status: string) => + ['completed', 'failed', 'cancelled'].includes(status); const { events } = Schema; // Prepared statements for validation queries (performance optimization) @@ -444,7 +450,7 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { .from(Schema.runs) .where(eq(Schema.runs.runId, claim.runId)) .limit(1); - if (run && !['completed', 'failed', 'cancelled'].includes(run.status)) { + if (run && !isRunTerminal(run.status)) { return claim; } @@ -490,46 +496,116 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { completedAt: Date, opts?: { releaseUnmaterializedClaims?: boolean } ): Promise { - await drizzle.transaction(async (tx) => { - // Plain createHook token guards (ttlSeconds <= 0) are released with - // the run. Cancellation additionally releases start-hook claims the - // workflow never materialized into a hook: a cancelled run did no - // fenced work, and cancel-then-retry (including start()'s own cleanup - // when queueing fails after admission) must be able to reuse the - // token immediately. - await tx - .delete(Schema.hookClaims) - .where( - and( - eq(Schema.hookClaims.runId, runId), - opts?.releaseUnmaterializedClaims - ? sql`(${Schema.hookClaims.ttlSeconds} <= 0 OR ${Schema.hookClaims.hookId} IS NULL)` - : sql`${Schema.hookClaims.ttlSeconds} <= 0` - ) - ); - // Retain the rest until at least `hookCreatedAt + ttl` (falling back - // to the claim's own createdAt when no hook entity exists), never - // shrinking an existing retention window and never expiring before - // run completion. - await tx - .update(Schema.hookClaims) - .set({ - expiresAt: sql`GREATEST( - COALESCE( - (SELECT h.created_at FROM ${Schema.hooks} h WHERE h.hook_id = ${Schema.hookClaims.hookId}), - ${Schema.hookClaims.createdAt} - ) + make_interval(secs => ${Schema.hookClaims.ttlSeconds}), - COALESCE(${Schema.hookClaims.expiresAt}, ${completedAt}), - ${completedAt} - )`, - }) - .where( - and( - eq(Schema.hookClaims.runId, runId), - gt(Schema.hookClaims.ttlSeconds, 0) - ) - ); - }); + // Two set-based statements (no read-modify-write a concurrent + // materialize could interleave with, and no transaction — they touch + // disjoint rows and are each independently atomic). + // + // 1. Release plain createHook token guards (ttlSeconds <= 0) with the + // run. Cancellation additionally releases start-hook claims the + // workflow never materialized into a hook: a cancelled run did no + // fenced work, and cancel-then-retry (including start()'s own + // cleanup when queueing fails after admission) must be able to reuse + // the token immediately. Piggyback deletion of anyone's expired + // retained-claim debris (index on expires_at) so the table does not + // grow forever — expired claims are only otherwise removed lazily on + // same-token reuse. + await drizzle.delete(Schema.hookClaims).where( + or( + and( + eq(Schema.hookClaims.runId, runId), + opts?.releaseUnmaterializedClaims + ? sql`(${Schema.hookClaims.ttlSeconds} <= 0 OR ${Schema.hookClaims.hookId} IS NULL)` + : sql`${Schema.hookClaims.ttlSeconds} <= 0` + ), + and( + lt(Schema.hookClaims.expiresAt, completedAt), + // Expired claims still fence their token while the owning run + // is active (getLiveHookClaim's liveness rule) — only sweep + // debris whose run is terminal or gone. + sql`NOT EXISTS ( + SELECT 1 FROM ${Schema.runs} r + WHERE r.id = ${Schema.hookClaims.runId} + AND r.status NOT IN ('completed', 'failed', 'cancelled') + )` + ) + ) + ); + // 2. Retain the rest until at least `hookCreatedAt + ttl` (falling back + // to the claim's own createdAt when no hook entity exists), never + // shrinking an existing retention window and never expiring before + // run completion. + await drizzle + .update(Schema.hookClaims) + .set({ + expiresAt: sql`GREATEST( + COALESCE( + (SELECT h.created_at FROM ${Schema.hooks} h WHERE h.hook_id = ${Schema.hookClaims.hookId}), + ${Schema.hookClaims.createdAt} + ) + make_interval(secs => ${Schema.hookClaims.ttlSeconds}), + COALESCE(${Schema.hookClaims.expiresAt}, ${completedAt}), + ${completedAt} + )`, + }) + .where( + and( + eq(Schema.hookClaims.runId, runId), + gt(Schema.hookClaims.ttlSeconds, 0) + ) + ); + } + + /** + * Pre-transaction ownership snapshot for a start-hook token: rejects + * tokens held by a hook entity or a live claim of ANOTHER run, and + * returns the same-run claim (if any) so callers can apply their + * duplicate-delivery semantics. getLiveHookClaim lazily reclaims dead + * claims here so the transactional insert can succeed afterwards. + */ + async function getOwnStartHookClaim(token: string, runId: string) { + const [[existingHook], existingClaim] = await Promise.all([ + getHookByToken.execute({ token }), + getLiveHookClaim(token), + ]); + if (existingHook && existingHook.runId !== runId) { + throw new HookConflictError(token, existingHook.runId); + } + if (existingClaim && existingClaim.runId !== runId) { + throw new HookConflictError(token, existingClaim.runId); + } + return existingClaim; + } + + /** + * Takes the token claim for `runId` inside `tx`. Tolerates a concurrent + * same-run claim, and a claim that vanished between the insert conflict + * and the re-read (concurrent reclaim) — in both cases the caller's run + * insert arbitrates the duplicate. Throws HookConflictError when another + * run owns the token. + */ + async function claimStartHookTokenInTx( + tx: Parameters[0]>[0], + startHook: ExperimentalStartHook, + runId: string + ): Promise { + const [createdClaim] = await tx + .insert(Schema.hookClaims) + .values({ + token: startHook.token, + runId, + ttlSeconds: startHook.ttlSeconds, + }) + .onConflictDoNothing() + .returning({ runId: Schema.hookClaims.runId }); + if (createdClaim) return; + + const [existing] = await tx + .select({ runId: Schema.hookClaims.runId }) + .from(Schema.hookClaims) + .where(eq(Schema.hookClaims.token, startHook.token)) + .limit(1); + if (existing && existing.runId !== runId) { + throw new HookConflictError(startHook.token, existing.runId); + } } return { @@ -570,8 +646,6 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { const now = new Date(); // Helper to check if run is in terminal state - const isRunTerminal = (status: string) => - ['completed', 'failed', 'cancelled'].includes(status); // Helper to check if step is in terminal state const isStepTerminal = (status: string) => @@ -620,10 +694,7 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { executionContext?: Record; attributes?: Record; allowReservedAttributes?: true; - experimentalStartHook?: { - token: string; - ttlSeconds: number; - }; + experimentalStartHook?: ExperimentalStartHook; }; if ( runInputData.deploymentId && @@ -643,28 +714,9 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { const workflowName = runInputData.workflowName; const input = runInputData.input; const startHook = runInputData.experimentalStartHook; - let hasSameRunStartClaim = false; - if (startHook) { - const [existingHook] = await getHookByToken.execute({ - token: startHook.token, - }); - if (existingHook && existingHook.runId !== effectiveRunId) { - throw new HookConflictError( - startHook.token, - existingHook.runId - ); - } - const existingClaim = await getLiveHookClaim(startHook.token); - if (existingClaim) { - if (existingClaim.runId !== effectiveRunId) { - throw new HookConflictError( - startHook.token, - existingClaim.runId - ); - } - hasSameRunStartClaim = true; - } - } + const sameRunStartClaim = startHook + ? await getOwnStartHookClaim(startHook.token, effectiveRunId) + : null; // Create run + optional start claim + run_created event // atomically. The transaction ensures we never have an // orphaned run without its run_created event or start claim. @@ -687,29 +739,8 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { .returning(); if (!createdRun) return undefined; - if (startHook && !hasSameRunStartClaim) { - const [createdClaim] = await tx - .insert(Schema.hookClaims) - .values({ - token: startHook.token, - runId: effectiveRunId, - ttlSeconds: startHook.ttlSeconds, - }) - .onConflictDoNothing() - .returning({ token: Schema.hookClaims.token }); - if (!createdClaim) { - const [existing] = await tx - .select({ runId: Schema.hookClaims.runId }) - .from(Schema.hookClaims) - .where(eq(Schema.hookClaims.token, startHook.token)) - .limit(1); - if (existing?.runId !== effectiveRunId) { - throw new HookConflictError( - startHook.token, - existing?.runId - ); - } - } + if (startHook && !sameRunStartClaim) { + await claimStartHookTokenInTx(tx, startHook, effectiveRunId); } const runCreatedEventId = `wevt_${ulid()}`; @@ -972,10 +1003,7 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { executionContext?: Record; attributes?: Record; allowReservedAttributes?: true; - experimentalStartHook?: { - token: string; - ttlSeconds: number; - }; + experimentalStartHook?: ExperimentalStartHook; }; validateAttributeChanges( Object.entries(eventData.attributes ?? {}).map(([key, value]) => ({ @@ -1002,59 +1030,20 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { const startHook = eventData.experimentalStartHook; const runValues = startHook ? await (async () => { - // Pre-checks outside the transaction: reject tokens held by a - // legacy hook entity without a claim row, and let - // getLiveHookClaim lazily reclaim dead claims so the insert - // below can succeed. Claim + run + event are committed in one - // transaction, so a live same-run claim means this exact - // run_created was already admitted — a duplicate delivery. - const [existingHook] = await getHookByToken.execute({ - token: startHook.token, - }); - if (existingHook && existingHook.runId !== effectiveRunId) { - throw new HookConflictError( - startHook.token, - existingHook.runId - ); - } - const existingClaim = await getLiveHookClaim(startHook.token); - if (existingClaim && existingClaim.runId !== effectiveRunId) { - throw new HookConflictError( - startHook.token, - existingClaim.runId - ); - } - if (existingClaim) { + // Claim + run + event are committed in one transaction, so a + // live same-run claim means this exact run_created was + // already admitted — a duplicate delivery. + const sameRunStartClaim = await getOwnStartHookClaim( + startHook.token, + effectiveRunId + ); + if (sameRunStartClaim) { throw new EntityConflictError( `Run "${effectiveRunId}" already exists` ); } const result = await drizzle.transaction(async (tx) => { - const [createdClaim] = await tx - .insert(Schema.hookClaims) - .values({ - token: startHook.token, - runId: effectiveRunId, - ttlSeconds: startHook.ttlSeconds, - }) - .onConflictDoNothing() - .returning({ runId: Schema.hookClaims.runId }); - if (!createdClaim) { - const [existing] = await tx - .select({ runId: Schema.hookClaims.runId }) - .from(Schema.hookClaims) - .where(eq(Schema.hookClaims.token, startHook.token)) - .limit(1); - // A same-run claim raced in (duplicate delivery of this - // run_created): fall through — the run insert below - // conflicts, yielding EntityConflictError. - if (existing && existing.runId !== effectiveRunId) { - throw new HookConflictError( - startHook.token, - existing.runId - ); - } - } + await claimStartHookTokenInTx(tx, startHook, effectiveRunId); const insertedRuns = await tx .insert(Schema.runs) @@ -1703,10 +1692,10 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { // Snapshot current token ownership. getLiveHookClaim lazily // reclaims dead claims, so a `null` claim means the token is free. - const [existingHook] = await getHookByToken.execute({ - token: eventData.token, - }); - const existingClaim = await getLiveHookClaim(eventData.token); + const [[existingHook], existingClaim] = await Promise.all([ + getHookByToken.execute({ token: eventData.token }), + getLiveHookClaim(eventData.token), + ]); const isOwnHookRow = (row: typeof existingHook) => row?.runId === effectiveRunId && row.hookId === data.correlationId; @@ -1838,10 +1827,10 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { // and the transaction. Re-read and converge: our own // (runId, hookId) winning concurrently is a duplicate/orphan; // anything else is a conflict. - const [racedHook] = await getHookByToken.execute({ - token: eventData.token, - }); - const racedClaim = await getLiveHookClaim(eventData.token); + const [[racedHook], racedClaim] = await Promise.all([ + getHookByToken.execute({ token: eventData.token }), + getLiveHookClaim(eventData.token), + ]); if (isOwnHookRow(racedHook)) { await adoptOwnHookRow(); } else { @@ -1873,44 +1862,39 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { if (!disposed) { throw new EntityConflictError(`Hook "${hookId}" already disposed`); } - const [claim] = await tx - .select({ - runId: Schema.hookClaims.runId, - ttlSeconds: Schema.hookClaims.ttlSeconds, - expiresAt: Schema.hookClaims.expiresAt, + // Settle the claim with two guarded statements instead of a + // SELECT-then-branch: release plain guards (no retention TTL), + // retain TTL-carrying claims until at least hookCreatedAt + ttl — + // never shrinking an existing window and never expiring before + // now. Both guard on the disposed hook's run so a claim owned by + // another run is left alone. For plain hooks (the common case) + // the UPDATE is a no-op index probe. + await tx + .delete(Schema.hookClaims) + .where( + and( + eq(Schema.hookClaims.token, disposed.token), + eq(Schema.hookClaims.runId, disposed.runId), + sql`${Schema.hookClaims.ttlSeconds} <= 0` + ) + ); + await tx + .update(Schema.hookClaims) + .set({ + hookId, + expiresAt: sql`GREATEST( + ${disposed.createdAt}::timestamp + make_interval(secs => ${Schema.hookClaims.ttlSeconds}), + COALESCE(${Schema.hookClaims.expiresAt}, ${now}::timestamp), + ${now}::timestamp + )`, }) - .from(Schema.hookClaims) - .where(eq(Schema.hookClaims.token, disposed.token)) - .limit(1); - // Only touch the claim if the disposed hook's run owns it. - if (!claim || claim.runId !== disposed.runId) return; - if (claim.ttlSeconds <= 0) { - await tx - .delete(Schema.hookClaims) - .where( - and( - eq(Schema.hookClaims.token, disposed.token), - eq(Schema.hookClaims.runId, disposed.runId) - ) - ); - } else { - const expiresAt = new Date( - Math.max( - disposed.createdAt.getTime() + claim.ttlSeconds * 1000, - claim.expiresAt?.getTime() ?? 0, - now.getTime() + .where( + and( + eq(Schema.hookClaims.token, disposed.token), + eq(Schema.hookClaims.runId, disposed.runId), + gt(Schema.hookClaims.ttlSeconds, 0) ) ); - await tx - .update(Schema.hookClaims) - .set({ expiresAt, hookId }) - .where( - and( - eq(Schema.hookClaims.token, disposed.token), - eq(Schema.hookClaims.runId, disposed.runId) - ) - ); - } }); } diff --git a/packages/world-vercel/src/events-v4.ts b/packages/world-vercel/src/events-v4.ts index a8a28ffe77..5d8aa4b071 100644 --- a/packages/world-vercel/src/events-v4.ts +++ b/packages/world-vercel/src/events-v4.ts @@ -21,7 +21,7 @@ * bytes — this module stays at the wire-bytes layer. */ -import { HookConflictError } from '@workflow/errors'; +import type { ExperimentalStartHook } from '@workflow/world'; import { decode } from 'cbor-x'; import { decodeFrames, encodeFrame, V4_FRAME_CONTENT_TYPE } from './frames.js'; import { getEventsDispatcher } from './http-client.js'; @@ -139,7 +139,7 @@ export interface CreateEventV4Input { attributes?: Record; /** Experimental start-hook token claim carried by run_created and the * resilient-start run_started fallback. */ - experimentalStartHook?: { token: string; ttlSeconds: number }; + experimentalStartHook?: ExperimentalStartHook; /** attr_set's attribute change list ({key, value|null} entries). */ changes?: Array>; /** attr_set's writer provenance ({type:'workflow'} or @@ -274,12 +274,6 @@ function errorFromV4Response( if (errorBody) message += ` ${errorBody}`; } - // A 409 carrying the server's hook_conflict code is a start-hook token - // conflict, not a generic entity conflict — surface it as the typed error - // the runtime and start() branch on. - if (statusCode === 409 && code === 'hook_conflict') { - return new HookConflictError(token ?? '', conflictingRunId); - } const retryAfter = parseRetryAfter( readHeader(responseHeaders, 'retry-after') ); @@ -289,6 +283,8 @@ function errorFromV4Response( retryAfter, code, url, + token, + conflictingRunId, mitigated: readHeader(responseHeaders, 'x-vercel-mitigated'), }); } diff --git a/packages/world-vercel/src/events.ts b/packages/world-vercel/src/events.ts index 8dbc72b9cb..6b082befac 100644 --- a/packages/world-vercel/src/events.ts +++ b/packages/world-vercel/src/events.ts @@ -40,6 +40,7 @@ import { type EventResult, EventSchema, EventTypeSchema, + type ExperimentalStartHook, type GetEventParams, type ListEventsByCorrelationIdParams, type ListEventsParams, @@ -185,7 +186,7 @@ interface SplitEventData { /** Initial run attributes (run_created / resilient-start run_started). */ attributes?: Record; /** Experimental start-hook token claim (run_created / resilient start). */ - experimentalStartHook?: { token: string; ttlSeconds: number }; + experimentalStartHook?: ExperimentalStartHook; /** attr_set change list, included verbatim in frame meta. */ changes?: Array>; /** attr_set writer provenance, included verbatim in frame meta. */ @@ -339,10 +340,8 @@ export function splitEventDataForV4(data: AnyEventRequest): SplitEventData { eventData.experimentalStartHook !== null && typeof eventData.experimentalStartHook === 'object' ) { - meta.experimentalStartHook = eventData.experimentalStartHook as { - token: string; - ttlSeconds: number; - }; + meta.experimentalStartHook = + eventData.experimentalStartHook as ExperimentalStartHook; } if (Array.isArray(eventData.changes)) { meta.changes = eventData.changes as Array>; diff --git a/packages/world-vercel/src/http-core.ts b/packages/world-vercel/src/http-core.ts index 18c5dff6e6..4e535372b9 100644 --- a/packages/world-vercel/src/http-core.ts +++ b/packages/world-vercel/src/http-core.ts @@ -19,6 +19,7 @@ import { getVercelOidcToken } from '@vercel/oidc'; import { EntityConflictError, + HookConflictError, RunExpiredError, ThrottleError, TooEarlyError, @@ -165,9 +166,19 @@ export function errorForResponse( code?: string; url?: string; mitigated?: string | null; + /** Hook token from a 409 `hook_conflict` response body. */ + token?: string; + /** Owning run from a 409 `hook_conflict` response body. */ + conflictingRunId?: string; } = {} ): Error { const { retryAfter, code, url, mitigated } = opts; + // A 409 carrying the server's hook_conflict code is a start-hook token + // conflict, not a generic entity conflict — surface it as the typed error + // the runtime and start() branch on. + if (status === 409 && code === 'hook_conflict') { + return new HookConflictError(opts.token ?? '', opts.conflictingRunId); + } if (status === 409) return new EntityConflictError(message); if (status === 410) return new RunExpiredError(message); if (status === 425) return new TooEarlyError(message, { retryAfter }); diff --git a/packages/world-vercel/src/utils.ts b/packages/world-vercel/src/utils.ts index aaaeee0cdf..5134f3fc74 100644 --- a/packages/world-vercel/src/utils.ts +++ b/packages/world-vercel/src/utils.ts @@ -412,13 +412,16 @@ export async function makeRequest({ }); if (!response.ok) { - const errorData: { message?: string; code?: string; error?: string } = - await parseResponseBody(response) - .then( - (r) => - r.data as { message?: string; code?: string; error?: string } - ) - .catch(() => ({})); + type ErrorBody = { + message?: string; + code?: string; + error?: string; + token?: string; + conflictingRunId?: string; + }; + const errorData: ErrorBody = await parseResponseBody(response) + .then((r) => r.data as ErrorBody) + .catch(() => ({})); const errorCode = errorData.code ?? errorData.error; logCurlRepro(request.method, url, headers); @@ -440,6 +443,8 @@ export async function makeRequest({ url, code: errorCode, retryAfter, + token: errorData.token, + conflictingRunId: errorData.conflictingRunId, mitigated: response.headers.get('x-vercel-mitigated'), }); span?.setAttributes({ diff --git a/packages/world/src/events.ts b/packages/world/src/events.ts index facd200e92..2fc33d0b7c 100644 --- a/packages/world/src/events.ts +++ b/packages/world/src/events.ts @@ -276,11 +276,20 @@ const AttrSetEventSchema = BaseEventSchema.extend({ }), }); -const ExperimentalStartHookSchema = z.object({ +/** + * Start-hook claim data: `start()` reserves `token` atomically with run + * admission, fencing duplicate starts for `ttlSeconds` past hook creation / + * run completion. Rides on `run_created` (and the resilient-start + * `run_started` fallback + queue runInput) so every admission path validates + * the same shape. + */ +export const ExperimentalStartHookSchema = z.object({ token: z.string().min(1), ttlSeconds: z.number().int().positive(), }); +export type ExperimentalStartHook = z.infer; + // ============================================================================= // Run lifecycle events // ============================================================================= diff --git a/packages/world/src/index.ts b/packages/world/src/index.ts index 8052636320..2e8189477d 100644 --- a/packages/world/src/index.ts +++ b/packages/world/src/index.ts @@ -27,6 +27,7 @@ export { EVENT_DATA_REF_FIELDS, EventSchema, EventTypeSchema, + ExperimentalStartHookSchema, stripEventDataRefs, } from './events.js'; export type * from './hooks.js'; diff --git a/packages/world/src/queue.ts b/packages/world/src/queue.ts index 2108745270..e0765eff57 100644 --- a/packages/world/src/queue.ts +++ b/packages/world/src/queue.ts @@ -115,10 +115,12 @@ export const RunInputSchema = z.object({ executionContext: z.record(z.string(), z.any()).optional(), /** Initial plaintext run attributes, for resilient run creation. */ attributes: z.record(z.string(), z.string()).optional(), - /** Start-hook claim data, for resilient run creation. */ + /** Start-hook claim data, for resilient run creation. Keep aligned with + * `ExperimentalStartHookSchema` in events.ts (this file uses `zod/v4`, so + * the schema instance cannot be shared). */ experimentalStartHook: z .object({ - token: z.string(), + token: z.string().min(1), ttlSeconds: z.number().int().positive(), }) .optional(),