From bd33c1071a94097f59bad046fd2edf87da768b78 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Thu, 9 Jul 2026 14:51:46 -0700 Subject: [PATCH 01/36] feat(core): add hook token retention contract --- .changeset/retained-hook-contract.md | 7 ++ .../v5/api-reference/workflow/create-hook.mdx | 45 ++++++++++++- docs/content/docs/v5/foundations/hooks.mdx | 3 +- .../docs/v5/foundations/idempotency.mdx | 4 +- packages/core/src/create-hook.ts | 41 ++++++++++-- packages/core/src/global.ts | 2 + .../src/runtime/suspension-handler.test.ts | 37 ++++++++++ .../core/src/runtime/suspension-handler.ts | 1 + packages/core/src/workflow/hook.test.ts | 67 +++++++++++++++++++ packages/core/src/workflow/hook.ts | 28 +++++++- packages/world-vercel/src/events-v4.ts | 5 ++ packages/world-vercel/src/events.test.ts | 17 +++++ packages/world-vercel/src/events.ts | 5 ++ packages/world/src/events.test.ts | 21 ++++++ packages/world/src/events.ts | 1 + 15 files changed, 271 insertions(+), 13 deletions(-) create mode 100644 .changeset/retained-hook-contract.md diff --git a/.changeset/retained-hook-contract.md b/.changeset/retained-hook-contract.md new file mode 100644 index 0000000000..80f5674a48 --- /dev/null +++ b/.changeset/retained-hook-contract.md @@ -0,0 +1,7 @@ +--- +'@workflow/core': minor +'@workflow/world': minor +'@workflow/world-vercel': minor +--- + +Add the experimental hook-token retention contract and forward its absolute reuse deadline through workflow events. diff --git a/docs/content/docs/v5/api-reference/workflow/create-hook.mdx b/docs/content/docs/v5/api-reference/workflow/create-hook.mdx index ec391a722b..687c34d900 100644 --- a/docs/content/docs/v5/api-reference/workflow/create-hook.mdx +++ b/docs/content/docs/v5/api-reference/workflow/create-hook.mdx @@ -66,7 +66,7 @@ export default Hook;`} The returned `Hook` object also implements `AsyncIterable`, which allows you to iterate over incoming payloads using `for await...of` syntax. -Use `hook.getConflict()` to check whether the hook token is already claimed by another active hook, without waiting for hook payload data. Calling `createHook()` on its own does not register the hook — registration is only committed when the workflow suspends. Awaiting `hook.getConflict()` suspends the workflow to commit the registration, then resolves with `null` once `hook_created` is recorded, or with the conflicting [`Run`](/docs/api-reference/workflow-api/get-run) if another active hook already owns the same token. +Use `hook.getConflict()` to check whether the hook token is already claimed by another hook or a retained token claim, without waiting for hook payload data. Calling `createHook()` on its own does not register the hook — registration is only committed when the workflow suspends. Awaiting `hook.getConflict()` suspends the workflow to commit the registration, then resolves with `null` once `hook_created` is recorded, or with the conflicting [`Run`](/docs/api-reference/workflow-api/get-run) if another run already owns the same token. ## Examples @@ -143,12 +143,53 @@ async function processOrder(orderId: string) { Because `createHook()` alone does not suspend the workflow, awaiting `hook.getConflict()` is what actually suspends the run and commits the hook registration. It only waits for registration — to receive payload data from a future `resumeHook()` call, await the hook itself or iterate it with `for await...of`. -On a conflict, the resolved value is a `Run` handle for the run that currently owns the token, with durable step-backed accessors. The duplicate run can decide in code how to handle it: return or log `conflict.runId`, inspect `await conflict.status`, wait on `await conflict.returnValue`, or cancel the owner with `await conflict.cancel()` and continue in the current run. See [Run idempotency](/docs/foundations/idempotency#run-idempotency) for these strategies in context. +On a conflict, the resolved value is a `Run` handle for the run that owns the token, with durable step-backed accessors. With retention configured, that run may already be terminal. The duplicate run can decide in code how to handle it: return or log `conflict.runId`, inspect `await conflict.status`, wait on `await conflict.returnValue`, or cancel an active owner with `await conflict.cancel()`. See [Run idempotency](/docs/foundations/idempotency#run-idempotency) for these strategies in context. Custom hook tokens are the recommended way to coordinate active workflow runs. Use a deterministic token from your domain, such as an order ID or conversation ID, create the hook near the beginning of the workflow, and check `await hook.getConflict()` before work that depends on owning the token. See [Run idempotency](/docs/foundations/idempotency#run-idempotency). +### Retaining a Token Claim After Completion + +By default, a hook token becomes reusable when its workflow run completes, fails, or is cancelled. Set `experimental_retention` when a late duplicate must still conflict after the owner reaches a terminal state: + +```typescript lineNumbers +import { createHook } from "workflow"; + +declare function processOwnedOrder(orderId: string): Promise; // @setup + +export async function processOrder(orderId: string) { + "use workflow"; + + // Do not use `using` for a retained claim. `using` calls dispose(), which + // explicitly releases the token. + const ownership = createHook({ // [!code highlight] + token: `order:${orderId}`, // [!code highlight] + experimental_retention: "30d", // [!code highlight] + }); // [!code highlight] + + const conflict = await ownership.getConflict(); + if (conflict) { + return { status: "duplicate" as const, runId: conflict.runId }; + } + + await processOwnedOrder(orderId); + return { status: "processed" as const }; +} +``` + +`experimental_retention` accepts the same values as [`sleep()`](/docs/api-reference/workflow/sleep): a duration string such as `"30d"`, a number of milliseconds, or an absolute `Date`. A relative duration starts when `createHook()` is evaluated, not when the run ends. The token remains claimed while the owner is active even if that deadline passes, and after a terminal outcome only until the deadline passes. In other words, the earliest reuse time is the later of the run becoming terminal and the configured deadline. + +Retention preserves only the token claim. The Hook itself is disposed when the run ends and cannot receive more payloads. A conflicting run can still use `getConflict()` to identify the run that owns the retained claim. + +Calling `hook.dispose()` always releases the token immediately, including when retention is configured. The `using` keyword calls `dispose()` when the scope exits, so leave an ownership Hook undisposed when you want terminal cleanup to retain its claim. This also provides an explicit way to opt out of retention on an application-controlled failure path. + + +This option is experimental and requires support from the configured World. Worlds that do not implement retained token claims ignore it, so it must not be the only idempotency mechanism for a deployment whose World support is unknown. `createWebhook()` does not accept this option. + + +The configured time is an earliest reuse deadline, not a precise timer. Scheduling, network latency, and backend cleanup can make the claim reusable slightly later, so very small millisecond values are not suitable for precise timing. A World may also impose a maximum retention window and reject a deadline beyond that limit. + ### Waiting for Multiple Payloads You can also wait for multiple payloads by using the `for await...of` syntax. diff --git a/docs/content/docs/v5/foundations/hooks.mdx b/docs/content/docs/v5/foundations/hooks.mdx index 867b7c6f32..ca89aa4772 100644 --- a/docs/content/docs/v5/foundations/hooks.mdx +++ b/docs/content/docs/v5/foundations/hooks.mdx @@ -112,7 +112,7 @@ export async function orderWorkflow(orderId: string) { } ``` -Calling `createHook()` on its own does not register the hook — registration is only committed when the workflow suspends. Awaiting `hook.getConflict()` suspends the workflow to commit the hook registration, then resolves with `null` once the hook is registered and ready to receive payloads, or with a `Run` handle for the run that owns the token if another active hook already claimed it (see [`HookConflictError`](/docs/errors/hook-conflict)). For `hook_conflict` events persisted by older worlds that did not record the owning run's ID, `getConflict()` rejects with `HookConflictError` instead of resolving with an incomplete handle. The conflicting run's accessors are durable steps, so the workflow can inspect `await conflict.status`, wait on `await conflict.returnValue`, or cancel the owner with `await conflict.cancel()` — see [Run idempotency](/docs/foundations/idempotency#run-idempotency) for these strategies. +Calling `createHook()` on its own does not register the hook — registration is only committed when the workflow suspends. Awaiting `hook.getConflict()` suspends the workflow to commit the hook registration, then resolves with `null` once the hook is registered and ready to receive payloads, or with a `Run` handle for the run that owns the token if another hook or retained claim already owns it (see [`HookConflictError`](/docs/errors/hook-conflict)). For `hook_conflict` events persisted by older worlds that did not record the owning run's ID, `getConflict()` rejects with `HookConflictError` instead of resolving with an incomplete handle. The conflicting run's accessors are durable steps, so the workflow can inspect `await conflict.status`, wait on `await conflict.returnValue`, or cancel an active owner with `await conflict.cancel()` — see [Run idempotency](/docs/foundations/idempotency#run-idempotency) for these strategies. ### Custom Tokens for Deterministic Hooks @@ -487,6 +487,7 @@ When using custom tokens with `createHook()`: - **Make them deterministic**: Base them on data the external system can reconstruct (like channel IDs, user IDs, etc.) - **Use namespacing**: Prefix tokens to avoid conflicts (e.g., `slack:${channelId}`, `github:${repoId}`) - **Include routing information**: Ensure the token contains enough information to identify the correct workflow instance +- **Choose an explicit retention policy for idempotency claims**: By default the token is reusable after the run ends. Use [`experimental_retention`](/docs/api-reference/workflow/create-hook#retaining-a-token-claim-after-completion) when late duplicates must still conflict, and do not automatically dispose that ownership hook. ### Response Handling in Webhooks diff --git a/docs/content/docs/v5/foundations/idempotency.mdx b/docs/content/docs/v5/foundations/idempotency.mdx index 190b5c9f81..6d445aef9c 100644 --- a/docs/content/docs/v5/foundations/idempotency.mdx +++ b/docs/content/docs/v5/foundations/idempotency.mdx @@ -149,7 +149,9 @@ export async function POST(request: Request) { This avoids creating a new run only after the first run has registered its hook. Because `start()` returns before the run body executes and calls `createHook()`, two concurrent requests can both observe "no hook yet" and each call `start()`. The race is resolved inside the workflow body, where the losing run observes `getConflict()` resolving with the active owner and returns without doing duplicate-sensitive work — and the route detects it by comparing the resumed hook's `runId` against the run it just started, without waiting for either run to finish. A native API for atomically starting a run and registering a hook is in the works. Until then, model recovery inside the workflow by checking `hook.getConflict()`. -This is active-run coordination. When the workflow completes and disposes the hook, the token can be used again. If a duplicate request after completion must return the original result instead of starting fresh work, persist that completed result under the same domain key. +By default, this is active-run coordination: when the workflow reaches a terminal state, the token can be used again. To reject late duplicates for a bounded window, set `experimental_retention` and leave the ownership Hook undisposed. The live Hook still disappears when the run ends; only its token claim remains. See [`createHook()` token retention](/docs/api-reference/workflow/create-hook#retaining-a-token-claim-after-completion) for the API, accepted deadline formats, and World support limitations. + +Retained claims reject duplicates but do not store or return the original workflow result by themselves. A duplicate receives the conflicting `Run` and can inspect its status or return value while that run remains available. If the result must outlive workflow-run retention, persist it under the same domain key in application storage. ### Conflict-handling strategies diff --git a/packages/core/src/create-hook.ts b/packages/core/src/create-hook.ts index c44daafbf2..6847784ba6 100644 --- a/packages/core/src/create-hook.ts +++ b/packages/core/src/create-hook.ts @@ -1,3 +1,4 @@ +import type { StringValue } from 'ms'; import { throwNotInWorkflowContext } from './context-errors.js'; import type { Run } from './runtime/run.js'; import type { Serializable } from './schemas.js'; @@ -32,8 +33,8 @@ export interface Hook extends AsyncIterable, Thenable { /** * Returns a promise that resolves with the conflicting {@link Run} if - * another active hook already owns this hook's token, or `null` once - * the hook has been registered and is ready to receive payloads. + * another hook or retained claim already owns this hook's token, or `null` + * once the hook has been registered and is ready to receive payloads. * * Calling `createHook()` alone does not register the hook — registration * only happens when the workflow suspends. Awaiting `getConflict()` @@ -42,13 +43,14 @@ export interface Hook extends AsyncIterable, Thenable { * waiting for payload data. * * When a conflict is detected, the resolved `Run` is the run that - * currently owns the token. The workflow can decide how to handle the - * duplicate in code: return or log `conflict.runId`, inspect + * owns the token. The owner may still be running or may have reached a + * terminal state while retaining its claim. The workflow can decide how to + * handle the duplicate in code: return or log `conflict.runId`, inspect * `await conflict.status`, await `conflict.returnValue`, or cancel the * owner with `await conflict.cancel()` and continue in the current run. * * Note that awaiting the hook's payload (`await hook`) when the token is - * already owned by another active hook still rejects with + * already owned by another hook or retained claim still rejects with * `HookConflictError`. In the rare case where the conflicting run cannot * be identified (a `hook_conflict` event persisted by an old world that * did not record the owning run's ID), `getConflict()` also rejects with @@ -147,6 +149,33 @@ export interface HookOptions { */ token?: string; + /** + * **Experimental.** Keeps this hook's token claimed after its workflow run + * reaches a terminal state, until the configured deadline has passed. + * + * Accepts the same values as `sleep()`: a duration string, a number of + * milliseconds, or an absolute `Date`. Relative durations are evaluated + * when `createHook()` runs and persisted as an absolute deadline. + * + * The live hook is not retained and cannot be resumed after the run ends. + * Only the token claim is retained, so another hook using the same token + * receives a conflict during the retention window. Calling `dispose()` + * (including through `using`) always releases the token immediately. + * + * World implementations may ignore this experimental option if they do not + * support retained hook-token claims. + * + * @example + * + * ```ts + * const hook = createHook({ + * token: `order:${orderId}`, + * experimental_retention: '30d', + * }); + * ``` + */ + experimental_retention?: StringValue | Date | number; + /** * Additional user-defined data to include with the hook payload. * @@ -179,7 +208,7 @@ export interface HookOptions { } export interface WebhookOptions - extends Omit { + extends Omit { /** * If set to a `Response` object, the webhook will automatically * respond with the specified response. diff --git a/packages/core/src/global.ts b/packages/core/src/global.ts index 14cf591fb2..29dfafc92b 100644 --- a/packages/core/src/global.ts +++ b/packages/core/src/global.ts @@ -17,6 +17,8 @@ export interface HookInvocationQueueItem { type: 'hook'; correlationId: string; token: string; + /** Earliest time a terminal run's token claim may be reused. */ + tokenReusableAfter?: Date; metadata?: Serializable; hasCreatedEvent?: boolean; /** Whether the workflow is awaiting `hook.getConflict()` for this hook */ diff --git a/packages/core/src/runtime/suspension-handler.test.ts b/packages/core/src/runtime/suspension-handler.test.ts index 9557ea6d84..f379ad0ade 100644 --- a/packages/core/src/runtime/suspension-handler.test.ts +++ b/packages/core/src/runtime/suspension-handler.test.ts @@ -30,6 +30,43 @@ function createWorld(eventsCreate: ReturnType): World { } describe('handleSuspension', () => { + it('persists a hook token retention deadline on hook_created', async () => { + const eventsCreate = vi.fn().mockImplementation(async (_runId, event) => ({ + event, + })); + const world = createWorld(eventsCreate); + const tokenReusableAfter = new Date('2026-08-01T00:00:00.000Z'); + const pending = new Map([ + [ + 'hook_retained', + { + type: 'hook' as const, + correlationId: 'hook_retained', + token: 'order:123', + tokenReusableAfter, + }, + ], + ]); + + await handleSuspension({ + suspension: new WorkflowSuspension(pending, globalThis), + world, + run, + }); + + expect(eventsCreate).toHaveBeenCalledWith( + run.runId, + expect.objectContaining({ + eventType: 'hook_created', + eventData: expect.objectContaining({ + token: 'order:123', + tokenReusableAfter, + }), + }), + expect.anything() + ); + }); + it('marks hook.getConflict()-awaited creations without converting them into wait timeouts', async () => { const eventsCreate = vi.fn().mockResolvedValue({ event: { diff --git a/packages/core/src/runtime/suspension-handler.ts b/packages/core/src/runtime/suspension-handler.ts index c54889bed2..f771afff2b 100644 --- a/packages/core/src/runtime/suspension-handler.ts +++ b/packages/core/src/runtime/suspension-handler.ts @@ -336,6 +336,7 @@ export async function handleSuspension({ correlationId: queueItem.correlationId, eventData: { token: queueItem.token, + tokenReusableAfter: queueItem.tokenReusableAfter, metadata: hookMetadata, isWebhook: queueItem.isWebhook ?? false, ...(queueItem.isSystem && { isSystem: true }), diff --git a/packages/core/src/workflow/hook.test.ts b/packages/core/src/workflow/hook.test.ts index eedf3c8aee..8cd217e1be 100644 --- a/packages/core/src/workflow/hook.test.ts +++ b/packages/core/src/workflow/hook.test.ts @@ -1173,6 +1173,73 @@ describe('createCreateHook', () => { } }); + it('evaluates hook token retention with the same duration parser as sleep', () => { + const ctx = setupWorkflowContext([]); + const createHook = createCreateHook(ctx); + const dateNow = vi.spyOn(Date, 'now').mockReturnValue(1_000_000); + try { + createHook({ experimental_retention: 1_000 }); + + const queueItem = ctx.invocationsQueue.values().next().value; + expect(queueItem?.type).toBe('hook'); + if (queueItem?.type === 'hook') { + expect(queueItem.tokenReusableAfter).toEqual(new Date(1_001_000)); + } + } finally { + dateNow.mockRestore(); + } + }); + + it.each([ + { + name: 'uses the persisted deadline', + eventData: { + token: 'test-token', + tokenReusableAfter: new Date('2026-07-15T00:00:00.000Z'), + }, + expected: new Date('2026-07-15T00:00:00.000Z'), + }, + { + name: 'keeps an old event without retention unretained', + eventData: { token: 'test-token' }, + expected: undefined, + }, + ])('$name on replay', async ({ eventData, expected }) => { + const ctx = setupWorkflowContext([ + { + eventId: 'evnt_0', + runId: 'wrun_123', + eventType: 'hook_created', + correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + eventData, + createdAt: new Date(), + }, + ]); + const createHook = createCreateHook(ctx); + const hook = createHook({ + token: 'test-token', + experimental_retention: '30d', + }); + + await expect(hook.getConflict()).resolves.toBeNull(); + + const queueItem = ctx.invocationsQueue.values().next().value; + expect(queueItem?.type).toBe('hook'); + if (queueItem?.type === 'hook') { + expect(queueItem.tokenReusableAfter).toEqual(expected); + } + }); + + it('rejects token retention for a webhook hook', () => { + const ctx = setupWorkflowContext([]); + const createHook = createCreateHook(ctx); + + expect(() => + createHook({ isWebhook: true, experimental_retention: '30d' }) + ).toThrow('Webhook hooks do not support `experimental_retention`.'); + expect(ctx.invocationsQueue.size).toBe(0); + }); + it('should throw when an empty string token is provided', () => { const ctx = setupWorkflowContext([]); const createHook = createCreateHook(ctx); diff --git a/packages/core/src/workflow/hook.ts b/packages/core/src/workflow/hook.ts index 329c97caf5..a8ba662788 100644 --- a/packages/core/src/workflow/hook.ts +++ b/packages/core/src/workflow/hook.ts @@ -1,6 +1,12 @@ import { HookConflictError, ReplayDivergenceError } from '@workflow/errors'; -import { type PromiseWithResolvers, withResolvers } from '@workflow/utils'; +import { WORKFLOW_DESERIALIZE } from '@workflow/serde'; +import { + type PromiseWithResolvers, + parseDurationToDate, + withResolvers, +} from '@workflow/utils'; import type { HookConflictEvent } from '@workflow/world'; +import { getSerializationClass, RUN_CLASS_ID } from '../class-serialization.js'; import type { Hook, HookOptions } from '../create-hook.js'; import { EventConsumerResult } from '../events-consumer.js'; import { WorkflowSuspension } from '../global.js'; @@ -11,8 +17,6 @@ import { scheduleWhenIdle, type WorkflowOrchestratorContext, } from '../private.js'; -import { WORKFLOW_DESERIALIZE } from '@workflow/serde'; -import { getSerializationClass, RUN_CLASS_ID } from '../class-serialization.js'; import type { Run } from '../runtime/run.js'; import { hydrateStepReturnValue } from '../serialization.js'; @@ -69,9 +73,22 @@ export function createCreateHook(ctx: WorkflowOrchestratorContext) { ); } + if ( + options.isWebhook === true && + options.experimental_retention !== undefined + ) { + throw new Error( + 'Webhook hooks do not support `experimental_retention`. Use a non-webhook `createHook()` with `resumeHook()` for retained token claims.' + ); + } + // Generate hook ID and token const correlationId = `hook_${ctx.generateUlid()}`; const token = options.token ?? ctx.generateNanoid(); + const tokenReusableAfter = + options.experimental_retention === undefined + ? undefined + : parseDurationToDate(options.experimental_retention); // Add hook creation to invocations queue (using Map for O(1) operations) const isWebhook = options.isWebhook ?? false; @@ -80,6 +97,7 @@ export function createCreateHook(ctx: WorkflowOrchestratorContext) { type: 'hook', correlationId, token, + tokenReusableAfter, metadata: options.metadata, isWebhook, }); @@ -162,6 +180,10 @@ export function createCreateHook(ctx: WorkflowOrchestratorContext) { const queueItem = ctx.invocationsQueue.get(correlationId); if (queueItem && queueItem.type === 'hook') { queueItem.hasCreatedEvent = true; + // The event log is authoritative on replay. In particular, an old + // event with no retention field must not gain retention merely + // because newly deployed workflow code added the option. + queueItem.tokenReusableAfter = event.eventData.tokenReusableAfter; } hasCreated = true; diff --git a/packages/world-vercel/src/events-v4.ts b/packages/world-vercel/src/events-v4.ts index 421c9f0bf5..d63d56b917 100644 --- a/packages/world-vercel/src/events-v4.ts +++ b/packages/world-vercel/src/events-v4.ts @@ -126,6 +126,8 @@ export interface CreateEventV4Input { * the step entity for premature-delivery pacing and observability. */ retryAfter?: Date; hookToken?: string; + /** Earliest time a terminal hook token claim may be reused. */ + hookTokenReusableAfter?: Date; hookIsWebhook?: boolean; hookIsSystem?: boolean; errorCode?: string; @@ -228,6 +230,9 @@ function buildPostFrameMeta( if (input.resumeAt !== undefined) meta.resumeAt = input.resumeAt; if (input.retryAfter !== undefined) meta.retryAfter = input.retryAfter; if (input.hookToken !== undefined) meta.hookToken = input.hookToken; + if (input.hookTokenReusableAfter !== undefined) { + meta.hookTokenReusableAfter = input.hookTokenReusableAfter; + } if (input.hookIsWebhook !== undefined) meta.hookIsWebhook = input.hookIsWebhook; if (input.hookIsSystem !== undefined) meta.hookIsSystem = input.hookIsSystem; diff --git a/packages/world-vercel/src/events.test.ts b/packages/world-vercel/src/events.test.ts index c2107a2790..554454e5ce 100644 --- a/packages/world-vercel/src/events.test.ts +++ b/packages/world-vercel/src/events.test.ts @@ -120,6 +120,23 @@ describe('createWorkflowRunEvent with v1Compat', () => { * actually reach the frame meta with the right values and renames. */ describe('splitEventDataForV4 attribute fields', () => { + it('carries the hook token retention deadline in the frame meta', () => { + const tokenReusableAfter = new Date('2026-07-10T12:00:00.000Z'); + const { payload, meta } = splitEventDataForV4({ + eventType: 'hook_created', + correlationId: 'hook_1', + specVersion: 5, + eventData: { + token: 'order:123', + tokenReusableAfter, + }, + } as AnyEventRequest); + + expect(payload).toBeUndefined(); + expect(meta.hookToken).toBe('order:123'); + expect(meta.hookTokenReusableAfter).toEqual(tokenReusableAfter); + }); + it('carries attr_set changes/writer/allowReservedAttributes in the frame meta', () => { const { payload, meta } = splitEventDataForV4({ eventType: 'attr_set', diff --git a/packages/world-vercel/src/events.ts b/packages/world-vercel/src/events.ts index adf076ced8..9f899d468a 100644 --- a/packages/world-vercel/src/events.ts +++ b/packages/world-vercel/src/events.ts @@ -177,6 +177,7 @@ interface SplitEventData { resumeAt?: Date; retryAfter?: Date; hookToken?: string; + hookTokenReusableAfter?: Date; hookIsWebhook?: boolean; hookIsSystem?: boolean; errorCode?: string; @@ -220,6 +221,7 @@ type MetaSourceField = | 'resumeAt' | 'retryAfter' | 'token' + | 'tokenReusableAfter' | 'isWebhook' | 'isSystem' | 'errorCode' @@ -315,6 +317,9 @@ export function splitEventDataForV4(data: AnyEventRequest): SplitEventData { if (typeof eventData.token === 'string') { meta.hookToken = eventData.token; } + if (eventData.tokenReusableAfter instanceof Date) { + meta.hookTokenReusableAfter = eventData.tokenReusableAfter; + } if (typeof eventData.isWebhook === 'boolean') { meta.hookIsWebhook = eventData.isWebhook; } diff --git a/packages/world/src/events.test.ts b/packages/world/src/events.test.ts index 34d661b20d..f311c47c7e 100644 --- a/packages/world/src/events.test.ts +++ b/packages/world/src/events.test.ts @@ -1,6 +1,27 @@ import { describe, expect, it } from 'vitest'; import { CreateEventSchema, EventSchema } from './events'; +describe('hook_created token retention', () => { + it('coerces tokenReusableAfter to a Date', () => { + const parsed = CreateEventSchema.parse({ + eventType: 'hook_created', + correlationId: 'hook_1', + specVersion: 5, + eventData: { + token: 'order:123', + tokenReusableAfter: '2026-08-01T00:00:00.000Z', + }, + }); + + expect(parsed.eventType).toBe('hook_created'); + if (parsed.eventType === 'hook_created') { + expect(parsed.eventData.tokenReusableAfter).toEqual( + new Date('2026-08-01T00:00:00.000Z') + ); + } + }); +}); + describe('run_cancelled cancelReason', () => { it('accepts a run_cancelled create request with no eventData', () => { const parsed = CreateEventSchema.parse({ diff --git a/packages/world/src/events.ts b/packages/world/src/events.ts index d30b814fbd..b5e828836c 100644 --- a/packages/world/src/events.ts +++ b/packages/world/src/events.ts @@ -232,6 +232,7 @@ export const HookCreatedEventSchema = BaseEventSchema.extend({ correlationId: z.string(), eventData: z.object({ token: z.string(), + tokenReusableAfter: z.coerce.date().optional(), metadata: SerializedDataSchema.optional(), isWebhook: z.boolean().optional(), isSystem: z.boolean().optional(), From f7a58654a87e263ebacfb0fafeb9c48415b7dd35 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 10 Jul 2026 10:00:27 -0700 Subject: [PATCH 02/36] refactor(core): constrain hook retention options --- packages/core/src/create-hook.ts | 89 +++++++++++++------------ packages/core/src/workflow/hook.test.ts | 1 + 2 files changed, 46 insertions(+), 44 deletions(-) diff --git a/packages/core/src/create-hook.ts b/packages/core/src/create-hook.ts index 6847784ba6..7521aea75c 100644 --- a/packages/core/src/create-hook.ts +++ b/packages/core/src/create-hook.ts @@ -121,7 +121,7 @@ export interface Webhook extends Hook { url: string; } -export interface HookOptions { +interface HookBaseOptions { /** * Unique token that is used to associate with the hook. * @@ -149,33 +149,6 @@ export interface HookOptions { */ token?: string; - /** - * **Experimental.** Keeps this hook's token claimed after its workflow run - * reaches a terminal state, until the configured deadline has passed. - * - * Accepts the same values as `sleep()`: a duration string, a number of - * milliseconds, or an absolute `Date`. Relative durations are evaluated - * when `createHook()` runs and persisted as an absolute deadline. - * - * The live hook is not retained and cannot be resumed after the run ends. - * Only the token claim is retained, so another hook using the same token - * receives a conflict during the retention window. Calling `dispose()` - * (including through `using`) always releases the token immediately. - * - * World implementations may ignore this experimental option if they do not - * support retained hook-token claims. - * - * @example - * - * ```ts - * const hook = createHook({ - * token: `order:${orderId}`, - * experimental_retention: '30d', - * }); - * ``` - */ - experimental_retention?: StringValue | Date | number; - /** * Additional user-defined data to include with the hook payload. * @@ -191,24 +164,52 @@ export interface HookOptions { * ``` */ metadata?: Serializable; - - /** - * Whether this hook can be resumed via the public webhook endpoint. - * - * When `true`, the hook can be triggered by sending an HTTP request to the - * public workflow webhook URL. This is automatically set when using - * `createWebhook()`. - * - * When `false` (the default), the hook can only be resumed server-side - * via `resumeHook()`. - * - * @default false - */ - isWebhook?: boolean; } -export interface WebhookOptions - extends Omit { +export type HookOptions = HookBaseOptions & + ( + | { + /** + * **Experimental.** Keeps this hook's token claimed after its workflow + * run reaches a terminal state, until the configured deadline passes. + * + * Accepts the same values as `sleep()`: a duration string, a number of + * milliseconds, or an absolute `Date`. Relative durations are evaluated + * when `createHook()` runs and persisted as an absolute deadline. + * + * The live hook is not retained and cannot be resumed after the run ends. + * Only the token claim is retained, so another hook using the same token + * receives a conflict during the retention window. Calling `dispose()` + * (including through `using`) always releases the token immediately. + * + * World implementations may ignore this experimental option if they do + * not support retained hook-token claims. + * + * @example + * + * ```ts + * const hook = createHook({ + * token: `order:${orderId}`, + * experimental_retention: '30d', + * }); + * ``` + */ + experimental_retention?: StringValue | Date | number; + + /** + * Whether this hook can be resumed via the public webhook endpoint. + * + * The default `false` restricts the hook to server-side `resumeHook()`. + * `createWebhook()` sets this internal option to `true`. + * + * @default false + */ + isWebhook?: false; + } + | { isWebhook: true; experimental_retention?: never } + ); + +export interface WebhookOptions extends Omit { /** * If set to a `Response` object, the webhook will automatically * respond with the specified response. diff --git a/packages/core/src/workflow/hook.test.ts b/packages/core/src/workflow/hook.test.ts index 8cd217e1be..4960d6cf2c 100644 --- a/packages/core/src/workflow/hook.test.ts +++ b/packages/core/src/workflow/hook.test.ts @@ -1235,6 +1235,7 @@ describe('createCreateHook', () => { const createHook = createCreateHook(ctx); expect(() => + // @ts-expect-error Verify the runtime error for untyped JavaScript callers. createHook({ isWebhook: true, experimental_retention: '30d' }) ).toThrow('Webhook hooks do not support `experimental_retention`.'); expect(ctx.invocationsQueue.size).toBe(0); From 4fe2918d8d3d68a78d7b83a50a24ffc8c358c1f1 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 10 Jul 2026 10:12:42 -0700 Subject: [PATCH 03/36] fix(core): preserve boolean hook visibility options --- packages/core/src/create-hook.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/core/src/create-hook.ts b/packages/core/src/create-hook.ts index 7521aea75c..4eb27739e4 100644 --- a/packages/core/src/create-hook.ts +++ b/packages/core/src/create-hook.ts @@ -194,7 +194,7 @@ export type HookOptions = HookBaseOptions & * }); * ``` */ - experimental_retention?: StringValue | Date | number; + experimental_retention: StringValue | Date | number; /** * Whether this hook can be resumed via the public webhook endpoint. @@ -206,7 +206,7 @@ export type HookOptions = HookBaseOptions & */ isWebhook?: false; } - | { isWebhook: true; experimental_retention?: never } + | { experimental_retention?: never; isWebhook?: boolean } ); export interface WebhookOptions extends Omit { From 0cd74018a6c640212b5a3447dbf352a84efd0bf6 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 10 Jul 2026 10:20:55 -0700 Subject: [PATCH 04/36] revert(core): preserve HookOptions interface --- packages/core/src/create-hook.ts | 89 ++++++++++++------------- packages/core/src/workflow/hook.test.ts | 1 - 2 files changed, 44 insertions(+), 46 deletions(-) diff --git a/packages/core/src/create-hook.ts b/packages/core/src/create-hook.ts index 4eb27739e4..6847784ba6 100644 --- a/packages/core/src/create-hook.ts +++ b/packages/core/src/create-hook.ts @@ -121,7 +121,7 @@ export interface Webhook extends Hook { url: string; } -interface HookBaseOptions { +export interface HookOptions { /** * Unique token that is used to associate with the hook. * @@ -149,6 +149,33 @@ interface HookBaseOptions { */ token?: string; + /** + * **Experimental.** Keeps this hook's token claimed after its workflow run + * reaches a terminal state, until the configured deadline has passed. + * + * Accepts the same values as `sleep()`: a duration string, a number of + * milliseconds, or an absolute `Date`. Relative durations are evaluated + * when `createHook()` runs and persisted as an absolute deadline. + * + * The live hook is not retained and cannot be resumed after the run ends. + * Only the token claim is retained, so another hook using the same token + * receives a conflict during the retention window. Calling `dispose()` + * (including through `using`) always releases the token immediately. + * + * World implementations may ignore this experimental option if they do not + * support retained hook-token claims. + * + * @example + * + * ```ts + * const hook = createHook({ + * token: `order:${orderId}`, + * experimental_retention: '30d', + * }); + * ``` + */ + experimental_retention?: StringValue | Date | number; + /** * Additional user-defined data to include with the hook payload. * @@ -164,52 +191,24 @@ interface HookBaseOptions { * ``` */ metadata?: Serializable; -} - -export type HookOptions = HookBaseOptions & - ( - | { - /** - * **Experimental.** Keeps this hook's token claimed after its workflow - * run reaches a terminal state, until the configured deadline passes. - * - * Accepts the same values as `sleep()`: a duration string, a number of - * milliseconds, or an absolute `Date`. Relative durations are evaluated - * when `createHook()` runs and persisted as an absolute deadline. - * - * The live hook is not retained and cannot be resumed after the run ends. - * Only the token claim is retained, so another hook using the same token - * receives a conflict during the retention window. Calling `dispose()` - * (including through `using`) always releases the token immediately. - * - * World implementations may ignore this experimental option if they do - * not support retained hook-token claims. - * - * @example - * - * ```ts - * const hook = createHook({ - * token: `order:${orderId}`, - * experimental_retention: '30d', - * }); - * ``` - */ - experimental_retention: StringValue | Date | number; - /** - * Whether this hook can be resumed via the public webhook endpoint. - * - * The default `false` restricts the hook to server-side `resumeHook()`. - * `createWebhook()` sets this internal option to `true`. - * - * @default false - */ - isWebhook?: false; - } - | { experimental_retention?: never; isWebhook?: boolean } - ); + /** + * Whether this hook can be resumed via the public webhook endpoint. + * + * When `true`, the hook can be triggered by sending an HTTP request to the + * public workflow webhook URL. This is automatically set when using + * `createWebhook()`. + * + * When `false` (the default), the hook can only be resumed server-side + * via `resumeHook()`. + * + * @default false + */ + isWebhook?: boolean; +} -export interface WebhookOptions extends Omit { +export interface WebhookOptions + extends Omit { /** * If set to a `Response` object, the webhook will automatically * respond with the specified response. diff --git a/packages/core/src/workflow/hook.test.ts b/packages/core/src/workflow/hook.test.ts index 4960d6cf2c..8cd217e1be 100644 --- a/packages/core/src/workflow/hook.test.ts +++ b/packages/core/src/workflow/hook.test.ts @@ -1235,7 +1235,6 @@ describe('createCreateHook', () => { const createHook = createCreateHook(ctx); expect(() => - // @ts-expect-error Verify the runtime error for untyped JavaScript callers. createHook({ isWebhook: true, experimental_retention: '30d' }) ).toThrow('Webhook hooks do not support `experimental_retention`.'); expect(ctx.invocationsQueue.size).toBe(0); From 8d54bac5598f914e2087e936b3574ef3e01076fb Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 10 Jul 2026 10:29:37 -0700 Subject: [PATCH 05/36] docs(core): clarify retained conflict ownership --- .../docs/v5/foundations/idempotency.mdx | 48 ++++--------------- packages/core/src/create-hook.ts | 8 +++- 2 files changed, 16 insertions(+), 40 deletions(-) diff --git a/docs/content/docs/v5/foundations/idempotency.mdx b/docs/content/docs/v5/foundations/idempotency.mdx index 6d445aef9c..2251552b3b 100644 --- a/docs/content/docs/v5/foundations/idempotency.mdx +++ b/docs/content/docs/v5/foundations/idempotency.mdx @@ -159,6 +159,8 @@ Some workflow systems resolve duplicate IDs with a fixed, pre-declared policy The example above implements **reject the duplicate**: return the owner's `runId` and let the caller decide. Other common strategies: +A conflict means the current run did not acquire the token. It must not perform protected work unless a new Hook later registers without a conflict. A retained terminal claim cannot receive payloads or be released by cancelling its run; reject the duplicate or adopt the stored run result until the claim expires. + **Adopt the owner's result.** Wait for the active run to finish and return its result, so callers cannot tell which run did the work: ```typescript lineNumbers @@ -184,7 +186,7 @@ export async function processOrder(orderId: string) { } ``` -**Inspect the owner before deciding.** Branch on the owner's live state: +**Inspect the owner before deciding.** Branch on the owner's state without assuming the current run owns the token: ```typescript lineNumbers import { createHook } from "workflow"; @@ -202,17 +204,14 @@ export async function processOrder(orderId: string) { const conflict = await request.getConflict(); if (conflict) { const status = await conflict.status; // [!code highlight] - if (status === "running") { - return { status: "duplicate" as const, runId: conflict.runId }; - } - // Owner already reached a terminal state; its hook will be released. + return { status: "duplicate" as const, ownerStatus: status, runId: conflict.runId }; } return await processOwnedOrder(orderId); } ``` -**Signal the owner instead of doing the work.** The duplicate run knows the token, so it can deliver this run's input to the owner's hook from a step: +**Signal an active owner instead of doing the work.** Without retention, the duplicate run can deliver its input to an owner that is still active. `resumeHook()` can race owner completion, so production code should handle `HookNotFoundError`: ```typescript lineNumbers import { createHook } from "workflow"; @@ -233,6 +232,10 @@ export async function processOrder(orderId: string, confirmed: boolean) { const conflict = await request.getConflict(); if (conflict) { + const status = await conflict.status; + if (status !== "pending" && status !== "running") { + return { status: "duplicate" as const, runId: conflict.runId }; + } await forwardToOwner(token, { confirmed }); // [!code highlight] return { status: "forwarded" as const, runId: conflict.runId }; } @@ -241,38 +244,7 @@ export async function processOrder(orderId: string, confirmed: boolean) { } ``` -**Supersede the owner.** Newest-wins: cancel the active run, then claim the released token. Cancellation disposes the owner's hooks; the retry loop covers the window where that disposal has not propagated yet: - -```typescript lineNumbers -import { createHook } from "workflow"; - -type OrderRequest = { confirmed: boolean }; -declare function chargeOrder(orderId: string): Promise; // @setup - -export async function processOrderNewestWins(orderId: string) { - "use workflow"; - - const token = `order:${orderId}`; - - for (let attempt = 0; attempt < 3; attempt++) { - using request = createHook({ token }); - - const conflict = await request.getConflict(); - if (!conflict) { - // Token claimed — this run is now the owner. - const { confirmed } = await request; - if (confirmed) { - await chargeOrder(orderId); - } - return { status: "processed" as const }; - } - - await conflict.cancel(); // [!code highlight] - } - - throw new Error(`Could not claim ${token} after cancelling the owner`); -} -``` +**Supersede an active owner.** This strategy applies only when retention is disabled. Cancellation does not transfer ownership: after cancelling, create a new Hook and proceed only after `getConflict()` returns `null`. The cancellation can race a terminal transition, so handle that outcome and retry the claim. With retention enabled, cancellation makes the owner terminal but leaves its claim in place until the deadline. If duplicate requests should only reuse the active run without sending data, use [`getHookByToken()`](/docs/api-reference/workflow-api/get-hook-by-token) as an advisory pre-check before calling `start()`. The workflow should still check `hook.getConflict()`, because the lookup and `start()` are not atomic. diff --git a/packages/core/src/create-hook.ts b/packages/core/src/create-hook.ts index 6847784ba6..2bc28b512a 100644 --- a/packages/core/src/create-hook.ts +++ b/packages/core/src/create-hook.ts @@ -46,8 +46,12 @@ export interface Hook extends AsyncIterable, Thenable { * owns the token. The owner may still be running or may have reached a * terminal state while retaining its claim. The workflow can decide how to * handle the duplicate in code: return or log `conflict.runId`, inspect - * `await conflict.status`, await `conflict.returnValue`, or cancel the - * owner with `await conflict.cancel()` and continue in the current run. + * `await conflict.status`, or await `conflict.returnValue`. + * + * A conflict means this hook did not acquire the token. Cancelling the owner + * does not transfer ownership, and a retained claim survives cancellation + * until its deadline. Never perform protected work until a new hook has + * registered successfully without a conflict. * * Note that awaiting the hook's payload (`await hook`) when the token is * already owned by another hook or retained claim still rejects with From eda3a409d7810193dae445c42dad3ea29147c0d5 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 10 Jul 2026 10:32:49 -0700 Subject: [PATCH 06/36] docs(core): retain newest-wins conflict pattern --- .../v5/api-reference/workflow/create-hook.mdx | 2 +- docs/content/docs/v5/foundations/hooks.mdx | 2 +- .../docs/v5/foundations/idempotency.mdx | 37 ++++++++++++++++++- 3 files changed, 38 insertions(+), 3 deletions(-) diff --git a/docs/content/docs/v5/api-reference/workflow/create-hook.mdx b/docs/content/docs/v5/api-reference/workflow/create-hook.mdx index 687c34d900..1ca75355a5 100644 --- a/docs/content/docs/v5/api-reference/workflow/create-hook.mdx +++ b/docs/content/docs/v5/api-reference/workflow/create-hook.mdx @@ -143,7 +143,7 @@ async function processOrder(orderId: string) { Because `createHook()` alone does not suspend the workflow, awaiting `hook.getConflict()` is what actually suspends the run and commits the hook registration. It only waits for registration — to receive payload data from a future `resumeHook()` call, await the hook itself or iterate it with `for await...of`. -On a conflict, the resolved value is a `Run` handle for the run that owns the token, with durable step-backed accessors. With retention configured, that run may already be terminal. The duplicate run can decide in code how to handle it: return or log `conflict.runId`, inspect `await conflict.status`, wait on `await conflict.returnValue`, or cancel an active owner with `await conflict.cancel()`. See [Run idempotency](/docs/foundations/idempotency#run-idempotency) for these strategies in context. +On a conflict, the resolved value is a `Run` handle for the run that owns the token, with durable step-backed accessors. With retention configured, that run may already be terminal. The duplicate run can decide in code how to handle it: return or log `conflict.runId`, inspect `await conflict.status`, wait on `await conflict.returnValue`, or cancel an active owner with `await conflict.cancel()`. Cancellation does not transfer ownership, and a retained claim survives it until its deadline. See [Run idempotency](/docs/foundations/idempotency#run-idempotency) for these strategies in context. Custom hook tokens are the recommended way to coordinate active workflow runs. Use a deterministic token from your domain, such as an order ID or conversation ID, create the hook near the beginning of the workflow, and check `await hook.getConflict()` before work that depends on owning the token. See [Run idempotency](/docs/foundations/idempotency#run-idempotency). diff --git a/docs/content/docs/v5/foundations/hooks.mdx b/docs/content/docs/v5/foundations/hooks.mdx index ca89aa4772..5e1005bb29 100644 --- a/docs/content/docs/v5/foundations/hooks.mdx +++ b/docs/content/docs/v5/foundations/hooks.mdx @@ -112,7 +112,7 @@ export async function orderWorkflow(orderId: string) { } ``` -Calling `createHook()` on its own does not register the hook — registration is only committed when the workflow suspends. Awaiting `hook.getConflict()` suspends the workflow to commit the hook registration, then resolves with `null` once the hook is registered and ready to receive payloads, or with a `Run` handle for the run that owns the token if another hook or retained claim already owns it (see [`HookConflictError`](/docs/errors/hook-conflict)). For `hook_conflict` events persisted by older worlds that did not record the owning run's ID, `getConflict()` rejects with `HookConflictError` instead of resolving with an incomplete handle. The conflicting run's accessors are durable steps, so the workflow can inspect `await conflict.status`, wait on `await conflict.returnValue`, or cancel an active owner with `await conflict.cancel()` — see [Run idempotency](/docs/foundations/idempotency#run-idempotency) for these strategies. +Calling `createHook()` on its own does not register the hook — registration is only committed when the workflow suspends. Awaiting `hook.getConflict()` suspends the workflow to commit the hook registration, then resolves with `null` once the hook is registered and ready to receive payloads, or with a `Run` handle for the run that owns the token if another hook or retained claim already owns it (see [`HookConflictError`](/docs/errors/hook-conflict)). For `hook_conflict` events persisted by older worlds that did not record the owning run's ID, `getConflict()` rejects with `HookConflictError` instead of resolving with an incomplete handle. The conflicting run's accessors are durable steps, so the workflow can inspect `await conflict.status`, wait on `await conflict.returnValue`, or cancel an active owner with `await conflict.cancel()`. Cancellation does not transfer ownership, and a retained claim survives it until its deadline — see [Run idempotency](/docs/foundations/idempotency#run-idempotency) for these strategies. ### Custom Tokens for Deterministic Hooks diff --git a/docs/content/docs/v5/foundations/idempotency.mdx b/docs/content/docs/v5/foundations/idempotency.mdx index 2251552b3b..927a4acdce 100644 --- a/docs/content/docs/v5/foundations/idempotency.mdx +++ b/docs/content/docs/v5/foundations/idempotency.mdx @@ -244,7 +244,42 @@ export async function processOrder(orderId: string, confirmed: boolean) { } ``` -**Supersede an active owner.** This strategy applies only when retention is disabled. Cancellation does not transfer ownership: after cancelling, create a new Hook and proceed only after `getConflict()` returns `null`. The cancellation can race a terminal transition, so handle that outcome and retry the claim. With retention enabled, cancellation makes the owner terminal but leaves its claim in place until the deadline. +**Supersede the owner.** Newest-wins: cancel the active run, then claim the released token. Cancellation disposes the owner's hooks; the retry loop covers the window where that disposal has not propagated yet: + +```typescript lineNumbers +import { createHook } from "workflow"; + +type OrderRequest = { confirmed: boolean }; +declare function chargeOrder(orderId: string): Promise; // @setup + +export async function processOrderNewestWins(orderId: string) { + "use workflow"; + + const token = `order:${orderId}`; + + for (let attempt = 0; attempt < 3; attempt++) { + using request = createHook({ token }); + + const conflict = await request.getConflict(); + if (!conflict) { + // Token claimed — this run is now the owner. + const { confirmed } = await request; + if (confirmed) { + await chargeOrder(orderId); + } + return { status: "processed" as const }; + } + + await conflict.cancel(); // [!code highlight] + } + + throw new Error(`Could not claim ${token} after cancelling the owner`); +} +``` + + +Do not combine this pattern with `experimental_retention`. Cancellation makes a retained owner terminal but does not release its token claim; the claim remains until its deadline. Cancellation also does not transfer ownership by itself, so protected work must remain behind a later `getConflict()` that returns `null`. + If duplicate requests should only reuse the active run without sending data, use [`getHookByToken()`](/docs/api-reference/workflow-api/get-hook-by-token) as an advisory pre-check before calling `start()`. The workflow should still check `hook.getConflict()`, because the lookup and `start()` are not atomic. From 58a5a4bbac5e15d30dab35b406e527c27c8ffcea Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 10 Jul 2026 10:38:42 -0700 Subject: [PATCH 07/36] docs(core): simplify hook retention guidance --- .../v5/api-reference/workflow/create-hook.mdx | 26 +++++++------ docs/content/docs/v5/foundations/hooks.mdx | 6 ++- .../docs/v5/foundations/idempotency.mdx | 8 ++-- packages/core/src/create-hook.ts | 38 +++++++++---------- 4 files changed, 40 insertions(+), 38 deletions(-) diff --git a/docs/content/docs/v5/api-reference/workflow/create-hook.mdx b/docs/content/docs/v5/api-reference/workflow/create-hook.mdx index 1ca75355a5..033518c5ce 100644 --- a/docs/content/docs/v5/api-reference/workflow/create-hook.mdx +++ b/docs/content/docs/v5/api-reference/workflow/create-hook.mdx @@ -66,7 +66,7 @@ export default Hook;`} The returned `Hook` object also implements `AsyncIterable`, which allows you to iterate over incoming payloads using `for await...of` syntax. -Use `hook.getConflict()` to check whether the hook token is already claimed by another hook or a retained token claim, without waiting for hook payload data. Calling `createHook()` on its own does not register the hook — registration is only committed when the workflow suspends. Awaiting `hook.getConflict()` suspends the workflow to commit the registration, then resolves with `null` once `hook_created` is recorded, or with the conflicting [`Run`](/docs/api-reference/workflow-api/get-run) if another run already owns the same token. +Use `hook.getConflict()` to check whether another run already owns the token. This also detects tokens that remain reserved after their run has finished. Calling `createHook()` on its own does not register the hook — registration is only committed when the workflow suspends. Awaiting `hook.getConflict()` suspends the workflow to commit the registration, then resolves with `null` once `hook_created` is recorded, or with the conflicting [`Run`](/docs/api-reference/workflow-api/get-run). ## Examples @@ -143,15 +143,15 @@ async function processOrder(orderId: string) { Because `createHook()` alone does not suspend the workflow, awaiting `hook.getConflict()` is what actually suspends the run and commits the hook registration. It only waits for registration — to receive payload data from a future `resumeHook()` call, await the hook itself or iterate it with `for await...of`. -On a conflict, the resolved value is a `Run` handle for the run that owns the token, with durable step-backed accessors. With retention configured, that run may already be terminal. The duplicate run can decide in code how to handle it: return or log `conflict.runId`, inspect `await conflict.status`, wait on `await conflict.returnValue`, or cancel an active owner with `await conflict.cancel()`. Cancellation does not transfer ownership, and a retained claim survives it until its deadline. See [Run idempotency](/docs/foundations/idempotency#run-idempotency) for these strategies in context. +On a conflict, `getConflict()` returns the run that owns the token. That run may still be active, or it may have finished while keeping the token reserved. The duplicate can return the owner's `runId`, inspect its status, or use its return value. A conflict always means the duplicate does not own the token. See [Run idempotency](/docs/foundations/idempotency#run-idempotency) for complete examples. Custom hook tokens are the recommended way to coordinate active workflow runs. Use a deterministic token from your domain, such as an order ID or conversation ID, create the hook near the beginning of the workflow, and check `await hook.getConflict()` before work that depends on owning the token. See [Run idempotency](/docs/foundations/idempotency#run-idempotency). -### Retaining a Token Claim After Completion +### Keep a Token Reserved After the Run Ends -By default, a hook token becomes reusable when its workflow run completes, fails, or is cancelled. Set `experimental_retention` when a late duplicate must still conflict after the owner reaches a terminal state: +By default, a hook token becomes reusable when its workflow run ends. Set `experimental_retention` to keep the token reserved for late duplicate requests: ```typescript lineNumbers import { createHook } from "workflow"; @@ -161,8 +161,7 @@ declare function processOwnedOrder(orderId: string): Promise; // @setup export async function processOrder(orderId: string) { "use workflow"; - // Do not use `using` for a retained claim. `using` calls dispose(), which - // explicitly releases the token. + // Do not use `using` here. It calls dispose() and releases the token. const ownership = createHook({ // [!code highlight] token: `order:${orderId}`, // [!code highlight] experimental_retention: "30d", // [!code highlight] @@ -178,17 +177,22 @@ export async function processOrder(orderId: string) { } ``` -`experimental_retention` accepts the same values as [`sleep()`](/docs/api-reference/workflow/sleep): a duration string such as `"30d"`, a number of milliseconds, or an absolute `Date`. A relative duration starts when `createHook()` is evaluated, not when the run ends. The token remains claimed while the owner is active even if that deadline passes, and after a terminal outcome only until the deadline passes. In other words, the earliest reuse time is the later of the run becoming terminal and the configured deadline. +`experimental_retention` accepts the same values as [`sleep()`](/docs/api-reference/workflow/sleep): a duration string such as `"30d"`, a number of milliseconds, or an absolute `Date`. -Retention preserves only the token claim. The Hook itself is disposed when the run ends and cannot receive more payloads. A conflicting run can still use `getConflict()` to identify the run that owns the retained claim. +The duration starts when `createHook()` runs, not when the workflow ends. For example, with `"30d"`: -Calling `hook.dispose()` always releases the token immediately, including when retention is configured. The `using` keyword calls `dispose()` when the scope exits, so leave an ownership Hook undisposed when you want terminal cleanup to retain its claim. This also provides an explicit way to opt out of retention on an application-controlled failure path. +- If the workflow finishes after 1 day, the token stays reserved for 29 more days. +- If the workflow runs for 40 days, the token stays reserved until the workflow finishes. + +After the workflow ends, the Hook cannot receive more payloads. Only the token remains reserved. Another Hook using the same token receives a conflict and can use `getConflict()` to find the original run. + +Calling `hook.dispose()` releases the token immediately, even when retention is configured. The `using` keyword calls `dispose()` automatically, so do not use `using` for a Hook whose token should remain reserved after the run ends. -This option is experimental and requires support from the configured World. Worlds that do not implement retained token claims ignore it, so it must not be the only idempotency mechanism for a deployment whose World support is unknown. `createWebhook()` does not accept this option. +This option is experimental. A World that does not support it will ignore it. `createWebhook()` does not accept this option. -The configured time is an earliest reuse deadline, not a precise timer. Scheduling, network latency, and backend cleanup can make the claim reusable slightly later, so very small millisecond values are not suitable for precise timing. A World may also impose a maximum retention window and reject a deadline beyond that limit. +The token may become reusable slightly after the deadline because of scheduling, network latency, or cleanup delays. Do not use short millisecond values as a precise timer. A World may also reject a deadline beyond its maximum retention window. ### Waiting for Multiple Payloads diff --git a/docs/content/docs/v5/foundations/hooks.mdx b/docs/content/docs/v5/foundations/hooks.mdx index 5e1005bb29..8c833457cd 100644 --- a/docs/content/docs/v5/foundations/hooks.mdx +++ b/docs/content/docs/v5/foundations/hooks.mdx @@ -112,7 +112,9 @@ export async function orderWorkflow(orderId: string) { } ``` -Calling `createHook()` on its own does not register the hook — registration is only committed when the workflow suspends. Awaiting `hook.getConflict()` suspends the workflow to commit the hook registration, then resolves with `null` once the hook is registered and ready to receive payloads, or with a `Run` handle for the run that owns the token if another hook or retained claim already owns it (see [`HookConflictError`](/docs/errors/hook-conflict)). For `hook_conflict` events persisted by older worlds that did not record the owning run's ID, `getConflict()` rejects with `HookConflictError` instead of resolving with an incomplete handle. The conflicting run's accessors are durable steps, so the workflow can inspect `await conflict.status`, wait on `await conflict.returnValue`, or cancel an active owner with `await conflict.cancel()`. Cancellation does not transfer ownership, and a retained claim survives it until its deadline — see [Run idempotency](/docs/foundations/idempotency#run-idempotency) for these strategies. +Calling `createHook()` on its own does not register the hook — registration is only committed when the workflow suspends. Awaiting `hook.getConflict()` suspends the workflow to commit the hook registration, then resolves with `null` once the hook is ready, or with the run that already owns the token. The owner may still be active, or its token may remain reserved after it finished. A conflict means the current Hook does not own the token. See [Run idempotency](/docs/foundations/idempotency#run-idempotency) for handling examples. + +Events written by older Worlds may not include the owner's run ID. For those events, `getConflict()` throws `HookConflictError`. ### Custom Tokens for Deterministic Hooks @@ -487,7 +489,7 @@ When using custom tokens with `createHook()`: - **Make them deterministic**: Base them on data the external system can reconstruct (like channel IDs, user IDs, etc.) - **Use namespacing**: Prefix tokens to avoid conflicts (e.g., `slack:${channelId}`, `github:${repoId}`) - **Include routing information**: Ensure the token contains enough information to identify the correct workflow instance -- **Choose an explicit retention policy for idempotency claims**: By default the token is reusable after the run ends. Use [`experimental_retention`](/docs/api-reference/workflow/create-hook#retaining-a-token-claim-after-completion) when late duplicates must still conflict, and do not automatically dispose that ownership hook. +- **Decide how long the token should stay reserved**: By default the token is reusable after the run ends. Use [`experimental_retention`](/docs/api-reference/workflow/create-hook#keep-a-token-reserved-after-the-run-ends) to block late duplicates, and do not automatically dispose that Hook. ### Response Handling in Webhooks diff --git a/docs/content/docs/v5/foundations/idempotency.mdx b/docs/content/docs/v5/foundations/idempotency.mdx index 927a4acdce..4dd415ffab 100644 --- a/docs/content/docs/v5/foundations/idempotency.mdx +++ b/docs/content/docs/v5/foundations/idempotency.mdx @@ -149,9 +149,9 @@ export async function POST(request: Request) { This avoids creating a new run only after the first run has registered its hook. Because `start()` returns before the run body executes and calls `createHook()`, two concurrent requests can both observe "no hook yet" and each call `start()`. The race is resolved inside the workflow body, where the losing run observes `getConflict()` resolving with the active owner and returns without doing duplicate-sensitive work — and the route detects it by comparing the resumed hook's `runId` against the run it just started, without waiting for either run to finish. A native API for atomically starting a run and registering a hook is in the works. Until then, model recovery inside the workflow by checking `hook.getConflict()`. -By default, this is active-run coordination: when the workflow reaches a terminal state, the token can be used again. To reject late duplicates for a bounded window, set `experimental_retention` and leave the ownership Hook undisposed. The live Hook still disappears when the run ends; only its token claim remains. See [`createHook()` token retention](/docs/api-reference/workflow/create-hook#retaining-a-token-claim-after-completion) for the API, accepted deadline formats, and World support limitations. +By default, the token becomes reusable when the workflow ends. Set `experimental_retention` to keep it reserved for late duplicate requests. The Hook itself is still removed and cannot be resumed. See [`createHook()` token retention](/docs/api-reference/workflow/create-hook#keep-a-token-reserved-after-the-run-ends) for examples and supported deadline formats. -Retained claims reject duplicates but do not store or return the original workflow result by themselves. A duplicate receives the conflicting `Run` and can inspect its status or return value while that run remains available. If the result must outlive workflow-run retention, persist it under the same domain key in application storage. +`experimental_retention` prevents duplicate work; it does not save the result. A duplicate can use `conflict.returnValue` while the original run is still available. If callers need the result for longer, save it in your own database under the same key, such as the order ID. ### Conflict-handling strategies @@ -159,7 +159,7 @@ Some workflow systems resolve duplicate IDs with a fixed, pre-declared policy The example above implements **reject the duplicate**: return the owner's `runId` and let the caller decide. Other common strategies: -A conflict means the current run did not acquire the token. It must not perform protected work unless a new Hook later registers without a conflict. A retained terminal claim cannot receive payloads or be released by cancelling its run; reject the duplicate or adopt the stored run result until the claim expires. +If `getConflict()` returns a run, the current run does not own the token and must not do the work guarded by that token. With retention enabled, the owner may already be finished. In that case its Hook cannot be resumed, and cancelling the finished run will not free the token. **Adopt the owner's result.** Wait for the active run to finish and return its result, so callers cannot tell which run did the work: @@ -278,7 +278,7 @@ export async function processOrderNewestWins(orderId: string) { ``` -Do not combine this pattern with `experimental_retention`. Cancellation makes a retained owner terminal but does not release its token claim; the claim remains until its deadline. Cancellation also does not transfer ownership by itself, so protected work must remain behind a later `getConflict()` that returns `null`. +Do not combine this pattern with `experimental_retention`. Cancelling the owner does not free a retained token. The loop may continue only after a new Hook calls `getConflict()` and receives `null`. If duplicate requests should only reuse the active run without sending data, use [`getHookByToken()`](/docs/api-reference/workflow-api/get-hook-by-token) as an advisory pre-check before calling `start()`. The workflow should still check `hook.getConflict()`, because the lookup and `start()` are not atomic. diff --git a/packages/core/src/create-hook.ts b/packages/core/src/create-hook.ts index 2bc28b512a..87fe559602 100644 --- a/packages/core/src/create-hook.ts +++ b/packages/core/src/create-hook.ts @@ -33,8 +33,8 @@ export interface Hook extends AsyncIterable, Thenable { /** * Returns a promise that resolves with the conflicting {@link Run} if - * another hook or retained claim already owns this hook's token, or `null` - * once the hook has been registered and is ready to receive payloads. + * another Hook already owns this token, including a token kept reserved + * after its run ended, or `null` once this Hook has registered. * * Calling `createHook()` alone does not register the hook — registration * only happens when the workflow suspends. Awaiting `getConflict()` @@ -42,19 +42,16 @@ export interface Hook extends AsyncIterable, Thenable { * used to claim the token (and detect token conflicts early) without * waiting for payload data. * - * When a conflict is detected, the resolved `Run` is the run that - * owns the token. The owner may still be running or may have reached a - * terminal state while retaining its claim. The workflow can decide how to - * handle the duplicate in code: return or log `conflict.runId`, inspect - * `await conflict.status`, or await `conflict.returnValue`. + * When a conflict is detected, the resolved `Run` owns the token. It may + * still be active, or its token may remain reserved after it finished. The + * duplicate can use its run ID, status, or return value. * - * A conflict means this hook did not acquire the token. Cancelling the owner - * does not transfer ownership, and a retained claim survives cancellation - * until its deadline. Never perform protected work until a new hook has - * registered successfully without a conflict. + * A conflict means this Hook does not own the token. Cancelling the owner + * does not transfer ownership or free a retained token. Do not perform + * protected work until a new Hook registers without a conflict. * * Note that awaiting the hook's payload (`await hook`) when the token is - * already owned by another hook or retained claim still rejects with + * already owned by another Hook or reserved by a finished run still rejects with * `HookConflictError`. In the rare case where the conflicting run cannot * be identified (a `hook_conflict` event persisted by an old world that * did not record the owning run's ID), `getConflict()` also rejects with @@ -154,20 +151,19 @@ export interface HookOptions { token?: string; /** - * **Experimental.** Keeps this hook's token claimed after its workflow run - * reaches a terminal state, until the configured deadline has passed. + * **Experimental.** Keeps this Hook's token reserved after its workflow run + * ends, until the configured deadline. * * Accepts the same values as `sleep()`: a duration string, a number of - * milliseconds, or an absolute `Date`. Relative durations are evaluated - * when `createHook()` runs and persisted as an absolute deadline. + * milliseconds, or an absolute `Date`. Relative durations start when + * `createHook()` runs, not when the workflow ends. * - * The live hook is not retained and cannot be resumed after the run ends. - * Only the token claim is retained, so another hook using the same token - * receives a conflict during the retention window. Calling `dispose()` - * (including through `using`) always releases the token immediately. + * The Hook cannot be resumed after the run ends. Another Hook using the same + * token receives a conflict until the deadline. Calling `dispose()` + * (including through `using`) releases the token immediately. * * World implementations may ignore this experimental option if they do not - * support retained hook-token claims. + * support token retention. * * @example * From efa640407ff04f142ae62435ef9afb856486e59a Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 10 Jul 2026 10:40:29 -0700 Subject: [PATCH 08/36] docs(core): explain retained token cleanup --- docs/content/docs/v5/deploying/building-a-world.mdx | 4 ++-- docs/content/docs/v5/how-it-works/event-sourcing.mdx | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/content/docs/v5/deploying/building-a-world.mdx b/docs/content/docs/v5/deploying/building-a-world.mdx index 3e62c9e389..8e7556eb46 100644 --- a/docs/content/docs/v5/deploying/building-a-world.mdx +++ b/docs/content/docs/v5/deploying/building-a-world.mdx @@ -93,9 +93,9 @@ interface Storage { **Run Creation:** For `run_created` events, the `runId` parameter may be a client-provided string or `null`. When `null`, your World generates and returns a new `runId`. -**Hook Tokens:** Hook tokens must be unique. If a `hook_created` event conflicts with an existing token, return a `hook_conflict` event instead and include the active hook owner's run ID as `eventData.conflictingRunId`. +**Hook Tokens:** Hook tokens must be unique. If a `hook_created` event conflicts with an active Hook or a token still reserved after its run ended, return a `hook_conflict` event and include the owner's run ID as `eventData.conflictingRunId`. -**Automatic Hook Disposal:** When a workflow reaches a terminal state (`completed`, `failed`, or `cancelled`), automatically dispose of all associated hooks to release tokens for reuse. +**Automatic Hook Cleanup:** When a workflow ends, remove its live Hooks. Release each token by default. If your World supports `experimental_retention`, keep the token reserved until its `tokenReusableAfter` deadline. ## Queue Interface diff --git a/docs/content/docs/v5/how-it-works/event-sourcing.mdx b/docs/content/docs/v5/how-it-works/event-sourcing.mdx index 0c2fa23296..e3ca1afa6d 100644 --- a/docs/content/docs/v5/how-it-works/event-sourcing.mdx +++ b/docs/content/docs/v5/how-it-works/event-sourcing.mdx @@ -129,7 +129,7 @@ Unlike other entities, hooks don't have a `status` field—the states above are While a hook is active, its token is reserved and cannot be used by other workflows. If a workflow attempts to create a hook with a token that is already in use by another active hook, a `hook_conflict` event is recorded instead of `hook_created`. Current worlds include the token and the run ID that currently owns it, though older persisted events or world implementations may only include the token. This causes `hook.getConflict()` to resolve with the conflicting run and the hook's payload promise to reject with a `HookConflictError`, which you can detect with `HookConflictError.is(error)`. See the [hook-conflict error](/docs/errors/hook-conflict) documentation for more details. -When a hook is disposed (either explicitly or when its workflow completes), the token is released and can be claimed by future workflows. Hooks are automatically disposed when a workflow reaches a terminal state (`completed`, `failed`, or `cancelled`). The `hook_disposed` event is only needed for explicit disposal before workflow completion. +When a workflow ends, its Hooks are removed and cannot receive more payloads. Their tokens are reusable by default. With `experimental_retention`, a token can stay reserved until its deadline even though the Hook is gone. An explicit `hook_disposed` event always releases the token immediately. See [Hooks & Webhooks](/docs/foundations/hooks) for more on how hooks and webhooks work. From 1771b8d6b9592100fe742965d4206c07722e4787 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 10 Jul 2026 10:49:49 -0700 Subject: [PATCH 09/36] docs(core): simplify idempotency guidance --- docs/content/docs/v5/foundations/idempotency.mdx | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/docs/content/docs/v5/foundations/idempotency.mdx b/docs/content/docs/v5/foundations/idempotency.mdx index 4dd415ffab..2f52a4e9f8 100644 --- a/docs/content/docs/v5/foundations/idempotency.mdx +++ b/docs/content/docs/v5/foundations/idempotency.mdx @@ -96,7 +96,7 @@ export async function processOrder(orderId: string): Promise { } ``` -The runtime creates the hook atomically. At most one active hook can own `order:${orderId}`, so duplicate workflow runs converge on one active owner. A duplicate run observes `getConflict()` resolving with the owner's `Run` and returns before it reaches `chargeOrder()`. The conflicting run's accessors (`status`, `returnValue`, `cancel()`, …) are durable steps, so the duplicate run can do more than report the owner — see [conflict-handling strategies](#conflict-handling-strategies) below. +Only one Hook can own `order:${orderId}`. If two runs use that token, `getConflict()` returns the first run to the duplicate. The duplicate returns before it reaches `chargeOrder()`. Outside the workflow, try to resume the hook first. If the hook is not registered yet, start the workflow and retry the resume until the new run creates the hook: @@ -146,18 +146,16 @@ export async function POST(request: Request) { ``` -This avoids creating a new run only after the first run has registered its hook. Because `start()` returns before the run body executes and calls `createHook()`, two concurrent requests can both observe "no hook yet" and each call `start()`. The race is resolved inside the workflow body, where the losing run observes `getConflict()` resolving with the active owner and returns without doing duplicate-sensitive work — and the route detects it by comparing the resumed hook's `runId` against the run it just started, without waiting for either run to finish. A native API for atomically starting a run and registering a hook is in the works. Until then, model recovery inside the workflow by checking `hook.getConflict()`. +Calling `resumeHook()` first avoids starting a duplicate after the Hook exists. It does not close the short gap between `start()` and `createHook()`: two requests can still start two runs. The `getConflict()` check inside the workflow stops the losing run before it does duplicate work. A native API that starts a run and creates its Hook atomically is planned. By default, the token becomes reusable when the workflow ends. Set `experimental_retention` to keep it reserved for late duplicate requests. The Hook itself is still removed and cannot be resumed. See [`createHook()` token retention](/docs/api-reference/workflow/create-hook#keep-a-token-reserved-after-the-run-ends) for examples and supported deadline formats. -`experimental_retention` prevents duplicate work; it does not save the result. A duplicate can use `conflict.returnValue` while the original run is still available. If callers need the result for longer, save it in your own database under the same key, such as the order ID. +Retention only reserves the token. It does not save the workflow's result. While Workflow still stores the original run, a duplicate can read `conflict.returnValue`. If the result must remain available after that run is deleted, save it in your own database under the same key, such as the order ID. ### Conflict-handling strategies -Some workflow systems resolve duplicate IDs with a fixed, pre-declared policy — typically a static choice between rejecting the new execution, deferring to the existing one, or terminating it. Workflow has no policy enum. `hook.getConflict()` hands the duplicate run the conflicting `Run` itself, and the policy is ordinary code — including policies that inspect state before deciding, which static configuration can't express. - -The example above implements **reject the duplicate**: return the owner's `runId` and let the caller decide. Other common strategies: +The example above rejects the duplicate by returning the owner's `runId`. You can also handle a conflict in these ways: If `getConflict()` returns a run, the current run does not own the token and must not do the work guarded by that token. With retention enabled, the owner may already be finished. In that case its Hook cannot be resumed, and cancelling the finished run will not free the token. @@ -278,7 +276,7 @@ export async function processOrderNewestWins(orderId: string) { ``` -Do not combine this pattern with `experimental_retention`. Cancelling the owner does not free a retained token. The loop may continue only after a new Hook calls `getConflict()` and receives `null`. +Do not use this newest-wins pattern with `experimental_retention`. Cancelling the owner removes its Hook, but the token stays reserved until its retention deadline. The new run cannot claim it before then. If duplicate requests should only reuse the active run without sending data, use [`getHookByToken()`](/docs/api-reference/workflow-api/get-hook-by-token) as an advisory pre-check before calling `start()`. The workflow should still check `hook.getConflict()`, because the lookup and `start()` are not atomic. From cdc543e78849a566856996c1ca03fdc044670db4 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 10 Jul 2026 10:59:41 -0700 Subject: [PATCH 10/36] docs(core): clarify retained token results --- docs/content/docs/v5/foundations/idempotency.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/docs/v5/foundations/idempotency.mdx b/docs/content/docs/v5/foundations/idempotency.mdx index 2f52a4e9f8..60a9388c15 100644 --- a/docs/content/docs/v5/foundations/idempotency.mdx +++ b/docs/content/docs/v5/foundations/idempotency.mdx @@ -151,7 +151,7 @@ Calling `resumeHook()` first avoids starting a duplicate after the Hook exists. By default, the token becomes reusable when the workflow ends. Set `experimental_retention` to keep it reserved for late duplicate requests. The Hook itself is still removed and cannot be resumed. See [`createHook()` token retention](/docs/api-reference/workflow/create-hook#keep-a-token-reserved-after-the-run-ends) for examples and supported deadline formats. -Retention only reserves the token. It does not save the workflow's result. While Workflow still stores the original run, a duplicate can read `conflict.returnValue`. If the result must remain available after that run is deleted, save it in your own database under the same key, such as the order ID. +`experimental_retention` stops another run from using the token. It does not cache what the first run returned. A duplicate can read `conflict.returnValue` only while the first run is still stored. If callers need that result after old runs are deleted, save it in your own database under the same key, such as the order ID. ### Conflict-handling strategies From 08fe5207a7abeb7e69157428f75758b988ecdba9 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 10 Jul 2026 11:53:01 -0700 Subject: [PATCH 11/36] refactor(core): rename hook token expiration option --- .changeset/retained-hook-contract.md | 2 +- .../v5/api-reference/workflow/create-hook.mdx | 16 ++++++++------ .../docs/v5/deploying/building-a-world.mdx | 2 +- docs/content/docs/v5/foundations/hooks.mdx | 2 +- .../docs/v5/foundations/idempotency.mdx | 10 ++++----- .../docs/v5/how-it-works/event-sourcing.mdx | 2 +- packages/core/src/create-hook.ts | 22 +++++++++++-------- .../src/runtime/suspension-handler.test.ts | 2 +- packages/core/src/workflow/hook.test.ts | 14 ++++++------ packages/core/src/workflow/hook.ts | 12 +++++----- packages/world-vercel/src/events.test.ts | 2 +- packages/world/src/events.test.ts | 2 +- 12 files changed, 47 insertions(+), 41 deletions(-) diff --git a/.changeset/retained-hook-contract.md b/.changeset/retained-hook-contract.md index 80f5674a48..282a570fed 100644 --- a/.changeset/retained-hook-contract.md +++ b/.changeset/retained-hook-contract.md @@ -4,4 +4,4 @@ '@workflow/world-vercel': minor --- -Add the experimental hook-token retention contract and forward its absolute reuse deadline through workflow events. +Add `experimental_expires` for keeping a Hook token unavailable after its run ends, and forward the absolute expiration time through workflow events. diff --git a/docs/content/docs/v5/api-reference/workflow/create-hook.mdx b/docs/content/docs/v5/api-reference/workflow/create-hook.mdx index 033518c5ce..e5600d309b 100644 --- a/docs/content/docs/v5/api-reference/workflow/create-hook.mdx +++ b/docs/content/docs/v5/api-reference/workflow/create-hook.mdx @@ -149,9 +149,9 @@ On a conflict, `getConflict()` returns the run that owns the token. That run may Custom hook tokens are the recommended way to coordinate active workflow runs. Use a deterministic token from your domain, such as an order ID or conversation ID, create the hook near the beginning of the workflow, and check `await hook.getConflict()` before work that depends on owning the token. See [Run idempotency](/docs/foundations/idempotency#run-idempotency). -### Keep a Token Reserved After the Run Ends +### Set When a Hook Token Expires -By default, a hook token becomes reusable when its workflow run ends. Set `experimental_retention` to keep the token reserved for late duplicate requests: +By default, a Hook token expires when its workflow run ends, so another Hook can use it. Set `experimental_expires` to keep the token reserved for late duplicate requests: ```typescript lineNumbers import { createHook } from "workflow"; @@ -164,7 +164,7 @@ export async function processOrder(orderId: string) { // Do not use `using` here. It calls dispose() and releases the token. const ownership = createHook({ // [!code highlight] token: `order:${orderId}`, // [!code highlight] - experimental_retention: "30d", // [!code highlight] + experimental_expires: "30d", // [!code highlight] }); // [!code highlight] const conflict = await ownership.getConflict(); @@ -177,22 +177,24 @@ export async function processOrder(orderId: string) { } ``` -`experimental_retention` accepts the same values as [`sleep()`](/docs/api-reference/workflow/sleep): a duration string such as `"30d"`, a number of milliseconds, or an absolute `Date`. +`experimental_expires` accepts the same values as [`sleep()`](/docs/api-reference/workflow/sleep): a duration string such as `"30d"`, a number of milliseconds, or an absolute `Date`. The duration starts when `createHook()` runs, not when the workflow ends. For example, with `"30d"`: - If the workflow finishes after 1 day, the token stays reserved for 29 more days. - If the workflow runs for 40 days, the token stays reserved until the workflow finishes. -After the workflow ends, the Hook cannot receive more payloads. Only the token remains reserved. Another Hook using the same token receives a conflict and can use `getConflict()` to find the original run. +Expiration never stops an active Hook. If the configured time passes while the workflow is still running, the token expires when the run ends. -Calling `hook.dispose()` releases the token immediately, even when retention is configured. The `using` keyword calls `dispose()` automatically, so do not use `using` for a Hook whose token should remain reserved after the run ends. +After the workflow ends, the Hook cannot receive more payloads. Until the token expires, another Hook using the same token receives a conflict and can use `getConflict()` to find the original run. + +Calling `hook.dispose()` expires the token immediately. The `using` keyword calls `dispose()` automatically, so do not use `using` when the token should stay reserved after the run ends. This option is experimental. A World that does not support it will ignore it. `createWebhook()` does not accept this option. -The token may become reusable slightly after the deadline because of scheduling, network latency, or cleanup delays. Do not use short millisecond values as a precise timer. A World may also reject a deadline beyond its maximum retention window. +The token may expire slightly late because of scheduling, network latency, or cleanup delays. Do not use short millisecond values as a precise timer. A World may also reject an expiration time beyond its maximum supported window. ### Waiting for Multiple Payloads diff --git a/docs/content/docs/v5/deploying/building-a-world.mdx b/docs/content/docs/v5/deploying/building-a-world.mdx index 8e7556eb46..219dbbd285 100644 --- a/docs/content/docs/v5/deploying/building-a-world.mdx +++ b/docs/content/docs/v5/deploying/building-a-world.mdx @@ -95,7 +95,7 @@ interface Storage { **Hook Tokens:** Hook tokens must be unique. If a `hook_created` event conflicts with an active Hook or a token still reserved after its run ended, return a `hook_conflict` event and include the owner's run ID as `eventData.conflictingRunId`. -**Automatic Hook Cleanup:** When a workflow ends, remove its live Hooks. Release each token by default. If your World supports `experimental_retention`, keep the token reserved until its `tokenReusableAfter` deadline. +**Automatic Hook Cleanup:** When a workflow ends, remove its live Hooks. Expire each token by default. If your World supports `experimental_expires`, keep the token reserved until `tokenReusableAfter`. ## Queue Interface diff --git a/docs/content/docs/v5/foundations/hooks.mdx b/docs/content/docs/v5/foundations/hooks.mdx index 8c833457cd..9b5d3203bc 100644 --- a/docs/content/docs/v5/foundations/hooks.mdx +++ b/docs/content/docs/v5/foundations/hooks.mdx @@ -489,7 +489,7 @@ When using custom tokens with `createHook()`: - **Make them deterministic**: Base them on data the external system can reconstruct (like channel IDs, user IDs, etc.) - **Use namespacing**: Prefix tokens to avoid conflicts (e.g., `slack:${channelId}`, `github:${repoId}`) - **Include routing information**: Ensure the token contains enough information to identify the correct workflow instance -- **Decide how long the token should stay reserved**: By default the token is reusable after the run ends. Use [`experimental_retention`](/docs/api-reference/workflow/create-hook#keep-a-token-reserved-after-the-run-ends) to block late duplicates, and do not automatically dispose that Hook. +- **Decide when the token should expire**: By default the token expires when the run ends. Use [`experimental_expires`](/docs/api-reference/workflow/create-hook#set-when-a-hook-token-expires) to block late duplicates, and do not automatically dispose that Hook. ### Response Handling in Webhooks diff --git a/docs/content/docs/v5/foundations/idempotency.mdx b/docs/content/docs/v5/foundations/idempotency.mdx index 60a9388c15..d486458c83 100644 --- a/docs/content/docs/v5/foundations/idempotency.mdx +++ b/docs/content/docs/v5/foundations/idempotency.mdx @@ -149,15 +149,15 @@ export async function POST(request: Request) { Calling `resumeHook()` first avoids starting a duplicate after the Hook exists. It does not close the short gap between `start()` and `createHook()`: two requests can still start two runs. The `getConflict()` check inside the workflow stops the losing run before it does duplicate work. A native API that starts a run and creates its Hook atomically is planned. -By default, the token becomes reusable when the workflow ends. Set `experimental_retention` to keep it reserved for late duplicate requests. The Hook itself is still removed and cannot be resumed. See [`createHook()` token retention](/docs/api-reference/workflow/create-hook#keep-a-token-reserved-after-the-run-ends) for examples and supported deadline formats. +By default, the token expires when the workflow ends. Set `experimental_expires` to keep it reserved for late duplicate requests. The Hook itself is still removed and cannot be resumed. See [`createHook()` token expiration](/docs/api-reference/workflow/create-hook#set-when-a-hook-token-expires) for examples and supported values. -`experimental_retention` stops another run from using the token. It does not cache what the first run returned. A duplicate can read `conflict.returnValue` only while the first run is still stored. If callers need that result after old runs are deleted, save it in your own database under the same key, such as the order ID. +`experimental_expires` stops another run from using the token until it expires. It does not cache what the first run returned. A duplicate can read `conflict.returnValue` only while the first run is still stored. If callers need that result after old runs are deleted, save it in your own database under the same key, such as the order ID. ### Conflict-handling strategies The example above rejects the duplicate by returning the owner's `runId`. You can also handle a conflict in these ways: -If `getConflict()` returns a run, the current run does not own the token and must not do the work guarded by that token. With retention enabled, the owner may already be finished. In that case its Hook cannot be resumed, and cancelling the finished run will not free the token. +If `getConflict()` returns a run, the current run does not own the token and must not do the work guarded by that token. With a later expiration time, the owner may already be finished. In that case its Hook cannot be resumed, and cancelling the finished run will not expire the token. **Adopt the owner's result.** Wait for the active run to finish and return its result, so callers cannot tell which run did the work: @@ -209,7 +209,7 @@ export async function processOrder(orderId: string) { } ``` -**Signal an active owner instead of doing the work.** Without retention, the duplicate run can deliver its input to an owner that is still active. `resumeHook()` can race owner completion, so production code should handle `HookNotFoundError`: +**Signal an active owner instead of doing the work.** With the default expiration behavior, the duplicate run can deliver its input to an owner that is still active. `resumeHook()` can race owner completion, so production code should handle `HookNotFoundError`: ```typescript lineNumbers import { createHook } from "workflow"; @@ -276,7 +276,7 @@ export async function processOrderNewestWins(orderId: string) { ``` -Do not use this newest-wins pattern with `experimental_retention`. Cancelling the owner removes its Hook, but the token stays reserved until its retention deadline. The new run cannot claim it before then. +Do not use this newest-wins pattern with `experimental_expires`. Cancelling the owner removes its Hook, but the token stays reserved until it expires. The new run cannot use it before then. If duplicate requests should only reuse the active run without sending data, use [`getHookByToken()`](/docs/api-reference/workflow-api/get-hook-by-token) as an advisory pre-check before calling `start()`. The workflow should still check `hook.getConflict()`, because the lookup and `start()` are not atomic. diff --git a/docs/content/docs/v5/how-it-works/event-sourcing.mdx b/docs/content/docs/v5/how-it-works/event-sourcing.mdx index e3ca1afa6d..dcf8f34f79 100644 --- a/docs/content/docs/v5/how-it-works/event-sourcing.mdx +++ b/docs/content/docs/v5/how-it-works/event-sourcing.mdx @@ -129,7 +129,7 @@ Unlike other entities, hooks don't have a `status` field—the states above are While a hook is active, its token is reserved and cannot be used by other workflows. If a workflow attempts to create a hook with a token that is already in use by another active hook, a `hook_conflict` event is recorded instead of `hook_created`. Current worlds include the token and the run ID that currently owns it, though older persisted events or world implementations may only include the token. This causes `hook.getConflict()` to resolve with the conflicting run and the hook's payload promise to reject with a `HookConflictError`, which you can detect with `HookConflictError.is(error)`. See the [hook-conflict error](/docs/errors/hook-conflict) documentation for more details. -When a workflow ends, its Hooks are removed and cannot receive more payloads. Their tokens are reusable by default. With `experimental_retention`, a token can stay reserved until its deadline even though the Hook is gone. An explicit `hook_disposed` event always releases the token immediately. +When a workflow ends, its Hooks are removed and cannot receive more payloads. Their tokens expire by default. With `experimental_expires`, a token can stay reserved after its Hook is gone. An explicit `hook_disposed` event always expires the token immediately. See [Hooks & Webhooks](/docs/foundations/hooks) for more on how hooks and webhooks work. diff --git a/packages/core/src/create-hook.ts b/packages/core/src/create-hook.ts index 87fe559602..30e2292157 100644 --- a/packages/core/src/create-hook.ts +++ b/packages/core/src/create-hook.ts @@ -151,30 +151,34 @@ export interface HookOptions { token?: string; /** - * **Experimental.** Keeps this Hook's token reserved after its workflow run - * ends, until the configured deadline. + * **Experimental.** Sets when this Hook's token expires after its workflow + * run ends. * * Accepts the same values as `sleep()`: a duration string, a number of * milliseconds, or an absolute `Date`. Relative durations start when * `createHook()` runs, not when the workflow ends. * - * The Hook cannot be resumed after the run ends. Another Hook using the same - * token receives a conflict until the deadline. Calling `dispose()` - * (including through `using`) releases the token immediately. + * Expiration never stops an active Hook. If the configured time passes while + * the workflow is still running, the token expires when the run ends. After + * the run ends, the Hook cannot be resumed. Another Hook using the same token + * receives a conflict until the token expires. + * + * Calling `dispose()` (including through `using`) expires the token + * immediately. * * World implementations may ignore this experimental option if they do not - * support token retention. + * support token expiration. * * @example * * ```ts * const hook = createHook({ * token: `order:${orderId}`, - * experimental_retention: '30d', + * experimental_expires: '30d', * }); * ``` */ - experimental_retention?: StringValue | Date | number; + experimental_expires?: StringValue | Date | number; /** * Additional user-defined data to include with the hook payload. @@ -208,7 +212,7 @@ export interface HookOptions { } export interface WebhookOptions - extends Omit { + extends Omit { /** * If set to a `Response` object, the webhook will automatically * respond with the specified response. diff --git a/packages/core/src/runtime/suspension-handler.test.ts b/packages/core/src/runtime/suspension-handler.test.ts index f379ad0ade..d0a3468ca1 100644 --- a/packages/core/src/runtime/suspension-handler.test.ts +++ b/packages/core/src/runtime/suspension-handler.test.ts @@ -30,7 +30,7 @@ function createWorld(eventsCreate: ReturnType): World { } describe('handleSuspension', () => { - it('persists a hook token retention deadline on hook_created', async () => { + it('persists when a hook token becomes reusable on hook_created', async () => { const eventsCreate = vi.fn().mockImplementation(async (_runId, event) => ({ event, })); diff --git a/packages/core/src/workflow/hook.test.ts b/packages/core/src/workflow/hook.test.ts index 8cd217e1be..b5cbbfaefd 100644 --- a/packages/core/src/workflow/hook.test.ts +++ b/packages/core/src/workflow/hook.test.ts @@ -1173,12 +1173,12 @@ describe('createCreateHook', () => { } }); - it('evaluates hook token retention with the same duration parser as sleep', () => { + it('evaluates hook token expiration with the same duration parser as sleep', () => { const ctx = setupWorkflowContext([]); const createHook = createCreateHook(ctx); const dateNow = vi.spyOn(Date, 'now').mockReturnValue(1_000_000); try { - createHook({ experimental_retention: 1_000 }); + createHook({ experimental_expires: 1_000 }); const queueItem = ctx.invocationsQueue.values().next().value; expect(queueItem?.type).toBe('hook'); @@ -1200,7 +1200,7 @@ describe('createCreateHook', () => { expected: new Date('2026-07-15T00:00:00.000Z'), }, { - name: 'keeps an old event without retention unretained', + name: 'keeps an old event without an expiration time unchanged', eventData: { token: 'test-token' }, expected: undefined, }, @@ -1218,7 +1218,7 @@ describe('createCreateHook', () => { const createHook = createCreateHook(ctx); const hook = createHook({ token: 'test-token', - experimental_retention: '30d', + experimental_expires: '30d', }); await expect(hook.getConflict()).resolves.toBeNull(); @@ -1230,13 +1230,13 @@ describe('createCreateHook', () => { } }); - it('rejects token retention for a webhook hook', () => { + it('rejects token expiration for a webhook hook', () => { const ctx = setupWorkflowContext([]); const createHook = createCreateHook(ctx); expect(() => - createHook({ isWebhook: true, experimental_retention: '30d' }) - ).toThrow('Webhook hooks do not support `experimental_retention`.'); + createHook({ isWebhook: true, experimental_expires: '30d' }) + ).toThrow('Webhook hooks do not support `experimental_expires`.'); expect(ctx.invocationsQueue.size).toBe(0); }); diff --git a/packages/core/src/workflow/hook.ts b/packages/core/src/workflow/hook.ts index a8ba662788..fd5ab48fa7 100644 --- a/packages/core/src/workflow/hook.ts +++ b/packages/core/src/workflow/hook.ts @@ -75,10 +75,10 @@ export function createCreateHook(ctx: WorkflowOrchestratorContext) { if ( options.isWebhook === true && - options.experimental_retention !== undefined + options.experimental_expires !== undefined ) { throw new Error( - 'Webhook hooks do not support `experimental_retention`. Use a non-webhook `createHook()` with `resumeHook()` for retained token claims.' + 'Webhook hooks do not support `experimental_expires`. Use a non-webhook `createHook()` with `resumeHook()`.' ); } @@ -86,9 +86,9 @@ export function createCreateHook(ctx: WorkflowOrchestratorContext) { const correlationId = `hook_${ctx.generateUlid()}`; const token = options.token ?? ctx.generateNanoid(); const tokenReusableAfter = - options.experimental_retention === undefined + options.experimental_expires === undefined ? undefined - : parseDurationToDate(options.experimental_retention); + : parseDurationToDate(options.experimental_expires); // Add hook creation to invocations queue (using Map for O(1) operations) const isWebhook = options.isWebhook ?? false; @@ -181,8 +181,8 @@ export function createCreateHook(ctx: WorkflowOrchestratorContext) { if (queueItem && queueItem.type === 'hook') { queueItem.hasCreatedEvent = true; // The event log is authoritative on replay. In particular, an old - // event with no retention field must not gain retention merely - // because newly deployed workflow code added the option. + // event with no expiration field must not gain one merely because + // newly deployed workflow code added the option. queueItem.tokenReusableAfter = event.eventData.tokenReusableAfter; } hasCreated = true; diff --git a/packages/world-vercel/src/events.test.ts b/packages/world-vercel/src/events.test.ts index 554454e5ce..e7569b4c19 100644 --- a/packages/world-vercel/src/events.test.ts +++ b/packages/world-vercel/src/events.test.ts @@ -120,7 +120,7 @@ describe('createWorkflowRunEvent with v1Compat', () => { * actually reach the frame meta with the right values and renames. */ describe('splitEventDataForV4 attribute fields', () => { - it('carries the hook token retention deadline in the frame meta', () => { + it('carries the hook token expiration time in the frame meta', () => { const tokenReusableAfter = new Date('2026-07-10T12:00:00.000Z'); const { payload, meta } = splitEventDataForV4({ eventType: 'hook_created', diff --git a/packages/world/src/events.test.ts b/packages/world/src/events.test.ts index f311c47c7e..0735422957 100644 --- a/packages/world/src/events.test.ts +++ b/packages/world/src/events.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from 'vitest'; import { CreateEventSchema, EventSchema } from './events'; -describe('hook_created token retention', () => { +describe('hook_created token expiration', () => { it('coerces tokenReusableAfter to a Date', () => { const parsed = CreateEventSchema.parse({ eventType: 'hook_created', From f8e6c6c3c3c0dee8a113f46853c81c398d13a67e Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 10 Jul 2026 11:55:58 -0700 Subject: [PATCH 12/36] chore(core): name hook expiration changeset --- .../{retained-hook-contract.md => hook-token-expiration.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .changeset/{retained-hook-contract.md => hook-token-expiration.md} (100%) diff --git a/.changeset/retained-hook-contract.md b/.changeset/hook-token-expiration.md similarity index 100% rename from .changeset/retained-hook-contract.md rename to .changeset/hook-token-expiration.md From 2b675947261a7610def6ec1757945af9c6d08ada Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 10 Jul 2026 13:35:54 -0700 Subject: [PATCH 13/36] docs(core): simplify Hook expiration language --- .../v5/api-reference/workflow/create-hook.mdx | 27 ++++------- .../docs/v5/deploying/building-a-world.mdx | 2 +- docs/content/docs/v5/foundations/hooks.mdx | 5 +- .../docs/v5/foundations/idempotency.mdx | 24 ++++------ .../docs/v5/how-it-works/event-sourcing.mdx | 10 ++-- packages/core/src/create-hook.ts | 46 ++++++------------- 6 files changed, 42 insertions(+), 72 deletions(-) diff --git a/docs/content/docs/v5/api-reference/workflow/create-hook.mdx b/docs/content/docs/v5/api-reference/workflow/create-hook.mdx index e5600d309b..bc52ea16bd 100644 --- a/docs/content/docs/v5/api-reference/workflow/create-hook.mdx +++ b/docs/content/docs/v5/api-reference/workflow/create-hook.mdx @@ -66,7 +66,7 @@ export default Hook;`} The returned `Hook` object also implements `AsyncIterable`, which allows you to iterate over incoming payloads using `for await...of` syntax. -Use `hook.getConflict()` to check whether another run already owns the token. This also detects tokens that remain reserved after their run has finished. Calling `createHook()` on its own does not register the hook — registration is only committed when the workflow suspends. Awaiting `hook.getConflict()` suspends the workflow to commit the registration, then resolves with `null` once `hook_created` is recorded, or with the conflicting [`Run`](/docs/api-reference/workflow-api/get-run). +`await hook.getConflict()` registers the Hook without waiting for a payload. It returns `null` if registration succeeds, or the [`Run`](/docs/api-reference/workflow-api/get-run) already using the token. This includes a finished run whose token is not yet reusable. ## Examples @@ -143,15 +143,15 @@ async function processOrder(orderId: string) { Because `createHook()` alone does not suspend the workflow, awaiting `hook.getConflict()` is what actually suspends the run and commits the hook registration. It only waits for registration — to receive payload data from a future `resumeHook()` call, await the hook itself or iterate it with `for await...of`. -On a conflict, `getConflict()` returns the run that owns the token. That run may still be active, or it may have finished while keeping the token reserved. The duplicate can return the owner's `runId`, inspect its status, or use its return value. A conflict always means the duplicate does not own the token. See [Run idempotency](/docs/foundations/idempotency#run-idempotency) for complete examples. +If `getConflict()` returns a run, this Hook cannot use the token. Return before doing work that depends on it. See [Run idempotency](/docs/foundations/idempotency#run-idempotency) for examples. Custom hook tokens are the recommended way to coordinate active workflow runs. Use a deterministic token from your domain, such as an order ID or conversation ID, create the hook near the beginning of the workflow, and check `await hook.getConflict()` before work that depends on owning the token. See [Run idempotency](/docs/foundations/idempotency#run-idempotency). -### Set When a Hook Token Expires +### Set When Another Hook Can Use the Token -By default, a Hook token expires when its workflow run ends, so another Hook can use it. Set `experimental_expires` to keep the token reserved for late duplicate requests: +By default, another Hook can use the token after its workflow ends. Set `experimental_expires` to choose a later time: ```typescript lineNumbers import { createHook } from "workflow"; @@ -162,12 +162,12 @@ export async function processOrder(orderId: string) { "use workflow"; // Do not use `using` here. It calls dispose() and releases the token. - const ownership = createHook({ // [!code highlight] + const hook = createHook({ // [!code highlight] token: `order:${orderId}`, // [!code highlight] experimental_expires: "30d", // [!code highlight] }); // [!code highlight] - const conflict = await ownership.getConflict(); + const conflict = await hook.getConflict(); if (conflict) { return { status: "duplicate" as const, runId: conflict.runId }; } @@ -177,24 +177,17 @@ export async function processOrder(orderId: string) { } ``` -`experimental_expires` accepts the same values as [`sleep()`](/docs/api-reference/workflow/sleep): a duration string such as `"30d"`, a number of milliseconds, or an absolute `Date`. +`experimental_expires` accepts the same values as [`sleep()`](/docs/api-reference/workflow/sleep): a duration string such as `"30d"`, a number of milliseconds, or an absolute `Date`. Durations start when `createHook()` runs. -The duration starts when `createHook()` runs, not when the workflow ends. For example, with `"30d"`: +The Hook remains active until the workflow ends, even if the configured time passes first. Another Hook can use the token only after both the workflow has ended and the configured time has passed. For example, `"30d"` keeps the token unavailable for 29 more days if the workflow ends after 1 day. A workflow that runs for more than 30 days releases the token when it ends. -- If the workflow finishes after 1 day, the token stays reserved for 29 more days. -- If the workflow runs for 40 days, the token stays reserved until the workflow finishes. - -Expiration never stops an active Hook. If the configured time passes while the workflow is still running, the token expires when the run ends. - -After the workflow ends, the Hook cannot receive more payloads. Until the token expires, another Hook using the same token receives a conflict and can use `getConflict()` to find the original run. - -Calling `hook.dispose()` expires the token immediately. The `using` keyword calls `dispose()` automatically, so do not use `using` when the token should stay reserved after the run ends. +`hook.dispose()` releases the token immediately. Because `using` calls `dispose()`, do not use it when the token should remain unavailable after the run ends. This option is experimental. A World that does not support it will ignore it. `createWebhook()` does not accept this option. -The token may expire slightly late because of scheduling, network latency, or cleanup delays. Do not use short millisecond values as a precise timer. A World may also reject an expiration time beyond its maximum supported window. +The token may become available late, so do not use this as an exact timer. A World may reject values beyond its supported maximum. ### Waiting for Multiple Payloads diff --git a/docs/content/docs/v5/deploying/building-a-world.mdx b/docs/content/docs/v5/deploying/building-a-world.mdx index 219dbbd285..aafbf7024a 100644 --- a/docs/content/docs/v5/deploying/building-a-world.mdx +++ b/docs/content/docs/v5/deploying/building-a-world.mdx @@ -95,7 +95,7 @@ interface Storage { **Hook Tokens:** Hook tokens must be unique. If a `hook_created` event conflicts with an active Hook or a token still reserved after its run ended, return a `hook_conflict` event and include the owner's run ID as `eventData.conflictingRunId`. -**Automatic Hook Cleanup:** When a workflow ends, remove its live Hooks. Expire each token by default. If your World supports `experimental_expires`, keep the token reserved until `tokenReusableAfter`. +**Automatic Hook Cleanup:** When a run ends, remove its live Hooks. Make each token available unless its `tokenReusableAfter` is still in the future. A `hook_disposed` event always makes the token available immediately. ## Queue Interface diff --git a/docs/content/docs/v5/foundations/hooks.mdx b/docs/content/docs/v5/foundations/hooks.mdx index 9b5d3203bc..ec541a2cc0 100644 --- a/docs/content/docs/v5/foundations/hooks.mdx +++ b/docs/content/docs/v5/foundations/hooks.mdx @@ -112,9 +112,7 @@ export async function orderWorkflow(orderId: string) { } ``` -Calling `createHook()` on its own does not register the hook — registration is only committed when the workflow suspends. Awaiting `hook.getConflict()` suspends the workflow to commit the hook registration, then resolves with `null` once the hook is ready, or with the run that already owns the token. The owner may still be active, or its token may remain reserved after it finished. A conflict means the current Hook does not own the token. See [Run idempotency](/docs/foundations/idempotency#run-idempotency) for handling examples. - -Events written by older Worlds may not include the owner's run ID. For those events, `getConflict()` throws `HookConflictError`. +`await hook.getConflict()` registers the Hook without waiting for data. It returns `null` if registration succeeds, or the run already using the token. See [Run idempotency](/docs/foundations/idempotency#run-idempotency) for handling examples. ### Custom Tokens for Deterministic Hooks @@ -489,7 +487,6 @@ When using custom tokens with `createHook()`: - **Make them deterministic**: Base them on data the external system can reconstruct (like channel IDs, user IDs, etc.) - **Use namespacing**: Prefix tokens to avoid conflicts (e.g., `slack:${channelId}`, `github:${repoId}`) - **Include routing information**: Ensure the token contains enough information to identify the correct workflow instance -- **Decide when the token should expire**: By default the token expires when the run ends. Use [`experimental_expires`](/docs/api-reference/workflow/create-hook#set-when-a-hook-token-expires) to block late duplicates, and do not automatically dispose that Hook. ### Response Handling in Webhooks diff --git a/docs/content/docs/v5/foundations/idempotency.mdx b/docs/content/docs/v5/foundations/idempotency.mdx index d486458c83..7f985952cc 100644 --- a/docs/content/docs/v5/foundations/idempotency.mdx +++ b/docs/content/docs/v5/foundations/idempotency.mdx @@ -96,7 +96,7 @@ export async function processOrder(orderId: string): Promise { } ``` -Only one Hook can own `order:${orderId}`. If two runs use that token, `getConflict()` returns the first run to the duplicate. The duplicate returns before it reaches `chargeOrder()`. +Only one Hook can use `order:${orderId}`. If two runs race, one receives a conflict and returns before it reaches `chargeOrder()`. Outside the workflow, try to resume the hook first. If the hook is not registered yet, start the workflow and retry the resume until the new run creates the hook: @@ -149,17 +149,13 @@ export async function POST(request: Request) { Calling `resumeHook()` first avoids starting a duplicate after the Hook exists. It does not close the short gap between `start()` and `createHook()`: two requests can still start two runs. The `getConflict()` check inside the workflow stops the losing run before it does duplicate work. A native API that starts a run and creates its Hook atomically is planned. -By default, the token expires when the workflow ends. Set `experimental_expires` to keep it reserved for late duplicate requests. The Hook itself is still removed and cannot be resumed. See [`createHook()` token expiration](/docs/api-reference/workflow/create-hook#set-when-a-hook-token-expires) for examples and supported values. - -`experimental_expires` stops another run from using the token until it expires. It does not cache what the first run returned. A duplicate can read `conflict.returnValue` only while the first run is still stored. If callers need that result after old runs are deleted, save it in your own database under the same key, such as the order ID. +By default, another Hook can use the token after the workflow ends. Set `experimental_expires` to delay that. The original Hook still ends with the workflow and cannot be resumed. See [`createHook()` expiration](/docs/api-reference/workflow/create-hook#set-when-another-hook-can-use-the-token) for examples and supported values. ### Conflict-handling strategies -The example above rejects the duplicate by returning the owner's `runId`. You can also handle a conflict in these ways: - -If `getConflict()` returns a run, the current run does not own the token and must not do the work guarded by that token. With a later expiration time, the owner may already be finished. In that case its Hook cannot be resumed, and cancelling the finished run will not expire the token. +The example above rejects the duplicate by returning the other run's `runId`. -**Adopt the owner's result.** Wait for the active run to finish and return its result, so callers cannot tell which run did the work: +**Adopt the owner's result.** Wait for the active run to finish and return its result: ```typescript lineNumbers import { createHook } from "workflow"; @@ -184,7 +180,7 @@ export async function processOrder(orderId: string) { } ``` -**Inspect the owner before deciding.** Branch on the owner's state without assuming the current run owns the token: +**Inspect the owner before deciding.** Branch on its status: ```typescript lineNumbers import { createHook } from "workflow"; @@ -209,7 +205,7 @@ export async function processOrder(orderId: string) { } ``` -**Signal an active owner instead of doing the work.** With the default expiration behavior, the duplicate run can deliver its input to an owner that is still active. `resumeHook()` can race owner completion, so production code should handle `HookNotFoundError`: +**Signal an active owner instead of doing the work.** If the other run is still active, the duplicate can send it the input. `resumeHook()` can race the other run finishing, so production code should handle `HookNotFoundError`: ```typescript lineNumbers import { createHook } from "workflow"; @@ -242,7 +238,7 @@ export async function processOrder(orderId: string, confirmed: boolean) { } ``` -**Supersede the owner.** Newest-wins: cancel the active run, then claim the released token. Cancellation disposes the owner's hooks; the retry loop covers the window where that disposal has not propagated yet: +**Use the newest run.** With the default expiration, cancel the active run, then retry until its token becomes available: ```typescript lineNumbers import { createHook } from "workflow"; @@ -260,7 +256,7 @@ export async function processOrderNewestWins(orderId: string) { const conflict = await request.getConflict(); if (!conflict) { - // Token claimed — this run is now the owner. + // This run now owns the token. const { confirmed } = await request; if (confirmed) { await chargeOrder(orderId); @@ -271,12 +267,12 @@ export async function processOrderNewestWins(orderId: string) { await conflict.cancel(); // [!code highlight] } - throw new Error(`Could not claim ${token} after cancelling the owner`); + throw new Error(`Token ${token} is still unavailable after cancelling the other run`); } ``` -Do not use this newest-wins pattern with `experimental_expires`. Cancelling the owner removes its Hook, but the token stays reserved until it expires. The new run cannot use it before then. +This pattern does not work with `experimental_expires`: cancelling the old run does not make its token available early. If duplicate requests should only reuse the active run without sending data, use [`getHookByToken()`](/docs/api-reference/workflow-api/get-hook-by-token) as an advisory pre-check before calling `start()`. The workflow should still check `hook.getConflict()`, because the lookup and `start()` are not atomic. diff --git a/docs/content/docs/v5/how-it-works/event-sourcing.mdx b/docs/content/docs/v5/how-it-works/event-sourcing.mdx index dcf8f34f79..e7a6e5eac3 100644 --- a/docs/content/docs/v5/how-it-works/event-sourcing.mdx +++ b/docs/content/docs/v5/how-it-works/event-sourcing.mdx @@ -123,13 +123,13 @@ flowchart TD - `active`: Ready to receive payloads (hook exists in storage) - `disposed`: No longer accepting payloads (hook is deleted from storage) -- `conflicted`: Hook creation failed because the token is already in use by another workflow +- `conflicted`: Hook creation failed because another run already owns the token Unlike other entities, hooks don't have a `status` field—the states above are conceptual. An "active" hook is one that exists in storage, while "disposed" means the hook has been deleted. When a `hook_disposed` event is created, the hook record is removed rather than updated. -While a hook is active, its token is reserved and cannot be used by other workflows. If a workflow attempts to create a hook with a token that is already in use by another active hook, a `hook_conflict` event is recorded instead of `hook_created`. Current worlds include the token and the run ID that currently owns it, though older persisted events or world implementations may only include the token. This causes `hook.getConflict()` to resolve with the conflicting run and the hook's payload promise to reject with a `HookConflictError`, which you can detect with `HookConflictError.is(error)`. See the [hook-conflict error](/docs/errors/hook-conflict) documentation for more details. +While a Hook is active, its token is reserved. If another run tries to register the same token, a `hook_conflict` event is recorded instead of `hook_created`. The same conflict occurs after the owning run ends when `experimental_expires` keeps its token reserved. `hook.getConflict()` returns the owning run, while awaiting the Hook rejects with `HookConflictError`. See the [hook-conflict error](/docs/errors/hook-conflict) documentation for more details. -When a workflow ends, its Hooks are removed and cannot receive more payloads. Their tokens expire by default. With `experimental_expires`, a token can stay reserved after its Hook is gone. An explicit `hook_disposed` event always expires the token immediately. +When a workflow ends, its Hooks are removed. Their tokens normally become available again, but `experimental_expires` can delay that. A `hook_disposed` event makes a token available immediately. See [Hooks & Webhooks](/docs/foundations/hooks) for more on how hooks and webhooks work. @@ -188,9 +188,9 @@ Events are categorized by the entity type they affect. Each event contains metad | Event | Description | |-------|-------------| | `hook_created` | Creates a new hook in `active` state. Contains the hook token and optional metadata. | -| `hook_conflict` | Records that hook creation failed because the token is already in use by another active hook. Contains the token and, for current worlds, the active hook owner's run ID. The hook is not created: `hook.getConflict()` resolves with the conflicting run, and awaiting the hook payload rejects with a `HookConflictError`. | +| `hook_conflict` | Records that Hook creation failed because another run owns the token. Contains the token and, for current Worlds, the owner's run ID. `hook.getConflict()` returns that run, while awaiting the Hook rejects with `HookConflictError`. | | `hook_received` | Records that a payload was delivered to the hook. The hook remains `active` and can receive more payloads. | -| `hook_disposed` | Deletes the hook from storage (conceptually transitioning to `disposed` state). The token is released for reuse by future workflows. | +| `hook_disposed` | Deletes the Hook and makes its token available to other Hooks. | ### Wait Events diff --git a/packages/core/src/create-hook.ts b/packages/core/src/create-hook.ts index 30e2292157..44059d2331 100644 --- a/packages/core/src/create-hook.ts +++ b/packages/core/src/create-hook.ts @@ -32,30 +32,16 @@ export interface Hook extends AsyncIterable, Thenable { token: string; /** - * Returns a promise that resolves with the conflicting {@link Run} if - * another Hook already owns this token, including a token kept reserved - * after its run ended, or `null` once this Hook has registered. + * Returns the {@link Run} already using this token, or `null` when this Hook + * registers successfully. * * Calling `createHook()` alone does not register the hook — registration * only happens when the workflow suspends. Awaiting `getConflict()` - * suspends the workflow to commit the hook registration, so it can be - * used to claim the token (and detect token conflicts early) without - * waiting for payload data. + * suspends the workflow to commit the hook registration without waiting for + * payload data. * - * When a conflict is detected, the resolved `Run` owns the token. It may - * still be active, or its token may remain reserved after it finished. The - * duplicate can use its run ID, status, or return value. - * - * A conflict means this Hook does not own the token. Cancelling the owner - * does not transfer ownership or free a retained token. Do not perform - * protected work until a new Hook registers without a conflict. - * - * Note that awaiting the hook's payload (`await hook`) when the token is - * already owned by another Hook or reserved by a finished run still rejects with - * `HookConflictError`. In the rare case where the conflicting run cannot - * be identified (a `hook_conflict` event persisted by an old world that - * did not record the owning run's ID), `getConflict()` also rejects with - * `HookConflictError` rather than resolving with an incomplete value. + * If it returns a run, this Hook was not created. Awaiting the Hook instead + * rejects with `HookConflictError`. * * @example * ```ts @@ -63,9 +49,9 @@ export interface Hook extends AsyncIterable, Thenable { * const conflict = await hook.getConflict(); * if (conflict) { * // another run already owns this token - * return { dedupedTo: conflict.runId }; + * return { status: 'duplicate', runId: conflict.runId }; * } - * // token is now claimed, without waiting for payload data + * // this Hook registered without waiting for payload data * ``` */ getConflict(): Promise | null>; @@ -151,23 +137,21 @@ export interface HookOptions { token?: string; /** - * **Experimental.** Sets when this Hook's token expires after its workflow - * run ends. + * **Experimental.** Sets when another Hook can use this token after the run + * ends. * * Accepts the same values as `sleep()`: a duration string, a number of * milliseconds, or an absolute `Date`. Relative durations start when * `createHook()` runs, not when the workflow ends. * - * Expiration never stops an active Hook. If the configured time passes while - * the workflow is still running, the token expires when the run ends. After - * the run ends, the Hook cannot be resumed. Another Hook using the same token - * receives a conflict until the token expires. + * The Hook remains active until the workflow ends, even if the configured + * time passes first. Another Hook can use the token only after both the run + * has ended and the configured time has passed. * - * Calling `dispose()` (including through `using`) expires the token + * Calling `dispose()` (including through `using`) releases the token * immediately. * - * World implementations may ignore this experimental option if they do not - * support token expiration. + * World implementations may ignore this experimental option. * * @example * From 03cb84782e64dddf91428081a5064dca99fb5f14 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 10 Jul 2026 13:46:52 -0700 Subject: [PATCH 14/36] docs(core): clarify Hook expiration deadline --- docs/content/docs/v5/api-reference/workflow/create-hook.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/docs/v5/api-reference/workflow/create-hook.mdx b/docs/content/docs/v5/api-reference/workflow/create-hook.mdx index bc52ea16bd..dae45e6116 100644 --- a/docs/content/docs/v5/api-reference/workflow/create-hook.mdx +++ b/docs/content/docs/v5/api-reference/workflow/create-hook.mdx @@ -187,7 +187,7 @@ The Hook remains active until the workflow ends, even if the configured time pas This option is experimental. A World that does not support it will ignore it. `createWebhook()` does not accept this option. -The token may become available late, so do not use this as an exact timer. A World may reject values beyond its supported maximum. +`experimental_expires` stores a deadline; it does not schedule work at that time. When another Hook tries to use the token, the World compares the stored deadline. A World may reject values beyond its supported maximum. ### Waiting for Multiple Payloads From ad5442ff0b69d1186f9d987b4cab959e7febe1d6 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 10 Jul 2026 13:48:52 -0700 Subject: [PATCH 15/36] docs(core): remove Hook deadline caveat --- docs/content/docs/v5/api-reference/workflow/create-hook.mdx | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/content/docs/v5/api-reference/workflow/create-hook.mdx b/docs/content/docs/v5/api-reference/workflow/create-hook.mdx index dae45e6116..7a89394306 100644 --- a/docs/content/docs/v5/api-reference/workflow/create-hook.mdx +++ b/docs/content/docs/v5/api-reference/workflow/create-hook.mdx @@ -187,8 +187,6 @@ The Hook remains active until the workflow ends, even if the configured time pas This option is experimental. A World that does not support it will ignore it. `createWebhook()` does not accept this option. -`experimental_expires` stores a deadline; it does not schedule work at that time. When another Hook tries to use the token, the World compares the stored deadline. A World may reject values beyond its supported maximum. - ### Waiting for Multiple Payloads You can also wait for multiple payloads by using the `for await...of` syntax. From 9d60f994a03236a1d510891c500e4e622b0b45d8 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 10 Jul 2026 13:52:35 -0700 Subject: [PATCH 16/36] refactor(core): align Hook expiration field names --- docs/content/docs/v5/deploying/building-a-world.mdx | 2 +- packages/core/src/global.ts | 2 +- packages/core/src/runtime/suspension-handler.test.ts | 6 +++--- packages/core/src/runtime/suspension-handler.ts | 2 +- packages/core/src/workflow/hook.test.ts | 6 +++--- packages/core/src/workflow/hook.ts | 6 +++--- packages/world-vercel/src/events-v4.ts | 8 ++++---- packages/world-vercel/src/events.test.ts | 6 +++--- packages/world-vercel/src/events.ts | 8 ++++---- packages/world/src/events.test.ts | 6 +++--- packages/world/src/events.ts | 2 +- 11 files changed, 27 insertions(+), 27 deletions(-) diff --git a/docs/content/docs/v5/deploying/building-a-world.mdx b/docs/content/docs/v5/deploying/building-a-world.mdx index aafbf7024a..8ea39a370d 100644 --- a/docs/content/docs/v5/deploying/building-a-world.mdx +++ b/docs/content/docs/v5/deploying/building-a-world.mdx @@ -95,7 +95,7 @@ interface Storage { **Hook Tokens:** Hook tokens must be unique. If a `hook_created` event conflicts with an active Hook or a token still reserved after its run ended, return a `hook_conflict` event and include the owner's run ID as `eventData.conflictingRunId`. -**Automatic Hook Cleanup:** When a run ends, remove its live Hooks. Make each token available unless its `tokenReusableAfter` is still in the future. A `hook_disposed` event always makes the token available immediately. +**Automatic Hook Cleanup:** When a run ends, remove its live Hooks. Make each token available unless its `tokenExpiresAt` is still in the future. A `hook_disposed` event always makes the token available immediately. ## Queue Interface diff --git a/packages/core/src/global.ts b/packages/core/src/global.ts index 110cf103ac..80308f8070 100644 --- a/packages/core/src/global.ts +++ b/packages/core/src/global.ts @@ -38,7 +38,7 @@ export interface HookInvocationQueueItem { correlationId: string; token: string; /** Earliest time a terminal run's token claim may be reused. */ - tokenReusableAfter?: Date; + tokenExpiresAt?: Date; metadata?: Serializable; hasCreatedEvent?: boolean; /** Whether the workflow is awaiting `hook.getConflict()` for this hook */ diff --git a/packages/core/src/runtime/suspension-handler.test.ts b/packages/core/src/runtime/suspension-handler.test.ts index d0a3468ca1..0cfb460712 100644 --- a/packages/core/src/runtime/suspension-handler.test.ts +++ b/packages/core/src/runtime/suspension-handler.test.ts @@ -35,7 +35,7 @@ describe('handleSuspension', () => { event, })); const world = createWorld(eventsCreate); - const tokenReusableAfter = new Date('2026-08-01T00:00:00.000Z'); + const tokenExpiresAt = new Date('2026-08-01T00:00:00.000Z'); const pending = new Map([ [ 'hook_retained', @@ -43,7 +43,7 @@ describe('handleSuspension', () => { type: 'hook' as const, correlationId: 'hook_retained', token: 'order:123', - tokenReusableAfter, + tokenExpiresAt, }, ], ]); @@ -60,7 +60,7 @@ describe('handleSuspension', () => { eventType: 'hook_created', eventData: expect.objectContaining({ token: 'order:123', - tokenReusableAfter, + tokenExpiresAt, }), }), expect.anything() diff --git a/packages/core/src/runtime/suspension-handler.ts b/packages/core/src/runtime/suspension-handler.ts index f771afff2b..3b98b877e7 100644 --- a/packages/core/src/runtime/suspension-handler.ts +++ b/packages/core/src/runtime/suspension-handler.ts @@ -336,7 +336,7 @@ export async function handleSuspension({ correlationId: queueItem.correlationId, eventData: { token: queueItem.token, - tokenReusableAfter: queueItem.tokenReusableAfter, + tokenExpiresAt: queueItem.tokenExpiresAt, metadata: hookMetadata, isWebhook: queueItem.isWebhook ?? false, ...(queueItem.isSystem && { isSystem: true }), diff --git a/packages/core/src/workflow/hook.test.ts b/packages/core/src/workflow/hook.test.ts index b5cbbfaefd..e10000c020 100644 --- a/packages/core/src/workflow/hook.test.ts +++ b/packages/core/src/workflow/hook.test.ts @@ -1183,7 +1183,7 @@ describe('createCreateHook', () => { const queueItem = ctx.invocationsQueue.values().next().value; expect(queueItem?.type).toBe('hook'); if (queueItem?.type === 'hook') { - expect(queueItem.tokenReusableAfter).toEqual(new Date(1_001_000)); + expect(queueItem.tokenExpiresAt).toEqual(new Date(1_001_000)); } } finally { dateNow.mockRestore(); @@ -1195,7 +1195,7 @@ describe('createCreateHook', () => { name: 'uses the persisted deadline', eventData: { token: 'test-token', - tokenReusableAfter: new Date('2026-07-15T00:00:00.000Z'), + tokenExpiresAt: new Date('2026-07-15T00:00:00.000Z'), }, expected: new Date('2026-07-15T00:00:00.000Z'), }, @@ -1226,7 +1226,7 @@ describe('createCreateHook', () => { const queueItem = ctx.invocationsQueue.values().next().value; expect(queueItem?.type).toBe('hook'); if (queueItem?.type === 'hook') { - expect(queueItem.tokenReusableAfter).toEqual(expected); + expect(queueItem.tokenExpiresAt).toEqual(expected); } }); diff --git a/packages/core/src/workflow/hook.ts b/packages/core/src/workflow/hook.ts index fd5ab48fa7..426f32781c 100644 --- a/packages/core/src/workflow/hook.ts +++ b/packages/core/src/workflow/hook.ts @@ -85,7 +85,7 @@ export function createCreateHook(ctx: WorkflowOrchestratorContext) { // Generate hook ID and token const correlationId = `hook_${ctx.generateUlid()}`; const token = options.token ?? ctx.generateNanoid(); - const tokenReusableAfter = + const tokenExpiresAt = options.experimental_expires === undefined ? undefined : parseDurationToDate(options.experimental_expires); @@ -97,7 +97,7 @@ export function createCreateHook(ctx: WorkflowOrchestratorContext) { type: 'hook', correlationId, token, - tokenReusableAfter, + tokenExpiresAt, metadata: options.metadata, isWebhook, }); @@ -183,7 +183,7 @@ export function createCreateHook(ctx: WorkflowOrchestratorContext) { // The event log is authoritative on replay. In particular, an old // event with no expiration field must not gain one merely because // newly deployed workflow code added the option. - queueItem.tokenReusableAfter = event.eventData.tokenReusableAfter; + queueItem.tokenExpiresAt = event.eventData.tokenExpiresAt; } hasCreated = true; diff --git a/packages/world-vercel/src/events-v4.ts b/packages/world-vercel/src/events-v4.ts index b2697efce3..d16462724a 100644 --- a/packages/world-vercel/src/events-v4.ts +++ b/packages/world-vercel/src/events-v4.ts @@ -126,8 +126,8 @@ export interface CreateEventV4Input { * the step entity for premature-delivery pacing and observability. */ retryAfter?: Date; hookToken?: string; - /** Earliest time a terminal hook token claim may be reused. */ - hookTokenReusableAfter?: Date; + /** Time after which another Hook can use the token. */ + hookTokenExpiresAt?: Date; hookIsWebhook?: boolean; hookIsSystem?: boolean; errorCode?: string; @@ -234,8 +234,8 @@ function buildPostFrameMeta( if (input.resumeAt !== undefined) meta.resumeAt = input.resumeAt; if (input.retryAfter !== undefined) meta.retryAfter = input.retryAfter; if (input.hookToken !== undefined) meta.hookToken = input.hookToken; - if (input.hookTokenReusableAfter !== undefined) { - meta.hookTokenReusableAfter = input.hookTokenReusableAfter; + if (input.hookTokenExpiresAt !== undefined) { + meta.hookTokenExpiresAt = input.hookTokenExpiresAt; } if (input.hookIsWebhook !== undefined) meta.hookIsWebhook = input.hookIsWebhook; diff --git a/packages/world-vercel/src/events.test.ts b/packages/world-vercel/src/events.test.ts index c976f6003a..0060dc0963 100644 --- a/packages/world-vercel/src/events.test.ts +++ b/packages/world-vercel/src/events.test.ts @@ -121,20 +121,20 @@ describe('createWorkflowRunEvent with v1Compat', () => { */ describe('splitEventDataForV4 attribute fields', () => { it('carries the hook token expiration time in the frame meta', () => { - const tokenReusableAfter = new Date('2026-07-10T12:00:00.000Z'); + const tokenExpiresAt = new Date('2026-07-10T12:00:00.000Z'); const { payload, meta } = splitEventDataForV4({ eventType: 'hook_created', correlationId: 'hook_1', specVersion: 5, eventData: { token: 'order:123', - tokenReusableAfter, + tokenExpiresAt, }, } as AnyEventRequest); expect(payload).toBeUndefined(); expect(meta.hookToken).toBe('order:123'); - expect(meta.hookTokenReusableAfter).toEqual(tokenReusableAfter); + expect(meta.hookTokenExpiresAt).toEqual(tokenExpiresAt); }); it('carries attr_set changes/writer/allowReservedAttributes in the frame meta', () => { diff --git a/packages/world-vercel/src/events.ts b/packages/world-vercel/src/events.ts index 0575d8def8..bd9c2070a5 100644 --- a/packages/world-vercel/src/events.ts +++ b/packages/world-vercel/src/events.ts @@ -108,7 +108,7 @@ interface SplitEventData { resumeAt?: Date; retryAfter?: Date; hookToken?: string; - hookTokenReusableAfter?: Date; + hookTokenExpiresAt?: Date; hookIsWebhook?: boolean; hookIsSystem?: boolean; errorCode?: string; @@ -154,7 +154,7 @@ type MetaSourceField = | 'resumeAt' | 'retryAfter' | 'token' - | 'tokenReusableAfter' + | 'tokenExpiresAt' | 'isWebhook' | 'isSystem' | 'errorCode' @@ -255,8 +255,8 @@ export function splitEventDataForV4(data: AnyEventRequest): SplitEventData { if (typeof eventData.token === 'string') { meta.hookToken = eventData.token; } - if (eventData.tokenReusableAfter instanceof Date) { - meta.hookTokenReusableAfter = eventData.tokenReusableAfter; + if (eventData.tokenExpiresAt instanceof Date) { + meta.hookTokenExpiresAt = eventData.tokenExpiresAt; } if (typeof eventData.isWebhook === 'boolean') { meta.hookIsWebhook = eventData.isWebhook; diff --git a/packages/world/src/events.test.ts b/packages/world/src/events.test.ts index 7913b45019..2c9840260c 100644 --- a/packages/world/src/events.test.ts +++ b/packages/world/src/events.test.ts @@ -2,20 +2,20 @@ import { describe, expect, it } from 'vitest'; import { CreateEventSchema, EventSchema } from './events'; describe('hook_created token expiration', () => { - it('coerces tokenReusableAfter to a Date', () => { + it('coerces tokenExpiresAt to a Date', () => { const parsed = CreateEventSchema.parse({ eventType: 'hook_created', correlationId: 'hook_1', specVersion: 5, eventData: { token: 'order:123', - tokenReusableAfter: '2026-08-01T00:00:00.000Z', + tokenExpiresAt: '2026-08-01T00:00:00.000Z', }, }); expect(parsed.eventType).toBe('hook_created'); if (parsed.eventType === 'hook_created') { - expect(parsed.eventData.tokenReusableAfter).toEqual( + expect(parsed.eventData.tokenExpiresAt).toEqual( new Date('2026-08-01T00:00:00.000Z') ); } diff --git a/packages/world/src/events.ts b/packages/world/src/events.ts index b75627ab22..b094e7e452 100644 --- a/packages/world/src/events.ts +++ b/packages/world/src/events.ts @@ -383,7 +383,7 @@ export const HookCreatedEventSchema = BaseEventSchema.extend({ correlationId: z.string(), eventData: z.object({ token: z.string(), - tokenReusableAfter: z.coerce.date().optional(), + tokenExpiresAt: z.coerce.date().optional(), metadata: SerializedDataSchema.optional(), isWebhook: z.boolean().optional(), isSystem: z.boolean().optional(), From 0deba854815808f426d53132e6ebddf045b5905b Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 10 Jul 2026 14:25:28 -0700 Subject: [PATCH 17/36] docs(core): narrow Hook expiration documentation --- .../v5/api-reference/workflow/create-hook.mdx | 4 ++-- docs/content/docs/v5/foundations/hooks.mdx | 2 +- .../docs/v5/foundations/idempotency.mdx | 22 ++++++++++--------- .../docs/v5/how-it-works/event-sourcing.mdx | 8 +++---- 4 files changed, 19 insertions(+), 17 deletions(-) diff --git a/docs/content/docs/v5/api-reference/workflow/create-hook.mdx b/docs/content/docs/v5/api-reference/workflow/create-hook.mdx index 7a89394306..6e8a999eb8 100644 --- a/docs/content/docs/v5/api-reference/workflow/create-hook.mdx +++ b/docs/content/docs/v5/api-reference/workflow/create-hook.mdx @@ -66,7 +66,7 @@ export default Hook;`} The returned `Hook` object also implements `AsyncIterable`, which allows you to iterate over incoming payloads using `for await...of` syntax. -`await hook.getConflict()` registers the Hook without waiting for a payload. It returns `null` if registration succeeds, or the [`Run`](/docs/api-reference/workflow-api/get-run) already using the token. This includes a finished run whose token is not yet reusable. +Use `hook.getConflict()` to check whether the hook token is already claimed by another hook, including one kept reserved after its run ends, without waiting for hook payload data. Calling `createHook()` on its own does not register the hook — registration is only committed when the workflow suspends. Awaiting `hook.getConflict()` suspends the workflow to commit the registration, then resolves with `null` once `hook_created` is recorded, or with the conflicting [`Run`](/docs/api-reference/workflow-api/get-run). ## Examples @@ -143,7 +143,7 @@ async function processOrder(orderId: string) { Because `createHook()` alone does not suspend the workflow, awaiting `hook.getConflict()` is what actually suspends the run and commits the hook registration. It only waits for registration — to receive payload data from a future `resumeHook()` call, await the hook itself or iterate it with `for await...of`. -If `getConflict()` returns a run, this Hook cannot use the token. Return before doing work that depends on it. See [Run idempotency](/docs/foundations/idempotency#run-idempotency) for examples. +On a conflict, the resolved value is a `Run` handle for the run that owns the token, with durable step-backed accessors. The duplicate run can decide in code how to handle it: return or log `conflict.runId`, inspect `await conflict.status`, wait on `await conflict.returnValue`, or cancel the owner with `await conflict.cancel()` and continue in the current run. See [Run idempotency](/docs/foundations/idempotency#run-idempotency) for these strategies in context. Custom hook tokens are the recommended way to coordinate active workflow runs. Use a deterministic token from your domain, such as an order ID or conversation ID, create the hook near the beginning of the workflow, and check `await hook.getConflict()` before work that depends on owning the token. See [Run idempotency](/docs/foundations/idempotency#run-idempotency). diff --git a/docs/content/docs/v5/foundations/hooks.mdx b/docs/content/docs/v5/foundations/hooks.mdx index ec541a2cc0..348f31da8c 100644 --- a/docs/content/docs/v5/foundations/hooks.mdx +++ b/docs/content/docs/v5/foundations/hooks.mdx @@ -112,7 +112,7 @@ export async function orderWorkflow(orderId: string) { } ``` -`await hook.getConflict()` registers the Hook without waiting for data. It returns `null` if registration succeeds, or the run already using the token. See [Run idempotency](/docs/foundations/idempotency#run-idempotency) for handling examples. +Calling `createHook()` on its own does not register the hook — registration is only committed when the workflow suspends. Awaiting `hook.getConflict()` suspends the workflow to commit the hook registration, then resolves with `null` once the hook is registered and ready to receive payloads, or with a `Run` handle for the run that owns the token (see [`HookConflictError`](/docs/errors/hook-conflict)). For `hook_conflict` events persisted by older worlds that did not record the owning run's ID, `getConflict()` rejects with `HookConflictError` instead of resolving with an incomplete handle. The conflicting run's accessors are durable steps, so the workflow can inspect `await conflict.status`, wait on `await conflict.returnValue`, or cancel the owner with `await conflict.cancel()` — see [Run idempotency](/docs/foundations/idempotency#run-idempotency) for these strategies. ### Custom Tokens for Deterministic Hooks diff --git a/docs/content/docs/v5/foundations/idempotency.mdx b/docs/content/docs/v5/foundations/idempotency.mdx index 7f985952cc..e4d0f4647a 100644 --- a/docs/content/docs/v5/foundations/idempotency.mdx +++ b/docs/content/docs/v5/foundations/idempotency.mdx @@ -96,7 +96,7 @@ export async function processOrder(orderId: string): Promise { } ``` -Only one Hook can use `order:${orderId}`. If two runs race, one receives a conflict and returns before it reaches `chargeOrder()`. +The runtime creates the hook atomically. At most one hook can own `order:${orderId}`, so duplicate workflow runs converge on one owner. A duplicate run observes `getConflict()` resolving with the owner's `Run` and returns before it reaches `chargeOrder()`. The conflicting run's accessors (`status`, `returnValue`, `cancel()`, …) are durable steps, so the duplicate run can do more than report the owner — see [conflict-handling strategies](#conflict-handling-strategies) below. Outside the workflow, try to resume the hook first. If the hook is not registered yet, start the workflow and retry the resume until the new run creates the hook: @@ -146,16 +146,18 @@ export async function POST(request: Request) { ``` -Calling `resumeHook()` first avoids starting a duplicate after the Hook exists. It does not close the short gap between `start()` and `createHook()`: two requests can still start two runs. The `getConflict()` check inside the workflow stops the losing run before it does duplicate work. A native API that starts a run and creates its Hook atomically is planned. +This avoids creating a new run only after the first run has registered its hook. Because `start()` returns before the run body executes and calls `createHook()`, two concurrent requests can both observe "no hook yet" and each call `start()`. The race is resolved inside the workflow body, where the losing run observes `getConflict()` resolving with the active owner and returns without doing duplicate-sensitive work — and the route detects it by comparing the resumed hook's `runId` against the run it just started, without waiting for either run to finish. A native API for atomically starting a run and registering a hook is in the works. Until then, model recovery inside the workflow by checking `hook.getConflict()`. -By default, another Hook can use the token after the workflow ends. Set `experimental_expires` to delay that. The original Hook still ends with the workflow and cannot be resumed. See [`createHook()` expiration](/docs/api-reference/workflow/create-hook#set-when-another-hook-can-use-the-token) for examples and supported values. +This coordinates active runs by default: the token becomes available when its workflow ends. Set `experimental_expires` to keep it unavailable to late duplicates. The Hook itself still ends with the workflow and cannot be resumed. See [`createHook()` expiration](/docs/api-reference/workflow/create-hook#set-when-another-hook-can-use-the-token) for examples and supported values. ### Conflict-handling strategies -The example above rejects the duplicate by returning the other run's `runId`. +Some workflow systems resolve duplicate IDs with a fixed, pre-declared policy — typically a static choice between rejecting the new execution, deferring to the existing one, or terminating it. Workflow has no policy enum. `hook.getConflict()` hands the duplicate run the conflicting `Run` itself, and the policy is ordinary code — including policies that inspect state before deciding, which static configuration can't express. -**Adopt the owner's result.** Wait for the active run to finish and return its result: +The example above implements **reject the duplicate**: return the owner's `runId` and let the caller decide. Other common strategies: + +**Adopt the owner's result.** Wait for the active run to finish and return its result, so callers cannot tell which run did the work: ```typescript lineNumbers import { createHook } from "workflow"; @@ -180,7 +182,7 @@ export async function processOrder(orderId: string) { } ``` -**Inspect the owner before deciding.** Branch on its status: +**Inspect the owner before deciding.** Branch on the owner's state: ```typescript lineNumbers import { createHook } from "workflow"; @@ -205,7 +207,7 @@ export async function processOrder(orderId: string) { } ``` -**Signal an active owner instead of doing the work.** If the other run is still active, the duplicate can send it the input. `resumeHook()` can race the other run finishing, so production code should handle `HookNotFoundError`: +**Signal the owner instead of doing the work.** A conflict can refer to a finished run when `experimental_expires` is set, so check its status before sending data to its Hook: ```typescript lineNumbers import { createHook } from "workflow"; @@ -238,7 +240,7 @@ export async function processOrder(orderId: string, confirmed: boolean) { } ``` -**Use the newest run.** With the default expiration, cancel the active run, then retry until its token becomes available: +**Supersede the owner.** With the default expiration, cancel the active run, then claim the released token. The retry loop covers the window where cancellation cleanup has not propagated yet: ```typescript lineNumbers import { createHook } from "workflow"; @@ -256,7 +258,7 @@ export async function processOrderNewestWins(orderId: string) { const conflict = await request.getConflict(); if (!conflict) { - // This run now owns the token. + // Token claimed — this run is now the owner. const { confirmed } = await request; if (confirmed) { await chargeOrder(orderId); @@ -267,7 +269,7 @@ export async function processOrderNewestWins(orderId: string) { await conflict.cancel(); // [!code highlight] } - throw new Error(`Token ${token} is still unavailable after cancelling the other run`); + throw new Error(`Could not claim ${token} after cancelling the owner`); } ``` diff --git a/docs/content/docs/v5/how-it-works/event-sourcing.mdx b/docs/content/docs/v5/how-it-works/event-sourcing.mdx index e7a6e5eac3..f20c13a238 100644 --- a/docs/content/docs/v5/how-it-works/event-sourcing.mdx +++ b/docs/content/docs/v5/how-it-works/event-sourcing.mdx @@ -123,11 +123,11 @@ flowchart TD - `active`: Ready to receive payloads (hook exists in storage) - `disposed`: No longer accepting payloads (hook is deleted from storage) -- `conflicted`: Hook creation failed because another run already owns the token +- `conflicted`: Hook creation failed because the token is already in use by another workflow Unlike other entities, hooks don't have a `status` field—the states above are conceptual. An "active" hook is one that exists in storage, while "disposed" means the hook has been deleted. When a `hook_disposed` event is created, the hook record is removed rather than updated. -While a Hook is active, its token is reserved. If another run tries to register the same token, a `hook_conflict` event is recorded instead of `hook_created`. The same conflict occurs after the owning run ends when `experimental_expires` keeps its token reserved. `hook.getConflict()` returns the owning run, while awaiting the Hook rejects with `HookConflictError`. See the [hook-conflict error](/docs/errors/hook-conflict) documentation for more details. +While a hook is active, its token is reserved and cannot be used by other workflows. If a workflow attempts to create a hook with a token reserved by another run — either by an active hook or by `experimental_expires` after its run ended — a `hook_conflict` event is recorded instead of `hook_created`. Current worlds include the token and the run ID that owns it, though older persisted events or world implementations may only include the token. This causes `hook.getConflict()` to resolve with the conflicting run and the hook's payload promise to reject with a `HookConflictError`, which you can detect with `HookConflictError.is(error)`. See the [hook-conflict error](/docs/errors/hook-conflict) documentation for more details. When a workflow ends, its Hooks are removed. Their tokens normally become available again, but `experimental_expires` can delay that. A `hook_disposed` event makes a token available immediately. @@ -188,9 +188,9 @@ Events are categorized by the entity type they affect. Each event contains metad | Event | Description | |-------|-------------| | `hook_created` | Creates a new hook in `active` state. Contains the hook token and optional metadata. | -| `hook_conflict` | Records that Hook creation failed because another run owns the token. Contains the token and, for current Worlds, the owner's run ID. `hook.getConflict()` returns that run, while awaiting the Hook rejects with `HookConflictError`. | +| `hook_conflict` | Records that hook creation failed because another run owns the token. Contains the token and, for current worlds, the owner's run ID. The hook is not created: `hook.getConflict()` resolves with the conflicting run, and awaiting the hook payload rejects with a `HookConflictError`. | | `hook_received` | Records that a payload was delivered to the hook. The hook remains `active` and can receive more payloads. | -| `hook_disposed` | Deletes the Hook and makes its token available to other Hooks. | +| `hook_disposed` | Deletes the hook from storage (conceptually transitioning to `disposed` state). The token is released for reuse by future workflows. | ### Wait Events From ecd2c61f4f5839a745ec3fdcaae4c8f1c72cd572 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Mon, 13 Jul 2026 10:52:42 -0700 Subject: [PATCH 18/36] docs(core): clarify hook expiration availability --- docs/content/docs/v5/api-reference/workflow/create-hook.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/docs/v5/api-reference/workflow/create-hook.mdx b/docs/content/docs/v5/api-reference/workflow/create-hook.mdx index 6e8a999eb8..a9391cd4a0 100644 --- a/docs/content/docs/v5/api-reference/workflow/create-hook.mdx +++ b/docs/content/docs/v5/api-reference/workflow/create-hook.mdx @@ -184,7 +184,7 @@ The Hook remains active until the workflow ends, even if the configured time pas `hook.dispose()` releases the token immediately. Because `using` calls `dispose()`, do not use it when the token should remain unavailable after the run ends. -This option is experimental. A World that does not support it will ignore it. `createWebhook()` does not accept this option. +This option is experimental and is not currently supported by any built-in World. Worlds that do not support it ignore it. `createWebhook()` does not accept this option. ### Waiting for Multiple Payloads From 7eee23bd00d8637facb4461ffdc225c3fc322cbc Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Tue, 14 Jul 2026 12:43:53 -0700 Subject: [PATCH 19/36] Update packages/core/src/workflow/hook.ts Co-authored-by: Peter Wielander Signed-off-by: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> --- packages/core/src/workflow/hook.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/core/src/workflow/hook.ts b/packages/core/src/workflow/hook.ts index 426f32781c..d45c8633aa 100644 --- a/packages/core/src/workflow/hook.ts +++ b/packages/core/src/workflow/hook.ts @@ -180,9 +180,6 @@ export function createCreateHook(ctx: WorkflowOrchestratorContext) { const queueItem = ctx.invocationsQueue.get(correlationId); if (queueItem && queueItem.type === 'hook') { queueItem.hasCreatedEvent = true; - // The event log is authoritative on replay. In particular, an old - // event with no expiration field must not gain one merely because - // newly deployed workflow code added the option. queueItem.tokenExpiresAt = event.eventData.tokenExpiresAt; } hasCreated = true; From e65f1d29214db560b46d6c57562fe3e3045d7c9e Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Tue, 14 Jul 2026 12:58:18 -0700 Subject: [PATCH 20/36] docs(core): clarify Hook token expiration behavior --- .../docs/v5/api-reference/workflow/create-hook.mdx | 2 +- packages/core/src/create-hook.ts | 12 ++++-------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/docs/content/docs/v5/api-reference/workflow/create-hook.mdx b/docs/content/docs/v5/api-reference/workflow/create-hook.mdx index a9391cd4a0..4f0c91789a 100644 --- a/docs/content/docs/v5/api-reference/workflow/create-hook.mdx +++ b/docs/content/docs/v5/api-reference/workflow/create-hook.mdx @@ -179,7 +179,7 @@ export async function processOrder(orderId: string) { `experimental_expires` accepts the same values as [`sleep()`](/docs/api-reference/workflow/sleep): a duration string such as `"30d"`, a number of milliseconds, or an absolute `Date`. Durations start when `createHook()` runs. -The Hook remains active until the workflow ends, even if the configured time passes first. Another Hook can use the token only after both the workflow has ended and the configured time has passed. For example, `"30d"` keeps the token unavailable for 29 more days if the workflow ends after 1 day. A workflow that runs for more than 30 days releases the token when it ends. +`experimental_expires` keeps the token unavailable until both the workflow has ended and the configured expiration time has passed. It does not dispose an active Hook. For example, `"30d"` keeps the token unavailable for 29 more days if the workflow ends after 1 day. A workflow that runs for more than 30 days releases the token when it ends. `hook.dispose()` releases the token immediately. Because `using` calls `dispose()`, do not use it when the token should remain unavailable after the run ends. diff --git a/packages/core/src/create-hook.ts b/packages/core/src/create-hook.ts index 44059d2331..cb609b8c96 100644 --- a/packages/core/src/create-hook.ts +++ b/packages/core/src/create-hook.ts @@ -137,19 +137,15 @@ export interface HookOptions { token?: string; /** - * **Experimental.** Sets when another Hook can use this token after the run - * ends. + * **Experimental.** Keeps this Hook's token unavailable until both the run + * has ended and the configured expiration time has passed. * * Accepts the same values as `sleep()`: a duration string, a number of * milliseconds, or an absolute `Date`. Relative durations start when * `createHook()` runs, not when the workflow ends. * - * The Hook remains active until the workflow ends, even if the configured - * time passes first. Another Hook can use the token only after both the run - * has ended and the configured time has passed. - * - * Calling `dispose()` (including through `using`) releases the token - * immediately. + * This does not dispose an active Hook. Calling `dispose()` (including + * through `using`) releases the token immediately. * * World implementations may ignore this experimental option. * From 8d9923b5d79f756b40dd0be623294857eb61f23a Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Tue, 14 Jul 2026 13:04:54 -0700 Subject: [PATCH 21/36] docs(core): explain active Hook expiration behavior --- .../docs/v5/api-reference/workflow/create-hook.mdx | 2 +- packages/core/src/create-hook.ts | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/content/docs/v5/api-reference/workflow/create-hook.mdx b/docs/content/docs/v5/api-reference/workflow/create-hook.mdx index 4f0c91789a..9d35d89a50 100644 --- a/docs/content/docs/v5/api-reference/workflow/create-hook.mdx +++ b/docs/content/docs/v5/api-reference/workflow/create-hook.mdx @@ -179,7 +179,7 @@ export async function processOrder(orderId: string) { `experimental_expires` accepts the same values as [`sleep()`](/docs/api-reference/workflow/sleep): a duration string such as `"30d"`, a number of milliseconds, or an absolute `Date`. Durations start when `createHook()` runs. -`experimental_expires` keeps the token unavailable until both the workflow has ended and the configured expiration time has passed. It does not dispose an active Hook. For example, `"30d"` keeps the token unavailable for 29 more days if the workflow ends after 1 day. A workflow that runs for more than 30 days releases the token when it ends. +The Hook remains active until the workflow ends, even if the configured expiration time passes first. Another Hook can use the token only after both the workflow has ended and the configured time has passed. For example, `"30d"` keeps the token unavailable for 29 more days if the workflow ends after 1 day. A workflow that runs for more than 30 days releases the token when it ends. `hook.dispose()` releases the token immediately. Because `using` calls `dispose()`, do not use it when the token should remain unavailable after the run ends. diff --git a/packages/core/src/create-hook.ts b/packages/core/src/create-hook.ts index cb609b8c96..e0197205cb 100644 --- a/packages/core/src/create-hook.ts +++ b/packages/core/src/create-hook.ts @@ -144,8 +144,12 @@ export interface HookOptions { * milliseconds, or an absolute `Date`. Relative durations start when * `createHook()` runs, not when the workflow ends. * - * This does not dispose an active Hook. Calling `dispose()` (including - * through `using`) releases the token immediately. + * The Hook remains active until the workflow ends, even if the configured + * time passes first. Another Hook can use the token only after both the run + * has ended and the configured time has passed. + * + * Calling `dispose()` (including through `using`) releases the token + * immediately. * * World implementations may ignore this experimental option. * From d6ebc679cd88d19c4c06157678b27d68cf090141 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Wed, 15 Jul 2026 16:44:26 -0700 Subject: [PATCH 22/36] feat(world): advertise hook ttl capability --- .changeset/hook-token-expiration.md | 2 +- .../v5/api-reference/workflow/create-hook.mdx | 2 +- .../docs/v5/deploying/building-a-world.mdx | 9 +++- packages/core/src/create-hook.ts | 3 +- .../src/runtime/suspension-handler.test.ts | 43 ++++++++++++++++++- .../core/src/runtime/suspension-handler.ts | 9 ++++ packages/world/src/interfaces.ts | 10 +++++ 7 files changed, 72 insertions(+), 6 deletions(-) diff --git a/.changeset/hook-token-expiration.md b/.changeset/hook-token-expiration.md index 282a570fed..115824cea3 100644 --- a/.changeset/hook-token-expiration.md +++ b/.changeset/hook-token-expiration.md @@ -4,4 +4,4 @@ '@workflow/world-vercel': minor --- -Add `experimental_expires` for keeping a Hook token unavailable after its run ends, and forward the absolute expiration time through workflow events. +Add `experimental_expires` for keeping a Hook token unavailable after its run ends, and require supporting Worlds to advertise the `hookTtl` capability. diff --git a/docs/content/docs/v5/api-reference/workflow/create-hook.mdx b/docs/content/docs/v5/api-reference/workflow/create-hook.mdx index 9d35d89a50..67065f309a 100644 --- a/docs/content/docs/v5/api-reference/workflow/create-hook.mdx +++ b/docs/content/docs/v5/api-reference/workflow/create-hook.mdx @@ -184,7 +184,7 @@ The Hook remains active until the workflow ends, even if the configured expirati `hook.dispose()` releases the token immediately. Because `using` calls `dispose()`, do not use it when the token should remain unavailable after the run ends. -This option is experimental and is not currently supported by any built-in World. Worlds that do not support it ignore it. `createWebhook()` does not accept this option. +This option is experimental. If the configured World does not support it, the workflow fails when registering the Hook. `createWebhook()` does not accept this option. ### Waiting for Multiple Payloads diff --git a/docs/content/docs/v5/deploying/building-a-world.mdx b/docs/content/docs/v5/deploying/building-a-world.mdx index 8ea39a370d..8a6c1efdf0 100644 --- a/docs/content/docs/v5/deploying/building-a-world.mdx +++ b/docs/content/docs/v5/deploying/building-a-world.mdx @@ -32,7 +32,14 @@ A World connects workflows to the infrastructure that powers them. The World int {/* @skip-typecheck - interface definition, not runnable code */} ```typescript +interface WorldCapabilities { + hookTtl?: { + active: boolean; + }; +} + interface World extends Storage, Queue, Streamer { + capabilities?: WorldCapabilities; start?(): Promise; close?(): Promise; getEncryptionKeyForRun?(run: WorkflowRun): Promise; @@ -40,7 +47,7 @@ interface World extends Storage, Queue, Streamer { } ``` -The optional `start()` method initializes background tasks (for example, queue polling). The optional `close()` method releases resources like connection pools and listeners. The optional `getEncryptionKeyForRun()` method returns the AES-256 key used to encrypt data for a run; if it is not implemented, encryption is disabled. +The optional `capabilities` object advertises additional behavior. Set `hookTtl.active` to `true` only when the World implements Hook token expiration. The optional `start()` method initializes background tasks (for example, queue polling). The optional `close()` method releases resources like connection pools and listeners. The optional `getEncryptionKeyForRun()` method returns the AES-256 key used to encrypt data for a run; if it is not implemented, encryption is disabled. ## The Event Log Model diff --git a/packages/core/src/create-hook.ts b/packages/core/src/create-hook.ts index e0197205cb..e85572dd95 100644 --- a/packages/core/src/create-hook.ts +++ b/packages/core/src/create-hook.ts @@ -151,7 +151,8 @@ export interface HookOptions { * Calling `dispose()` (including through `using`) releases the token * immediately. * - * World implementations may ignore this experimental option. + * The workflow fails when registering the Hook if its World does not support + * this experimental option. * * @example * diff --git a/packages/core/src/runtime/suspension-handler.test.ts b/packages/core/src/runtime/suspension-handler.test.ts index 0cfb460712..516aa7d624 100644 --- a/packages/core/src/runtime/suspension-handler.test.ts +++ b/packages/core/src/runtime/suspension-handler.test.ts @@ -20,8 +20,12 @@ const run: WorkflowRun = { deploymentId: 'test-deployment', }; -function createWorld(eventsCreate: ReturnType): World { +function createWorld( + eventsCreate: ReturnType, + capabilities?: World['capabilities'] +): World { return { + capabilities, events: { create: eventsCreate, }, @@ -34,7 +38,9 @@ describe('handleSuspension', () => { const eventsCreate = vi.fn().mockImplementation(async (_runId, event) => ({ event, })); - const world = createWorld(eventsCreate); + const world = createWorld(eventsCreate, { + hookTtl: { active: true }, + }); const tokenExpiresAt = new Date('2026-08-01T00:00:00.000Z'); const pending = new Map([ [ @@ -67,6 +73,39 @@ describe('handleSuspension', () => { ); }); + it.each([ + ['missing', undefined], + ['inactive', { hookTtl: { active: false } }], + ] satisfies [ + string, + World['capabilities'], + ][])('rejects hook expiration when the capability is %s', async (_state, capabilities) => { + const eventsCreate = vi.fn(); + const world = createWorld(eventsCreate, capabilities); + const pending = new Map([ + [ + 'hook_expiring', + { + type: 'hook' as const, + correlationId: 'hook_expiring', + token: 'order:123', + tokenExpiresAt: new Date('2026-08-01T00:00:00.000Z'), + }, + ], + ]); + + await expect( + handleSuspension({ + suspension: new WorkflowSuspension(pending, globalThis), + world, + run, + }) + ).rejects.toThrow( + 'The configured World does not support `experimental_expires` for Hooks.' + ); + expect(eventsCreate).not.toHaveBeenCalled(); + }); + it('marks hook.getConflict()-awaited creations without converting them into wait timeouts', async () => { const eventsCreate = vi.fn().mockResolvedValue({ event: { diff --git a/packages/core/src/runtime/suspension-handler.ts b/packages/core/src/runtime/suspension-handler.ts index 2228b8d6ee..50a45e5f7d 100644 --- a/packages/core/src/runtime/suspension-handler.ts +++ b/packages/core/src/runtime/suspension-handler.ts @@ -128,6 +128,15 @@ async function createHookEvent({ hasHookConflict: boolean; hasAwaitedHookCreation: boolean; }> { + if ( + queueItem.tokenExpiresAt !== undefined && + world.capabilities?.hookTtl?.active !== true + ) { + throw new FatalError( + 'The configured World does not support `experimental_expires` for Hooks.' + ); + } + try { const result = await world.events.create(runId, hookEvent, { requestId, diff --git a/packages/world/src/interfaces.ts b/packages/world/src/interfaces.ts index cc312f5536..c61b5c5fb8 100644 --- a/packages/world/src/interfaces.ts +++ b/packages/world/src/interfaces.ts @@ -274,6 +274,13 @@ export interface Storage { /** * The "World" interface represents how Workflows are able to communicate with the outside world. */ +export interface WorldCapabilities { + /** Supports `experimental_expires` for Hooks. */ + hookTtl?: { + active: boolean; + }; +} + export interface World extends Queue, Streamer, Storage { /** * Optional analytics read namespace for observability surfaces. @@ -293,6 +300,9 @@ export interface World extends Queue, Streamer, Storage { */ specVersion: number; + /** Optional features supported by this World. */ + capabilities?: WorldCapabilities; + /** * Whether calling `process.exit(1)` from a queue handler is observed by * the World as a delivery failure that will be retried. From e560b350ed641c11723561dde727a9c92e836926 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Wed, 15 Jul 2026 17:10:36 -0700 Subject: [PATCH 23/36] fix(core): validate hook ttl capability after main merge --- .../core/src/runtime/suspension-handler.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/core/src/runtime/suspension-handler.ts b/packages/core/src/runtime/suspension-handler.ts index 3b7f5a2211..7f660cee69 100644 --- a/packages/core/src/runtime/suspension-handler.ts +++ b/packages/core/src/runtime/suspension-handler.ts @@ -142,15 +142,6 @@ async function createHookEvent({ hasHookConflict: boolean; hasAwaitedHookCreation: boolean; }> { - if ( - queueItem.tokenExpiresAt !== undefined && - world.capabilities?.hookTtl?.active !== true - ) { - throw new FatalError( - 'The configured World does not support `experimental_expires` for Hooks.' - ); - } - try { const result = await createEvent(hookEvent, { requestId, @@ -269,6 +260,15 @@ export async function handleSuspension({ (item) => !item.hasCreatedEvent ); + if ( + hooksNeedingCreation.some((item) => item.tokenExpiresAt !== undefined) && + world.capabilities?.hookTtl?.active !== true + ) { + throw new FatalError( + 'The configured World does not support `experimental_expires` for Hooks.' + ); + } + // Group hook items that need work by token, preserving queue-insertion // (workflow code) order within each token. Operations on one token must // apply in code order: a dispose() of an earlier hook releases the token From d45f8b00d19ba3ca2a38fe3cf924bcbf6e049321 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Thu, 16 Jul 2026 13:15:51 -0700 Subject: [PATCH 24/36] refactor(core): rename hook expiry to minimum retention --- .changeset/hook-min-retention.md | 7 +++++ .changeset/hook-token-expiration.md | 7 ----- .../v5/api-reference/workflow/create-hook.mdx | 10 +++---- .../docs/v5/deploying/building-a-world.mdx | 6 ++--- .../docs/v5/foundations/idempotency.mdx | 8 +++--- .../docs/v5/how-it-works/event-sourcing.mdx | 4 +-- packages/core/src/create-hook.ts | 13 ++++++---- packages/core/src/global.ts | 4 +-- .../src/runtime/suspension-handler.test.ts | 26 +++++++++---------- .../core/src/runtime/suspension-handler.ts | 10 ++++--- packages/core/src/workflow/hook.test.ts | 22 ++++++++-------- packages/core/src/workflow/hook.ts | 14 +++++----- packages/world-vercel/src/events-v4.ts | 8 +++--- packages/world-vercel/src/events.test.ts | 8 +++--- packages/world-vercel/src/events.ts | 8 +++--- packages/world/src/events.test.ts | 8 +++--- packages/world/src/events.ts | 2 +- packages/world/src/interfaces.ts | 4 +-- 18 files changed, 87 insertions(+), 82 deletions(-) create mode 100644 .changeset/hook-min-retention.md delete mode 100644 .changeset/hook-token-expiration.md diff --git a/.changeset/hook-min-retention.md b/.changeset/hook-min-retention.md new file mode 100644 index 0000000000..460134c120 --- /dev/null +++ b/.changeset/hook-min-retention.md @@ -0,0 +1,7 @@ +--- +'@workflow/core': minor +'@workflow/world': minor +'@workflow/world-vercel': minor +--- + +Add `experimental_minRetention` for keeping a Hook token unavailable after its run ends, and require supporting Worlds to advertise the `hookRetention` capability. diff --git a/.changeset/hook-token-expiration.md b/.changeset/hook-token-expiration.md deleted file mode 100644 index 115824cea3..0000000000 --- a/.changeset/hook-token-expiration.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -'@workflow/core': minor -'@workflow/world': minor -'@workflow/world-vercel': minor ---- - -Add `experimental_expires` for keeping a Hook token unavailable after its run ends, and require supporting Worlds to advertise the `hookTtl` capability. diff --git a/docs/content/docs/v5/api-reference/workflow/create-hook.mdx b/docs/content/docs/v5/api-reference/workflow/create-hook.mdx index 67065f309a..8596996f6e 100644 --- a/docs/content/docs/v5/api-reference/workflow/create-hook.mdx +++ b/docs/content/docs/v5/api-reference/workflow/create-hook.mdx @@ -149,9 +149,9 @@ On a conflict, the resolved value is a `Run` handle for the run that owns the to Custom hook tokens are the recommended way to coordinate active workflow runs. Use a deterministic token from your domain, such as an order ID or conversation ID, create the hook near the beginning of the workflow, and check `await hook.getConflict()` before work that depends on owning the token. See [Run idempotency](/docs/foundations/idempotency#run-idempotency). -### Set When Another Hook Can Use the Token +### Keep a Token Unavailable After the Run Ends -By default, another Hook can use the token after its workflow ends. Set `experimental_expires` to choose a later time: +By default, another Hook can use the token after its workflow ends. Set `experimental_minRetention` to keep the token unavailable for at least a specific time after `createHook()` runs: ```typescript lineNumbers import { createHook } from "workflow"; @@ -164,7 +164,7 @@ export async function processOrder(orderId: string) { // Do not use `using` here. It calls dispose() and releases the token. const hook = createHook({ // [!code highlight] token: `order:${orderId}`, // [!code highlight] - experimental_expires: "30d", // [!code highlight] + experimental_minRetention: "30d", // [!code highlight] }); // [!code highlight] const conflict = await hook.getConflict(); @@ -177,9 +177,9 @@ export async function processOrder(orderId: string) { } ``` -`experimental_expires` accepts the same values as [`sleep()`](/docs/api-reference/workflow/sleep): a duration string such as `"30d"`, a number of milliseconds, or an absolute `Date`. Durations start when `createHook()` runs. +`experimental_minRetention` accepts the same values as [`sleep()`](/docs/api-reference/workflow/sleep): a duration string such as `"30d"`, a number of milliseconds, or an absolute `Date`. Durations start when `createHook()` runs. -The Hook remains active until the workflow ends, even if the configured expiration time passes first. Another Hook can use the token only after both the workflow has ended and the configured time has passed. For example, `"30d"` keeps the token unavailable for 29 more days if the workflow ends after 1 day. A workflow that runs for more than 30 days releases the token when it ends. +The Hook remains active until the workflow ends, even if the configured time passes first. Another Hook can use the token only after both the workflow has ended and the configured time has passed. For example, `"30d"` keeps the token unavailable for 29 more days if the workflow ends after 1 day. A workflow that runs for more than 30 days releases the token when it ends. `hook.dispose()` releases the token immediately. Because `using` calls `dispose()`, do not use it when the token should remain unavailable after the run ends. diff --git a/docs/content/docs/v5/deploying/building-a-world.mdx b/docs/content/docs/v5/deploying/building-a-world.mdx index 8a6c1efdf0..3082ff545e 100644 --- a/docs/content/docs/v5/deploying/building-a-world.mdx +++ b/docs/content/docs/v5/deploying/building-a-world.mdx @@ -33,7 +33,7 @@ A World connects workflows to the infrastructure that powers them. The World int {/* @skip-typecheck - interface definition, not runnable code */} ```typescript interface WorldCapabilities { - hookTtl?: { + hookRetention?: { active: boolean; }; } @@ -47,7 +47,7 @@ interface World extends Storage, Queue, Streamer { } ``` -The optional `capabilities` object advertises additional behavior. Set `hookTtl.active` to `true` only when the World implements Hook token expiration. The optional `start()` method initializes background tasks (for example, queue polling). The optional `close()` method releases resources like connection pools and listeners. The optional `getEncryptionKeyForRun()` method returns the AES-256 key used to encrypt data for a run; if it is not implemented, encryption is disabled. +The optional `capabilities` object advertises additional behavior. Set `hookRetention.active` to `true` only when the World implements Hook token retention. The optional `start()` method initializes background tasks (for example, queue polling). The optional `close()` method releases resources like connection pools and listeners. The optional `getEncryptionKeyForRun()` method returns the AES-256 key used to encrypt data for a run; if it is not implemented, encryption is disabled. ## The Event Log Model @@ -102,7 +102,7 @@ interface Storage { **Hook Tokens:** Hook tokens must be unique. If a `hook_created` event conflicts with an active Hook or a token still reserved after its run ended, return a `hook_conflict` event and include the owner's run ID as `eventData.conflictingRunId`. -**Automatic Hook Cleanup:** When a run ends, remove its live Hooks. Make each token available unless its `tokenExpiresAt` is still in the future. A `hook_disposed` event always makes the token available immediately. +**Automatic Hook Cleanup:** When a run ends, remove its live Hooks. Make each token available unless its `tokenRetentionUntil` is still in the future. A `hook_disposed` event always makes the token available immediately. ## Queue Interface diff --git a/docs/content/docs/v5/foundations/idempotency.mdx b/docs/content/docs/v5/foundations/idempotency.mdx index e4d0f4647a..c892929075 100644 --- a/docs/content/docs/v5/foundations/idempotency.mdx +++ b/docs/content/docs/v5/foundations/idempotency.mdx @@ -149,7 +149,7 @@ export async function POST(request: Request) { This avoids creating a new run only after the first run has registered its hook. Because `start()` returns before the run body executes and calls `createHook()`, two concurrent requests can both observe "no hook yet" and each call `start()`. The race is resolved inside the workflow body, where the losing run observes `getConflict()` resolving with the active owner and returns without doing duplicate-sensitive work — and the route detects it by comparing the resumed hook's `runId` against the run it just started, without waiting for either run to finish. A native API for atomically starting a run and registering a hook is in the works. Until then, model recovery inside the workflow by checking `hook.getConflict()`. -This coordinates active runs by default: the token becomes available when its workflow ends. Set `experimental_expires` to keep it unavailable to late duplicates. The Hook itself still ends with the workflow and cannot be resumed. See [`createHook()` expiration](/docs/api-reference/workflow/create-hook#set-when-another-hook-can-use-the-token) for examples and supported values. +This coordinates active runs by default: the token becomes available when its workflow ends. Set `experimental_minRetention` to keep it unavailable to late duplicates. The Hook itself still ends with the workflow and cannot be resumed. See [`createHook()` minimum retention](/docs/api-reference/workflow/create-hook#keep-a-token-unavailable-after-the-run-ends) for examples and supported values. ### Conflict-handling strategies @@ -207,7 +207,7 @@ export async function processOrder(orderId: string) { } ``` -**Signal the owner instead of doing the work.** A conflict can refer to a finished run when `experimental_expires` is set, so check its status before sending data to its Hook: +**Signal the owner instead of doing the work.** A conflict can refer to a finished run when `experimental_minRetention` is set, so check its status before sending data to its Hook: ```typescript lineNumbers import { createHook } from "workflow"; @@ -240,7 +240,7 @@ export async function processOrder(orderId: string, confirmed: boolean) { } ``` -**Supersede the owner.** With the default expiration, cancel the active run, then claim the released token. The retry loop covers the window where cancellation cleanup has not propagated yet: +**Supersede the owner.** Without minimum retention, cancel the active run, then claim the released token. The retry loop covers the window where cancellation cleanup has not propagated yet: ```typescript lineNumbers import { createHook } from "workflow"; @@ -274,7 +274,7 @@ export async function processOrderNewestWins(orderId: string) { ``` -This pattern does not work with `experimental_expires`: cancelling the old run does not make its token available early. +This pattern does not work with `experimental_minRetention`: cancelling the old run does not make its token available early. If duplicate requests should only reuse the active run without sending data, use [`getHookByToken()`](/docs/api-reference/workflow-api/get-hook-by-token) as an advisory pre-check before calling `start()`. The workflow should still check `hook.getConflict()`, because the lookup and `start()` are not atomic. diff --git a/docs/content/docs/v5/how-it-works/event-sourcing.mdx b/docs/content/docs/v5/how-it-works/event-sourcing.mdx index f20c13a238..6bf4014dd3 100644 --- a/docs/content/docs/v5/how-it-works/event-sourcing.mdx +++ b/docs/content/docs/v5/how-it-works/event-sourcing.mdx @@ -127,9 +127,9 @@ flowchart TD Unlike other entities, hooks don't have a `status` field—the states above are conceptual. An "active" hook is one that exists in storage, while "disposed" means the hook has been deleted. When a `hook_disposed` event is created, the hook record is removed rather than updated. -While a hook is active, its token is reserved and cannot be used by other workflows. If a workflow attempts to create a hook with a token reserved by another run — either by an active hook or by `experimental_expires` after its run ended — a `hook_conflict` event is recorded instead of `hook_created`. Current worlds include the token and the run ID that owns it, though older persisted events or world implementations may only include the token. This causes `hook.getConflict()` to resolve with the conflicting run and the hook's payload promise to reject with a `HookConflictError`, which you can detect with `HookConflictError.is(error)`. See the [hook-conflict error](/docs/errors/hook-conflict) documentation for more details. +While a hook is active, its token is reserved and cannot be used by other workflows. If a workflow attempts to create a hook with a token reserved by another run — either by an active hook or by `experimental_minRetention` after its run ended — a `hook_conflict` event is recorded instead of `hook_created`. Current worlds include the token and the run ID that owns it, though older persisted events or world implementations may only include the token. This causes `hook.getConflict()` to resolve with the conflicting run and the hook's payload promise to reject with a `HookConflictError`, which you can detect with `HookConflictError.is(error)`. See the [hook-conflict error](/docs/errors/hook-conflict) documentation for more details. -When a workflow ends, its Hooks are removed. Their tokens normally become available again, but `experimental_expires` can delay that. A `hook_disposed` event makes a token available immediately. +When a workflow ends, its Hooks are removed. Their tokens normally become available again, but `experimental_minRetention` can delay that. A `hook_disposed` event makes a token available immediately. See [Hooks & Webhooks](/docs/foundations/hooks) for more on how hooks and webhooks work. diff --git a/packages/core/src/create-hook.ts b/packages/core/src/create-hook.ts index e85572dd95..c956563d20 100644 --- a/packages/core/src/create-hook.ts +++ b/packages/core/src/create-hook.ts @@ -137,8 +137,8 @@ export interface HookOptions { token?: string; /** - * **Experimental.** Keeps this Hook's token unavailable until both the run - * has ended and the configured expiration time has passed. + * **Experimental.** Keeps this Hook's token unavailable for at least the + * configured time after `createHook()` runs. * * Accepts the same values as `sleep()`: a duration string, a number of * milliseconds, or an absolute `Date`. Relative durations start when @@ -159,11 +159,11 @@ export interface HookOptions { * ```ts * const hook = createHook({ * token: `order:${orderId}`, - * experimental_expires: '30d', + * experimental_minRetention: '30d', * }); * ``` */ - experimental_expires?: StringValue | Date | number; + experimental_minRetention?: StringValue | Date | number; /** * Additional user-defined data to include with the hook payload. @@ -197,7 +197,10 @@ export interface HookOptions { } export interface WebhookOptions - extends Omit { + extends Omit< + HookOptions, + 'token' | 'isWebhook' | 'experimental_minRetention' + > { /** * If set to a `Response` object, the webhook will automatically * respond with the specified response. diff --git a/packages/core/src/global.ts b/packages/core/src/global.ts index 80308f8070..d54e365a5c 100644 --- a/packages/core/src/global.ts +++ b/packages/core/src/global.ts @@ -37,8 +37,8 @@ export interface HookInvocationQueueItem { type: 'hook'; correlationId: string; token: string; - /** Earliest time a terminal run's token claim may be reused. */ - tokenExpiresAt?: Date; + /** Earliest time the token can be reused after the run ends. */ + tokenRetentionUntil?: Date; metadata?: Serializable; hasCreatedEvent?: boolean; /** Whether the workflow is awaiting `hook.getConflict()` for this hook */ diff --git a/packages/core/src/runtime/suspension-handler.test.ts b/packages/core/src/runtime/suspension-handler.test.ts index 516aa7d624..4ebbe13760 100644 --- a/packages/core/src/runtime/suspension-handler.test.ts +++ b/packages/core/src/runtime/suspension-handler.test.ts @@ -34,22 +34,22 @@ function createWorld( } describe('handleSuspension', () => { - it('persists when a hook token becomes reusable on hook_created', async () => { + it('persists the token retention deadline on hook_created', async () => { const eventsCreate = vi.fn().mockImplementation(async (_runId, event) => ({ event, })); const world = createWorld(eventsCreate, { - hookTtl: { active: true }, + hookRetention: { active: true }, }); - const tokenExpiresAt = new Date('2026-08-01T00:00:00.000Z'); + const tokenRetentionUntil = new Date('2026-08-01T00:00:00.000Z'); const pending = new Map([ [ - 'hook_retained', + 'hook_with_retention', { type: 'hook' as const, - correlationId: 'hook_retained', + correlationId: 'hook_with_retention', token: 'order:123', - tokenExpiresAt, + tokenRetentionUntil, }, ], ]); @@ -66,7 +66,7 @@ describe('handleSuspension', () => { eventType: 'hook_created', eventData: expect.objectContaining({ token: 'order:123', - tokenExpiresAt, + tokenRetentionUntil, }), }), expect.anything() @@ -75,21 +75,21 @@ describe('handleSuspension', () => { it.each([ ['missing', undefined], - ['inactive', { hookTtl: { active: false } }], + ['inactive', { hookRetention: { active: false } }], ] satisfies [ string, World['capabilities'], - ][])('rejects hook expiration when the capability is %s', async (_state, capabilities) => { + ][])('rejects hook retention when the capability is %s', async (_state, capabilities) => { const eventsCreate = vi.fn(); const world = createWorld(eventsCreate, capabilities); const pending = new Map([ [ - 'hook_expiring', + 'hook_with_retention', { type: 'hook' as const, - correlationId: 'hook_expiring', + correlationId: 'hook_with_retention', token: 'order:123', - tokenExpiresAt: new Date('2026-08-01T00:00:00.000Z'), + tokenRetentionUntil: new Date('2026-08-01T00:00:00.000Z'), }, ], ]); @@ -101,7 +101,7 @@ describe('handleSuspension', () => { run, }) ).rejects.toThrow( - 'The configured World does not support `experimental_expires` for Hooks.' + 'The configured World does not support `experimental_minRetention` for Hooks.' ); expect(eventsCreate).not.toHaveBeenCalled(); }); diff --git a/packages/core/src/runtime/suspension-handler.ts b/packages/core/src/runtime/suspension-handler.ts index 7f660cee69..c1c979063f 100644 --- a/packages/core/src/runtime/suspension-handler.ts +++ b/packages/core/src/runtime/suspension-handler.ts @@ -261,11 +261,13 @@ export async function handleSuspension({ ); if ( - hooksNeedingCreation.some((item) => item.tokenExpiresAt !== undefined) && - world.capabilities?.hookTtl?.active !== true + hooksNeedingCreation.some( + (item) => item.tokenRetentionUntil !== undefined + ) && + world.capabilities?.hookRetention?.active !== true ) { throw new FatalError( - 'The configured World does not support `experimental_expires` for Hooks.' + 'The configured World does not support `experimental_minRetention` for Hooks.' ); } @@ -378,7 +380,7 @@ export async function handleSuspension({ correlationId: queueItem.correlationId, eventData: { token: queueItem.token, - tokenExpiresAt: queueItem.tokenExpiresAt, + tokenRetentionUntil: queueItem.tokenRetentionUntil, metadata: hookMetadata, isWebhook: queueItem.isWebhook ?? false, ...(queueItem.isSystem && { isSystem: true }), diff --git a/packages/core/src/workflow/hook.test.ts b/packages/core/src/workflow/hook.test.ts index e10000c020..5b6c3ed3b4 100644 --- a/packages/core/src/workflow/hook.test.ts +++ b/packages/core/src/workflow/hook.test.ts @@ -1173,17 +1173,17 @@ describe('createCreateHook', () => { } }); - it('evaluates hook token expiration with the same duration parser as sleep', () => { + it('evaluates minimum retention with the same duration parser as sleep', () => { const ctx = setupWorkflowContext([]); const createHook = createCreateHook(ctx); const dateNow = vi.spyOn(Date, 'now').mockReturnValue(1_000_000); try { - createHook({ experimental_expires: 1_000 }); + createHook({ experimental_minRetention: 1_000 }); const queueItem = ctx.invocationsQueue.values().next().value; expect(queueItem?.type).toBe('hook'); if (queueItem?.type === 'hook') { - expect(queueItem.tokenExpiresAt).toEqual(new Date(1_001_000)); + expect(queueItem.tokenRetentionUntil).toEqual(new Date(1_001_000)); } } finally { dateNow.mockRestore(); @@ -1192,15 +1192,15 @@ describe('createCreateHook', () => { it.each([ { - name: 'uses the persisted deadline', + name: 'uses the persisted retention deadline', eventData: { token: 'test-token', - tokenExpiresAt: new Date('2026-07-15T00:00:00.000Z'), + tokenRetentionUntil: new Date('2026-07-15T00:00:00.000Z'), }, expected: new Date('2026-07-15T00:00:00.000Z'), }, { - name: 'keeps an old event without an expiration time unchanged', + name: 'keeps an old event without a retention deadline unchanged', eventData: { token: 'test-token' }, expected: undefined, }, @@ -1218,7 +1218,7 @@ describe('createCreateHook', () => { const createHook = createCreateHook(ctx); const hook = createHook({ token: 'test-token', - experimental_expires: '30d', + experimental_minRetention: '30d', }); await expect(hook.getConflict()).resolves.toBeNull(); @@ -1226,17 +1226,17 @@ describe('createCreateHook', () => { const queueItem = ctx.invocationsQueue.values().next().value; expect(queueItem?.type).toBe('hook'); if (queueItem?.type === 'hook') { - expect(queueItem.tokenExpiresAt).toEqual(expected); + expect(queueItem.tokenRetentionUntil).toEqual(expected); } }); - it('rejects token expiration for a webhook hook', () => { + it('rejects minimum retention for a webhook hook', () => { const ctx = setupWorkflowContext([]); const createHook = createCreateHook(ctx); expect(() => - createHook({ isWebhook: true, experimental_expires: '30d' }) - ).toThrow('Webhook hooks do not support `experimental_expires`.'); + createHook({ isWebhook: true, experimental_minRetention: '30d' }) + ).toThrow('Webhook hooks do not support `experimental_minRetention`.'); expect(ctx.invocationsQueue.size).toBe(0); }); diff --git a/packages/core/src/workflow/hook.ts b/packages/core/src/workflow/hook.ts index d45c8633aa..b67ff42326 100644 --- a/packages/core/src/workflow/hook.ts +++ b/packages/core/src/workflow/hook.ts @@ -75,20 +75,20 @@ export function createCreateHook(ctx: WorkflowOrchestratorContext) { if ( options.isWebhook === true && - options.experimental_expires !== undefined + options.experimental_minRetention !== undefined ) { throw new Error( - 'Webhook hooks do not support `experimental_expires`. Use a non-webhook `createHook()` with `resumeHook()`.' + 'Webhook hooks do not support `experimental_minRetention`. Use a non-webhook `createHook()` with `resumeHook()`.' ); } // Generate hook ID and token const correlationId = `hook_${ctx.generateUlid()}`; const token = options.token ?? ctx.generateNanoid(); - const tokenExpiresAt = - options.experimental_expires === undefined + const tokenRetentionUntil = + options.experimental_minRetention === undefined ? undefined - : parseDurationToDate(options.experimental_expires); + : parseDurationToDate(options.experimental_minRetention); // Add hook creation to invocations queue (using Map for O(1) operations) const isWebhook = options.isWebhook ?? false; @@ -97,7 +97,7 @@ export function createCreateHook(ctx: WorkflowOrchestratorContext) { type: 'hook', correlationId, token, - tokenExpiresAt, + tokenRetentionUntil, metadata: options.metadata, isWebhook, }); @@ -180,7 +180,7 @@ export function createCreateHook(ctx: WorkflowOrchestratorContext) { const queueItem = ctx.invocationsQueue.get(correlationId); if (queueItem && queueItem.type === 'hook') { queueItem.hasCreatedEvent = true; - queueItem.tokenExpiresAt = event.eventData.tokenExpiresAt; + queueItem.tokenRetentionUntil = event.eventData.tokenRetentionUntil; } hasCreated = true; diff --git a/packages/world-vercel/src/events-v4.ts b/packages/world-vercel/src/events-v4.ts index c663ff1880..1351adaa6c 100644 --- a/packages/world-vercel/src/events-v4.ts +++ b/packages/world-vercel/src/events-v4.ts @@ -126,8 +126,8 @@ export interface CreateEventV4Input { * the step entity for premature-delivery pacing and observability. */ retryAfter?: Date; hookToken?: string; - /** Time after which another Hook can use the token. */ - hookTokenExpiresAt?: Date; + /** Earliest time another Hook can use the token after the run ends. */ + hookTokenRetentionUntil?: Date; hookIsWebhook?: boolean; hookIsSystem?: boolean; errorCode?: string; @@ -253,8 +253,8 @@ function buildPostFrameMeta( if (input.resumeAt !== undefined) meta.resumeAt = input.resumeAt; if (input.retryAfter !== undefined) meta.retryAfter = input.retryAfter; if (input.hookToken !== undefined) meta.hookToken = input.hookToken; - if (input.hookTokenExpiresAt !== undefined) { - meta.hookTokenExpiresAt = input.hookTokenExpiresAt; + if (input.hookTokenRetentionUntil !== undefined) { + meta.hookTokenRetentionUntil = input.hookTokenRetentionUntil; } if (input.hookIsWebhook !== undefined) meta.hookIsWebhook = input.hookIsWebhook; diff --git a/packages/world-vercel/src/events.test.ts b/packages/world-vercel/src/events.test.ts index 46c46c5ddc..00da3add94 100644 --- a/packages/world-vercel/src/events.test.ts +++ b/packages/world-vercel/src/events.test.ts @@ -204,21 +204,21 @@ describe('createWorkflowRunEvent stateUpdatedAt wire field', () => { * actually reach the frame meta with the right values and renames. */ describe('splitEventDataForV4 attribute fields', () => { - it('carries the hook token expiration time in the frame meta', () => { - const tokenExpiresAt = new Date('2026-07-10T12:00:00.000Z'); + it('carries the hook token retention deadline in the frame meta', () => { + const tokenRetentionUntil = new Date('2026-07-10T12:00:00.000Z'); const { payload, meta } = splitEventDataForV4({ eventType: 'hook_created', correlationId: 'hook_1', specVersion: 5, eventData: { token: 'order:123', - tokenExpiresAt, + tokenRetentionUntil, }, } as AnyEventRequest); expect(payload).toBeUndefined(); expect(meta.hookToken).toBe('order:123'); - expect(meta.hookTokenExpiresAt).toEqual(tokenExpiresAt); + expect(meta.hookTokenRetentionUntil).toEqual(tokenRetentionUntil); }); it('carries attr_set changes/writer/allowReservedAttributes in the frame meta', () => { diff --git a/packages/world-vercel/src/events.ts b/packages/world-vercel/src/events.ts index 2c3b132288..6284bb6e8b 100644 --- a/packages/world-vercel/src/events.ts +++ b/packages/world-vercel/src/events.ts @@ -136,7 +136,7 @@ interface SplitEventData { resumeAt?: Date; retryAfter?: Date; hookToken?: string; - hookTokenExpiresAt?: Date; + hookTokenRetentionUntil?: Date; hookIsWebhook?: boolean; hookIsSystem?: boolean; errorCode?: string; @@ -188,7 +188,7 @@ type MetaSourceField = | 'resumeAt' | 'retryAfter' | 'token' - | 'tokenExpiresAt' + | 'tokenRetentionUntil' | 'isWebhook' | 'isSystem' | 'errorCode' @@ -291,8 +291,8 @@ export function splitEventDataForV4(data: AnyEventRequest): SplitEventData { if (typeof eventData.token === 'string') { meta.hookToken = eventData.token; } - if (eventData.tokenExpiresAt instanceof Date) { - meta.hookTokenExpiresAt = eventData.tokenExpiresAt; + if (eventData.tokenRetentionUntil instanceof Date) { + meta.hookTokenRetentionUntil = eventData.tokenRetentionUntil; } if (typeof eventData.isWebhook === 'boolean') { meta.hookIsWebhook = eventData.isWebhook; diff --git a/packages/world/src/events.test.ts b/packages/world/src/events.test.ts index 2c9840260c..587d48535b 100644 --- a/packages/world/src/events.test.ts +++ b/packages/world/src/events.test.ts @@ -1,21 +1,21 @@ import { describe, expect, it } from 'vitest'; import { CreateEventSchema, EventSchema } from './events'; -describe('hook_created token expiration', () => { - it('coerces tokenExpiresAt to a Date', () => { +describe('hook_created token retention', () => { + it('coerces tokenRetentionUntil to a Date', () => { const parsed = CreateEventSchema.parse({ eventType: 'hook_created', correlationId: 'hook_1', specVersion: 5, eventData: { token: 'order:123', - tokenExpiresAt: '2026-08-01T00:00:00.000Z', + tokenRetentionUntil: '2026-08-01T00:00:00.000Z', }, }); expect(parsed.eventType).toBe('hook_created'); if (parsed.eventType === 'hook_created') { - expect(parsed.eventData.tokenExpiresAt).toEqual( + expect(parsed.eventData.tokenRetentionUntil).toEqual( new Date('2026-08-01T00:00:00.000Z') ); } diff --git a/packages/world/src/events.ts b/packages/world/src/events.ts index e6a9dcd9b0..c93f5463d6 100644 --- a/packages/world/src/events.ts +++ b/packages/world/src/events.ts @@ -394,7 +394,7 @@ export const HookCreatedEventSchema = BaseEventSchema.extend({ correlationId: z.string(), eventData: z.object({ token: z.string(), - tokenExpiresAt: z.coerce.date().optional(), + tokenRetentionUntil: z.coerce.date().optional(), metadata: SerializedDataSchema.optional(), isWebhook: z.boolean().optional(), isSystem: z.boolean().optional(), diff --git a/packages/world/src/interfaces.ts b/packages/world/src/interfaces.ts index c61b5c5fb8..b62e4becb1 100644 --- a/packages/world/src/interfaces.ts +++ b/packages/world/src/interfaces.ts @@ -275,8 +275,8 @@ export interface Storage { * The "World" interface represents how Workflows are able to communicate with the outside world. */ export interface WorldCapabilities { - /** Supports `experimental_expires` for Hooks. */ - hookTtl?: { + /** Supports `experimental_minRetention` for Hooks. */ + hookRetention?: { active: boolean; }; } From e26ca7c29ae67129afaf8ec30e6f511d51376fae Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Thu, 16 Jul 2026 13:17:13 -0700 Subject: [PATCH 25/36] docs: keep hook retention guidance on v5 --- docs/content/worlds/v4/building-a-world.mdx | 13 +++---------- docs/content/worlds/v5/building-a-world.mdx | 13 ++++++++++--- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/content/worlds/v4/building-a-world.mdx b/docs/content/worlds/v4/building-a-world.mdx index edcb57aa0e..bd0b1a047d 100644 --- a/docs/content/worlds/v4/building-a-world.mdx +++ b/docs/content/worlds/v4/building-a-world.mdx @@ -32,14 +32,7 @@ A World connects workflows to the infrastructure that powers them. The World int {/* @skip-typecheck - interface definition, not runnable code */} ```typescript -interface WorldCapabilities { - hookRetention?: { - active: boolean; - }; -} - interface World extends Storage, Queue, Streamer { - capabilities?: WorldCapabilities; start?(): Promise; close?(): Promise; getEncryptionKeyForRun?(run: WorkflowRun): Promise; @@ -47,7 +40,7 @@ interface World extends Storage, Queue, Streamer { } ``` -The optional `capabilities` object advertises additional behavior. Set `hookRetention.active` to `true` only when the World implements Hook token retention. The optional `start()` method initializes background tasks (for example, queue polling). The optional `close()` method releases resources like connection pools and listeners. The optional `getEncryptionKeyForRun()` method returns the AES-256 key used to encrypt data for a run; if it is not implemented, encryption is disabled. +The optional `start()` method initializes background tasks (for example, queue polling). The optional `close()` method releases resources like connection pools and listeners. The optional `getEncryptionKeyForRun()` method returns the AES-256 key used to encrypt data for a run; if it is not implemented, encryption is disabled. ## The Event Log Model @@ -100,9 +93,9 @@ interface Storage { **Run Creation:** For `run_created` events, the `runId` parameter may be a client-provided string or `null`. When `null`, your World generates and returns a new `runId`. -**Hook Tokens:** Hook tokens must be unique. If a `hook_created` event conflicts with an active Hook or a token still reserved after its run ended, return a `hook_conflict` event and include the owner's run ID as `eventData.conflictingRunId`. +**Hook Tokens:** Hook tokens must be unique. If a `hook_created` event conflicts with an existing token, return a `hook_conflict` event instead and include the active hook owner's run ID as `eventData.conflictingRunId`. -**Automatic Hook Cleanup:** When a run ends, remove its live Hooks. Make each token available unless its `tokenRetentionUntil` is still in the future. A `hook_disposed` event always makes the token available immediately. +**Automatic Hook Disposal:** When a workflow reaches a terminal state (`completed`, `failed`, or `cancelled`), automatically dispose of all associated hooks to release tokens for reuse. ## Queue Interface diff --git a/docs/content/worlds/v5/building-a-world.mdx b/docs/content/worlds/v5/building-a-world.mdx index bd0b1a047d..edcb57aa0e 100644 --- a/docs/content/worlds/v5/building-a-world.mdx +++ b/docs/content/worlds/v5/building-a-world.mdx @@ -32,7 +32,14 @@ A World connects workflows to the infrastructure that powers them. The World int {/* @skip-typecheck - interface definition, not runnable code */} ```typescript +interface WorldCapabilities { + hookRetention?: { + active: boolean; + }; +} + interface World extends Storage, Queue, Streamer { + capabilities?: WorldCapabilities; start?(): Promise; close?(): Promise; getEncryptionKeyForRun?(run: WorkflowRun): Promise; @@ -40,7 +47,7 @@ interface World extends Storage, Queue, Streamer { } ``` -The optional `start()` method initializes background tasks (for example, queue polling). The optional `close()` method releases resources like connection pools and listeners. The optional `getEncryptionKeyForRun()` method returns the AES-256 key used to encrypt data for a run; if it is not implemented, encryption is disabled. +The optional `capabilities` object advertises additional behavior. Set `hookRetention.active` to `true` only when the World implements Hook token retention. The optional `start()` method initializes background tasks (for example, queue polling). The optional `close()` method releases resources like connection pools and listeners. The optional `getEncryptionKeyForRun()` method returns the AES-256 key used to encrypt data for a run; if it is not implemented, encryption is disabled. ## The Event Log Model @@ -93,9 +100,9 @@ interface Storage { **Run Creation:** For `run_created` events, the `runId` parameter may be a client-provided string or `null`. When `null`, your World generates and returns a new `runId`. -**Hook Tokens:** Hook tokens must be unique. If a `hook_created` event conflicts with an existing token, return a `hook_conflict` event instead and include the active hook owner's run ID as `eventData.conflictingRunId`. +**Hook Tokens:** Hook tokens must be unique. If a `hook_created` event conflicts with an active Hook or a token still reserved after its run ended, return a `hook_conflict` event and include the owner's run ID as `eventData.conflictingRunId`. -**Automatic Hook Disposal:** When a workflow reaches a terminal state (`completed`, `failed`, or `cancelled`), automatically dispose of all associated hooks to release tokens for reuse. +**Automatic Hook Cleanup:** When a run ends, remove its live Hooks. Make each token available unless its `tokenRetentionUntil` is still in the future. A `hook_disposed` event always makes the token available immediately. ## Queue Interface From 6731caae0e4dfc2b8e7ab8e5434faf6c24bdcc46 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Thu, 16 Jul 2026 13:19:34 -0700 Subject: [PATCH 26/36] docs: define retained run availability --- docs/content/worlds/v5/building-a-world.mdx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/content/worlds/v5/building-a-world.mdx b/docs/content/worlds/v5/building-a-world.mdx index edcb57aa0e..8fb45f1334 100644 --- a/docs/content/worlds/v5/building-a-world.mdx +++ b/docs/content/worlds/v5/building-a-world.mdx @@ -102,6 +102,8 @@ interface Storage { **Hook Tokens:** Hook tokens must be unique. If a `hook_created` event conflicts with an active Hook or a token still reserved after its run ended, return a `hook_conflict` event and include the owner's run ID as `eventData.conflictingRunId`. +Keep the owning Run available for at least as long as its token remains unavailable, because conflicts return that Run. + **Automatic Hook Cleanup:** When a run ends, remove its live Hooks. Make each token available unless its `tokenRetentionUntil` is still in the future. A `hook_disposed` event always makes the token available immediately. ## Queue Interface From 02eceb9287a41daa554372b75efb18015f069d52 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Thu, 16 Jul 2026 13:43:20 -0700 Subject: [PATCH 27/36] fix(core): validate Hook retention at creation --- packages/core/src/create-hook.ts | 4 ++-- packages/core/src/private.ts | 2 ++ packages/core/src/runtime.ts | 3 ++- packages/core/src/workflow.test.ts | 22 ++++++++++++++++++++++ packages/core/src/workflow.ts | 10 ++++++++-- packages/core/src/workflow/hook.test.ts | 15 +++++++++++++++ packages/core/src/workflow/hook.ts | 15 ++++++++++++++- packages/world-vercel/src/events-v4.ts | 5 ++++- packages/world-vercel/src/events.test.ts | 11 ++++++++--- packages/world-vercel/src/events.ts | 5 +++++ packages/world/src/interfaces.ts | 12 ++++++++---- 11 files changed, 90 insertions(+), 14 deletions(-) diff --git a/packages/core/src/create-hook.ts b/packages/core/src/create-hook.ts index c956563d20..897cb87148 100644 --- a/packages/core/src/create-hook.ts +++ b/packages/core/src/create-hook.ts @@ -151,8 +151,8 @@ export interface HookOptions { * Calling `dispose()` (including through `using`) releases the token * immediately. * - * The workflow fails when registering the Hook if its World does not support - * this experimental option. + * `createHook()` throws if the configured World does not support this + * experimental option. * * @example * diff --git a/packages/core/src/private.ts b/packages/core/src/private.ts index 2379016d33..886ffdcf8f 100644 --- a/packages/core/src/private.ts +++ b/packages/core/src/private.ts @@ -3,6 +3,7 @@ */ import { withResolvers } from '@workflow/utils'; +import type { WorldCapabilities } from '@workflow/world'; import type { CryptoKey } from './encryption.js'; import type { EventsConsumer } from './events-consumer.js'; import type { QueueItem } from './global.js'; @@ -132,6 +133,7 @@ export function getStepFunction(stepId: string): StepFunction | undefined { export interface WorkflowOrchestratorContext { runId: string; encryptionKey: CryptoKey | undefined; + worldCapabilities?: WorldCapabilities; globalThis: typeof globalThis; eventsConsumer: EventsConsumer; /** diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 5fcd0fd680..1f52ade2f2 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -1436,7 +1436,8 @@ export function workflowEntrypoint( // fire-and-forget `*_created` events before the terminal // `awaitRunReady()` below, so gate those writes on the // backgrounded run_started too. Undefined outside turbo. - runReadyBarrier + runReadyBarrier, + world.capabilities ); runtimeLogger.debug('Workflow replay completed', { workflowRunId: runId, diff --git a/packages/core/src/workflow.test.ts b/packages/core/src/workflow.test.ts index fc21423518..2d903b94a7 100644 --- a/packages/core/src/workflow.test.ts +++ b/packages/core/src/workflow.test.ts @@ -5446,6 +5446,12 @@ describe('runWorkflow', () => { return 'done'; }`; + const FIRE_AND_FORGET_RETAINED_HOOK = `const createHook = globalThis[Symbol.for("WORKFLOW_CREATE_HOOK")]; + async function workflow() { + createHook({ token: 'fire-and-forget', experimental_minRetention: '30d' }); + return 'done'; + }`; + // `void sleep(...)` is the wait equivalent: it enqueues a wait_created the // workflow never awaits, drained the same way as the hook above. const FIRE_AND_FORGET_WAIT = `const sleep = globalThis[Symbol.for("WORKFLOW_SLEEP")]; @@ -5477,6 +5483,22 @@ describe('runWorkflow', () => { setWorld(undefined); }); + it('rejects unsupported retained Hooks before the final drain', async () => { + await expect( + runWorkflow( + `${FIRE_AND_FORGET_RETAINED_HOOK}${getWorkflowTransformCode('workflow')}`, + await makeRun(), + [], + noEncryptionKey, + undefined, + undefined, + { hookRetention: { active: false } } + ) + ).rejects.toThrow( + 'The configured World does not support `experimental_minRetention` for Hooks.' + ); + }); + it('does not write hook_created until the runReadyBarrier resolves', async () => { let releaseBarrier: () => void = () => {}; const runReadyBarrier = new Promise((resolve) => { diff --git a/packages/core/src/workflow.ts b/packages/core/src/workflow.ts index 4d8a138dcc..81a50eebc2 100644 --- a/packages/core/src/workflow.ts +++ b/packages/core/src/workflow.ts @@ -6,7 +6,7 @@ import { } from '@workflow/errors'; import { createWorkflowBaseUrl, withResolvers } from '@workflow/utils'; import { parseWorkflowName } from '@workflow/utils/parse-name'; -import type { Event, WorkflowRun } from '@workflow/world'; +import type { Event, WorkflowRun, WorldCapabilities } from '@workflow/world'; import { SPEC_VERSION_SUPPORTS_COMPRESSION } from '@workflow/world'; import * as nanoid from 'nanoid'; import { monotonicFactory } from 'ulid'; @@ -152,7 +152,12 @@ export async function runWorkflow( * committed at workflow completion order after the run's creation. Undefined * outside turbo, where `run_started` is awaited up front. */ - runReadyBarrier?: Promise + runReadyBarrier?: Promise, + /** + * Features supported by the World executing this workflow. Missing + * capabilities are treated as unsupported. + */ + worldCapabilities?: WorldCapabilities ): Promise { return trace(`workflow.run ${workflowRun.workflowName}`, async (span) => { span?.setAttributes({ @@ -239,6 +244,7 @@ export async function runWorkflow( const workflowContext: WorkflowOrchestratorContext = { runId: workflowRun.runId, encryptionKey, + worldCapabilities, globalThis: vmGlobalThis, onWorkflowError: workflowDiscontinuation.reject, eventsConsumer, diff --git a/packages/core/src/workflow/hook.test.ts b/packages/core/src/workflow/hook.test.ts index 5b6c3ed3b4..0c7f12d305 100644 --- a/packages/core/src/workflow/hook.test.ts +++ b/packages/core/src/workflow/hook.test.ts @@ -37,6 +37,7 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { return { runId: 'wrun_test', encryptionKey: undefined, + worldCapabilities: { hookRetention: { active: true } }, globalThis: context.globalThis, eventsConsumer: new EventsConsumer(events, { onUnconsumedEvent: () => {}, @@ -1190,6 +1191,20 @@ describe('createCreateHook', () => { } }); + it.each([ + ['missing', undefined], + ['inactive', { hookRetention: { active: false } }], + ])('rejects minimum retention when the capability is %s', (_state, capabilities) => { + const ctx = setupWorkflowContext([]); + ctx.worldCapabilities = capabilities; + const createHook = createCreateHook(ctx); + + expect(() => createHook({ experimental_minRetention: '30d' })).toThrow( + 'The configured World does not support `experimental_minRetention` for Hooks.' + ); + expect(ctx.invocationsQueue.size).toBe(0); + }); + it.each([ { name: 'uses the persisted retention deadline', diff --git a/packages/core/src/workflow/hook.ts b/packages/core/src/workflow/hook.ts index b67ff42326..aeb28fa490 100644 --- a/packages/core/src/workflow/hook.ts +++ b/packages/core/src/workflow/hook.ts @@ -1,4 +1,8 @@ -import { HookConflictError, ReplayDivergenceError } from '@workflow/errors'; +import { + FatalError, + HookConflictError, + ReplayDivergenceError, +} from '@workflow/errors'; import { WORKFLOW_DESERIALIZE } from '@workflow/serde'; import { type PromiseWithResolvers, @@ -82,6 +86,15 @@ export function createCreateHook(ctx: WorkflowOrchestratorContext) { ); } + if ( + options.experimental_minRetention !== undefined && + ctx.worldCapabilities?.hookRetention?.active !== true + ) { + throw new FatalError( + 'The configured World does not support `experimental_minRetention` for Hooks.' + ); + } + // Generate hook ID and token const correlationId = `hook_${ctx.generateUlid()}`; const token = options.token ?? ctx.generateNanoid(); diff --git a/packages/world-vercel/src/events-v4.ts b/packages/world-vercel/src/events-v4.ts index 1351adaa6c..e8e251d333 100644 --- a/packages/world-vercel/src/events-v4.ts +++ b/packages/world-vercel/src/events-v4.ts @@ -126,7 +126,10 @@ export interface CreateEventV4Input { * the step entity for premature-delivery pacing and observability. */ retryAfter?: Date; hookToken?: string; - /** Earliest time another Hook can use the token after the run ends. */ + /** + * Earliest time another Hook can use the token after the owning run ends. + * An active run always retains its token beyond this time. + */ hookTokenRetentionUntil?: Date; hookIsWebhook?: boolean; hookIsSystem?: boolean; diff --git a/packages/world-vercel/src/events.test.ts b/packages/world-vercel/src/events.test.ts index 00da3add94..c9ebe29a4b 100644 --- a/packages/world-vercel/src/events.test.ts +++ b/packages/world-vercel/src/events.test.ts @@ -204,15 +204,20 @@ describe('createWorkflowRunEvent stateUpdatedAt wire field', () => { * actually reach the frame meta with the right values and renames. */ describe('splitEventDataForV4 attribute fields', () => { - it('carries the hook token retention deadline in the frame meta', () => { - const tokenRetentionUntil = new Date('2026-07-10T12:00:00.000Z'); + it.each([ + new Date('2026-07-10T12:00:00.000Z'), + '2026-07-10T12:00:00.000Z', + ])('carries a %s Hook retention deadline in the frame meta', (value) => { + const tokenRetentionUntil = new Date( + typeof value === 'string' ? value : value.getTime() + ); const { payload, meta } = splitEventDataForV4({ eventType: 'hook_created', correlationId: 'hook_1', specVersion: 5, eventData: { token: 'order:123', - tokenRetentionUntil, + tokenRetentionUntil: value, }, } as AnyEventRequest); diff --git a/packages/world-vercel/src/events.ts b/packages/world-vercel/src/events.ts index 6284bb6e8b..7f4dbe8e51 100644 --- a/packages/world-vercel/src/events.ts +++ b/packages/world-vercel/src/events.ts @@ -293,6 +293,11 @@ export function splitEventDataForV4(data: AnyEventRequest): SplitEventData { } if (eventData.tokenRetentionUntil instanceof Date) { meta.hookTokenRetentionUntil = eventData.tokenRetentionUntil; + } else if (typeof eventData.tokenRetentionUntil === 'string') { + const parsed = new Date(eventData.tokenRetentionUntil); + if (!Number.isNaN(parsed.getTime())) { + meta.hookTokenRetentionUntil = parsed; + } } if (typeof eventData.isWebhook === 'boolean') { meta.hookIsWebhook = eventData.isWebhook; diff --git a/packages/world/src/interfaces.ts b/packages/world/src/interfaces.ts index a6b5e77617..3fcc0acc2a 100644 --- a/packages/world/src/interfaces.ts +++ b/packages/world/src/interfaces.ts @@ -290,16 +290,20 @@ export interface Storage { }; } -/** - * The "World" interface represents how Workflows are able to communicate with the outside world. - */ +/** Optional features a World can explicitly advertise to the runtime. */ export interface WorldCapabilities { - /** Supports `experimental_minRetention` for Hooks. */ + /** + * Supports `experimental_minRetention` for Hooks. Missing or inactive means + * the runtime rejects retained Hooks before registration. + */ hookRetention?: { active: boolean; }; } +/** + * The "World" interface represents how Workflows are able to communicate with the outside world. + */ export interface World extends Queue, Streamer, Storage { /** * Optional analytics read namespace for observability surfaces. From 8090ef9c8168febf56304683b331ca40ca4ff795 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Thu, 16 Jul 2026 15:32:19 -0700 Subject: [PATCH 28/36] feat(core): define retained Hook lookup semantics --- .../workflow-api/get-hook-by-token.mdx | 2 + .../workflow-api/resume-hook.mdx | 2 + .../v5/api-reference/workflow/create-hook.mdx | 2 + .../cookbook/common-patterns/idempotency.mdx | 2 +- .../docs/v5/foundations/idempotency.mdx | 2 +- .../docs/v5/how-it-works/event-sourcing.mdx | 8 +-- packages/core/src/create-hook.ts | 2 + packages/core/src/runtime/resume-hook.test.ts | 59 +++++++++++++++++++ packages/core/src/runtime/resume-hook.ts | 10 +++- packages/world/src/hooks.ts | 3 +- packages/world/src/interfaces.ts | 20 +++++-- 11 files changed, 100 insertions(+), 12 deletions(-) create mode 100644 packages/core/src/runtime/resume-hook.test.ts diff --git a/docs/content/docs/v5/api-reference/workflow-api/get-hook-by-token.mdx b/docs/content/docs/v5/api-reference/workflow-api/get-hook-by-token.mdx index 1448c48f3e..82638ae989 100644 --- a/docs/content/docs/v5/api-reference/workflow-api/get-hook-by-token.mdx +++ b/docs/content/docs/v5/api-reference/workflow-api/get-hook-by-token.mdx @@ -11,6 +11,8 @@ related: Retrieves a hook by its unique token, returning the associated workflow run information and any metadata that was set when the hook was created. This function is useful for inspecting hook details before deciding whether to resume a workflow. +When `experimental_minRetention` is set, this function continues to return the Hook after its workflow ends until retention ends. That Hook cannot be resumed. Use `getRun(hook.runId)` to inspect the finished run. + `getHookByToken` is a runtime function that must be called from outside a workflow function. diff --git a/docs/content/docs/v5/api-reference/workflow-api/resume-hook.mdx b/docs/content/docs/v5/api-reference/workflow-api/resume-hook.mdx index 6a1372d739..072337fb8f 100644 --- a/docs/content/docs/v5/api-reference/workflow-api/resume-hook.mdx +++ b/docs/content/docs/v5/api-reference/workflow-api/resume-hook.mdx @@ -14,6 +14,8 @@ Resumes a workflow run by sending a payload to a hook identified by its token. It creates a `hook_received` event and re-triggers the workflow to continue execution. +A Hook kept by `experimental_minRetention` after its workflow ends cannot be resumed. `resumeHook()` throws `HookNotFoundError` in that case. + `resumeHook` is a runtime function that must be called from outside a workflow function. diff --git a/docs/content/docs/v5/api-reference/workflow/create-hook.mdx b/docs/content/docs/v5/api-reference/workflow/create-hook.mdx index 8596996f6e..1162c4a963 100644 --- a/docs/content/docs/v5/api-reference/workflow/create-hook.mdx +++ b/docs/content/docs/v5/api-reference/workflow/create-hook.mdx @@ -181,6 +181,8 @@ export async function processOrder(orderId: string) { The Hook remains active until the workflow ends, even if the configured time passes first. Another Hook can use the token only after both the workflow has ended and the configured time has passed. For example, `"30d"` keeps the token unavailable for 29 more days if the workflow ends after 1 day. A workflow that runs for more than 30 days releases the token when it ends. +After the workflow ends, [`getHookByToken()`](/docs/api-reference/workflow-api/get-hook-by-token) can still find the Hook until retention ends, but the Hook cannot be resumed. + `hook.dispose()` releases the token immediately. Because `using` calls `dispose()`, do not use it when the token should remain unavailable after the run ends. diff --git a/docs/content/docs/v5/cookbook/common-patterns/idempotency.mdx b/docs/content/docs/v5/cookbook/common-patterns/idempotency.mdx index 733d1af356..b62d5bfb5c 100644 --- a/docs/content/docs/v5/cookbook/common-patterns/idempotency.mdx +++ b/docs/content/docs/v5/cookbook/common-patterns/idempotency.mdx @@ -90,6 +90,6 @@ The workflow should create the deterministic hook and check `await hook.getConfl - [`"use step"`](/docs/foundations/workflows-and-steps#step-functions) -- declares step functions with full Node.js access - [`getStepMetadata()`](/docs/api-reference/workflow/get-step-metadata) -- provides the deterministic `stepId` for idempotency keys - [`createHook()`](/docs/api-reference/workflow/create-hook) -- creates a hook with an optional deterministic token -- [`getHookByToken()`](/docs/api-reference/workflow-api/get-hook-by-token) -- finds the active hook for a token +- [`getHookByToken()`](/docs/api-reference/workflow-api/get-hook-by-token) -- finds the Hook that owns a token - [`resumeHook()`](/docs/api-reference/workflow-api/resume-hook) -- resumes the active hook when the duplicate request carries data - [`start()`](/docs/api-reference/workflow-api/start) -- starts a new workflow run diff --git a/docs/content/docs/v5/foundations/idempotency.mdx b/docs/content/docs/v5/foundations/idempotency.mdx index c892929075..56b42fd4a0 100644 --- a/docs/content/docs/v5/foundations/idempotency.mdx +++ b/docs/content/docs/v5/foundations/idempotency.mdx @@ -149,7 +149,7 @@ export async function POST(request: Request) { This avoids creating a new run only after the first run has registered its hook. Because `start()` returns before the run body executes and calls `createHook()`, two concurrent requests can both observe "no hook yet" and each call `start()`. The race is resolved inside the workflow body, where the losing run observes `getConflict()` resolving with the active owner and returns without doing duplicate-sensitive work — and the route detects it by comparing the resumed hook's `runId` against the run it just started, without waiting for either run to finish. A native API for atomically starting a run and registering a hook is in the works. Until then, model recovery inside the workflow by checking `hook.getConflict()`. -This coordinates active runs by default: the token becomes available when its workflow ends. Set `experimental_minRetention` to keep it unavailable to late duplicates. The Hook itself still ends with the workflow and cannot be resumed. See [`createHook()` minimum retention](/docs/api-reference/workflow/create-hook#keep-a-token-unavailable-after-the-run-ends) for examples and supported values. +This coordinates active runs by default: the token becomes available when its workflow ends. Set `experimental_minRetention` to keep it unavailable to late duplicates. After the workflow ends, the Hook can still be found with `getHookByToken()` until retention ends, but it cannot be resumed. See [`createHook()` minimum retention](/docs/api-reference/workflow/create-hook#keep-a-token-unavailable-after-the-run-ends) for examples and supported values. ### Conflict-handling strategies diff --git a/docs/content/docs/v5/how-it-works/event-sourcing.mdx b/docs/content/docs/v5/how-it-works/event-sourcing.mdx index 6bf4014dd3..70ae0baf5c 100644 --- a/docs/content/docs/v5/how-it-works/event-sourcing.mdx +++ b/docs/content/docs/v5/how-it-works/event-sourcing.mdx @@ -121,15 +121,15 @@ flowchart TD **Hook states:** -- `active`: Ready to receive payloads (hook exists in storage) -- `disposed`: No longer accepting payloads (hook is deleted from storage) +- `active`: Ready to receive payloads +- `disposed`: No longer accepting payloads - `conflicted`: Hook creation failed because the token is already in use by another workflow -Unlike other entities, hooks don't have a `status` field—the states above are conceptual. An "active" hook is one that exists in storage, while "disposed" means the hook has been deleted. When a `hook_disposed` event is created, the hook record is removed rather than updated. +Unlike other entities, hooks don't have a `status` field—the states above are conceptual. When a `hook_disposed` event is created, the hook record is removed rather than updated. While a hook is active, its token is reserved and cannot be used by other workflows. If a workflow attempts to create a hook with a token reserved by another run — either by an active hook or by `experimental_minRetention` after its run ended — a `hook_conflict` event is recorded instead of `hook_created`. Current worlds include the token and the run ID that owns it, though older persisted events or world implementations may only include the token. This causes `hook.getConflict()` to resolve with the conflicting run and the hook's payload promise to reject with a `HookConflictError`, which you can detect with `HookConflictError.is(error)`. See the [hook-conflict error](/docs/errors/hook-conflict) documentation for more details. -When a workflow ends, its Hooks are removed. Their tokens normally become available again, but `experimental_minRetention` can delay that. A `hook_disposed` event makes a token available immediately. +When a workflow ends, its Hooks can no longer be resumed. They are normally removed and their tokens become available again. With `experimental_minRetention`, a Hook remains readable and its token remains unavailable until retention ends. A `hook_disposed` event removes the Hook and makes its token available immediately. See [Hooks & Webhooks](/docs/foundations/hooks) for more on how hooks and webhooks work. diff --git a/packages/core/src/create-hook.ts b/packages/core/src/create-hook.ts index 897cb87148..e24a528e23 100644 --- a/packages/core/src/create-hook.ts +++ b/packages/core/src/create-hook.ts @@ -147,6 +147,8 @@ export interface HookOptions { * The Hook remains active until the workflow ends, even if the configured * time passes first. Another Hook can use the token only after both the run * has ended and the configured time has passed. + * After the run ends, the Hook can still be found with `getHookByToken()` + * until retention ends, but it cannot be resumed. * * Calling `dispose()` (including through `using`) releases the token * immediately. diff --git a/packages/core/src/runtime/resume-hook.test.ts b/packages/core/src/runtime/resume-hook.test.ts new file mode 100644 index 0000000000..b8c3387ec2 --- /dev/null +++ b/packages/core/src/runtime/resume-hook.test.ts @@ -0,0 +1,59 @@ +import { HookNotFoundError } from '@workflow/errors'; +import { + type Hook, + SPEC_VERSION_CURRENT, + type WorkflowRun, + type World, +} from '@workflow/world'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { resumeHook } from './resume-hook.js'; +import { setWorld } from './world.js'; + +vi.mock('@vercel/functions', () => ({ waitUntil: vi.fn() })); +vi.mock('../telemetry.js', () => ({ + linkToTraceCarrier: vi.fn(), + trace: vi.fn((_name, fn) => fn(undefined)), +})); + +describe('resumeHook', () => { + afterEach(() => setWorld(undefined)); + + it('rejects a retained Hook after its run ends', async () => { + const hook = { + runId: 'wrun_1', + hookId: 'hook_1', + token: 'order:1', + ownerId: 'owner_1', + projectId: 'project_1', + environment: 'production', + createdAt: new Date(), + } satisfies Hook; + const run = { + runId: hook.runId, + status: 'completed', + deploymentId: 'deployment_1', + workflowName: 'processOrder', + output: undefined, + completedAt: new Date(), + createdAt: new Date(), + updatedAt: new Date(), + attributes: {}, + } satisfies WorkflowRun; + const createEvent = vi.fn(); + const queue = vi.fn(); + + setWorld({ + specVersion: SPEC_VERSION_CURRENT, + hooks: { getByToken: vi.fn().mockResolvedValue(hook) }, + runs: { get: vi.fn().mockResolvedValue(run) }, + events: { create: createEvent }, + queue, + } as unknown as World); + + await expect(resumeHook(hook.token, {})).rejects.toSatisfy( + HookNotFoundError.is + ); + expect(createEvent).not.toHaveBeenCalled(); + expect(queue).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/core/src/runtime/resume-hook.ts b/packages/core/src/runtime/resume-hook.ts index 534eff9fe4..aeecf123ab 100644 --- a/packages/core/src/runtime/resume-hook.ts +++ b/packages/core/src/runtime/resume-hook.ts @@ -6,6 +6,7 @@ import { import { type Hook, isLegacySpecVersion, + isTerminalWorkflowRunStatus, SPEC_VERSION_CURRENT, SPEC_VERSION_LEGACY, SPEC_VERSION_SUPPORTS_COMPRESSION, @@ -56,6 +57,9 @@ async function getHookByTokenWithKey(token: string): Promise<{ * and hydrate the `metadata` property if it was set from within * the workflow run. * + * A Hook kept by minimum retention remains available here after its run ends, + * but cannot be resumed. + * * @param token - The unique token identifying the hook */ export async function getHookByToken(token: string): Promise { @@ -72,7 +76,7 @@ export async function getHookByToken(token: string): Promise { * @param tokenOrHook - The unique token identifying the hook, or the hook object itself * @param payload - The data payload to send to the hook * @returns Promise resolving to the hook - * @throws Error if the hook is not found or if there's an error during the process + * @throws {HookNotFoundError} If the Hook does not exist or its run has ended * * @example * @@ -127,6 +131,10 @@ export async function resumeHook( ...Attribute.WorkflowRunId(hook.runId), }); + if (isTerminalWorkflowRunStatus(workflowRun.status)) { + throw new HookNotFoundError(hook.token); + } + // Check the target run's capabilities to ensure we encode the // payload in a format the run's deployment can decode. For example, // runs created before encryption support was added cannot decode diff --git a/packages/world/src/hooks.ts b/packages/world/src/hooks.ts index 48319240f1..c5e3f7bb64 100644 --- a/packages/world/src/hooks.ts +++ b/packages/world/src/hooks.ts @@ -27,7 +27,8 @@ export const HookSchema = z.object({ }); /** - * Represents a hook that can be used to resume a paused workflow run. + * Represents a Hook. Hooks kept by minimum retention remain readable after + * their workflow runs end, but cannot be resumed. * * Note: metadata type is SerializedData to support both: * - specVersion >= 2: Uint8Array (binary devalue format) diff --git a/packages/world/src/interfaces.ts b/packages/world/src/interfaces.ts index 3fcc0acc2a..8adc8c4d2c 100644 --- a/packages/world/src/interfaces.ts +++ b/packages/world/src/interfaces.ts @@ -130,10 +130,10 @@ export interface Streamer { * - run_cancelled event for run cancellation * - hook_disposed event for explicit hook disposal (optional) * - * Note: Hooks are automatically disposed by the World implementation when a workflow - * reaches a terminal state (run_completed, run_failed, run_cancelled). This releases - * hook tokens for reuse by future workflows. The hook_disposed event is only needed - * for explicit disposal before workflow completion. + * When a workflow reaches a terminal state, its Hooks can no longer be resumed. + * Worlds normally remove them and release their tokens. A Hook with minimum + * retention remains readable and keeps its token unavailable until its retention + * ends. A hook_disposed event always removes the Hook and releases its token. */ export interface Storage { runs: { @@ -284,8 +284,20 @@ export interface Storage { }; hooks: { + /** + * Returns a Hook by ID. A Hook kept by minimum retention remains readable + * after its run ends, but cannot be resumed. + */ get(hookId: string, params?: GetHookParams): Promise; + /** + * Returns the Hook that owns a token, including a Hook kept by minimum + * retention after its run ends. + */ getByToken(token: string, params?: GetHookParams): Promise; + /** + * Lists Hooks, including Hooks kept by minimum retention after their runs + * end. + */ list(params: ListHooksParams): Promise>; }; } From b98df8604882ab6808dd60330e41842b595851a6 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Thu, 16 Jul 2026 15:55:00 -0700 Subject: [PATCH 29/36] refactor(core): simplify hook retention checks --- packages/core/src/runtime/resume-hook.test.ts | 3 ++ packages/core/src/runtime/resume-hook.ts | 36 ++++++++-------- .../src/runtime/suspension-handler.test.ts | 43 +------------------ .../core/src/runtime/suspension-handler.ts | 11 ----- packages/world-vercel/src/events.test.ts | 11 ++--- packages/world-vercel/src/events.ts | 5 --- 6 files changed, 27 insertions(+), 82 deletions(-) diff --git a/packages/core/src/runtime/resume-hook.test.ts b/packages/core/src/runtime/resume-hook.test.ts index b8c3387ec2..10f86746fc 100644 --- a/packages/core/src/runtime/resume-hook.test.ts +++ b/packages/core/src/runtime/resume-hook.test.ts @@ -40,6 +40,7 @@ describe('resumeHook', () => { attributes: {}, } satisfies WorkflowRun; const createEvent = vi.fn(); + const getEncryptionKeyForRun = vi.fn(); const queue = vi.fn(); setWorld({ @@ -47,6 +48,7 @@ describe('resumeHook', () => { hooks: { getByToken: vi.fn().mockResolvedValue(hook) }, runs: { get: vi.fn().mockResolvedValue(run) }, events: { create: createEvent }, + getEncryptionKeyForRun, queue, } as unknown as World); @@ -54,6 +56,7 @@ describe('resumeHook', () => { HookNotFoundError.is ); expect(createEvent).not.toHaveBeenCalled(); + expect(getEncryptionKeyForRun).not.toHaveBeenCalled(); expect(queue).not.toHaveBeenCalled(); }); }); diff --git a/packages/core/src/runtime/resume-hook.ts b/packages/core/src/runtime/resume-hook.ts index aeecf123ab..60df8d3f7f 100644 --- a/packages/core/src/runtime/resume-hook.ts +++ b/packages/core/src/runtime/resume-hook.ts @@ -28,19 +28,23 @@ import { getWorldLazy } from './get-world-lazy.js'; import { getWorkflowQueueName } from './helpers.js'; import { safeWaitUntil, waitedUntil } from './wait-until.js'; -/** - * Internal helper that returns the hook, the associated workflow run, - * and the resolved encryption key. - */ -async function getHookByTokenWithKey(token: string): Promise<{ +/** Get a Hook and its owning run without hydrating payload data. */ +async function getHookAndRun(token: string): Promise<{ hook: Hook; run: WorkflowRun; - encryptionKey: CryptoKey | undefined; }> { const world = await getWorldLazy(); const hook = await world.hooks.getByToken(token); const run = await world.runs.get(hook.runId); - const rawKey = await world.getEncryptionKeyForRun?.(run); + return { hook, run }; +} + +async function getHookByTokenWithKey(token: string): Promise<{ + hook: Hook; + encryptionKey: CryptoKey | undefined; +}> { + const { hook, run } = await getHookAndRun(token); + const rawKey = await (await getWorldLazy()).getEncryptionKeyForRun?.(run); const encryptionKey = rawKey ? await importKey(rawKey) : undefined; if (typeof hook.metadata !== 'undefined') { hook.metadata = await hydrateStepArguments( @@ -49,7 +53,7 @@ async function getHookByTokenWithKey(token: string): Promise<{ encryptionKey ); } - return { hook, run, encryptionKey }; + return { hook, encryptionKey }; } /** @@ -108,21 +112,13 @@ export async function resumeHook( try { let hook: Hook; let workflowRun: WorkflowRun; - let encryptionKey: CryptoKey | undefined; if (typeof tokenOrHook === 'string') { - const result = await getHookByTokenWithKey(tokenOrHook); + const result = await getHookAndRun(tokenOrHook); hook = result.hook; workflowRun = result.run; - encryptionKey = encryptionKeyOverride ?? result.encryptionKey; } else { hook = tokenOrHook; workflowRun = await world.runs.get(hook.runId); - if (encryptionKeyOverride) { - encryptionKey = encryptionKeyOverride; - } else { - const rawKey = await world.getEncryptionKeyForRun?.(workflowRun); - encryptionKey = rawKey ? await importKey(rawKey) : undefined; - } } span?.setAttributes({ @@ -135,6 +131,12 @@ export async function resumeHook( throw new HookNotFoundError(hook.token); } + let encryptionKey = encryptionKeyOverride; + if (!encryptionKey) { + const rawKey = await world.getEncryptionKeyForRun?.(workflowRun); + encryptionKey = rawKey ? await importKey(rawKey) : undefined; + } + // Check the target run's capabilities to ensure we encode the // payload in a format the run's deployment can decode. For example, // runs created before encryption support was added cannot decode diff --git a/packages/core/src/runtime/suspension-handler.test.ts b/packages/core/src/runtime/suspension-handler.test.ts index 4ebbe13760..404caea4d2 100644 --- a/packages/core/src/runtime/suspension-handler.test.ts +++ b/packages/core/src/runtime/suspension-handler.test.ts @@ -20,12 +20,8 @@ const run: WorkflowRun = { deploymentId: 'test-deployment', }; -function createWorld( - eventsCreate: ReturnType, - capabilities?: World['capabilities'] -): World { +function createWorld(eventsCreate: ReturnType): World { return { - capabilities, events: { create: eventsCreate, }, @@ -38,9 +34,7 @@ describe('handleSuspension', () => { const eventsCreate = vi.fn().mockImplementation(async (_runId, event) => ({ event, })); - const world = createWorld(eventsCreate, { - hookRetention: { active: true }, - }); + const world = createWorld(eventsCreate); const tokenRetentionUntil = new Date('2026-08-01T00:00:00.000Z'); const pending = new Map([ [ @@ -73,39 +67,6 @@ describe('handleSuspension', () => { ); }); - it.each([ - ['missing', undefined], - ['inactive', { hookRetention: { active: false } }], - ] satisfies [ - string, - World['capabilities'], - ][])('rejects hook retention when the capability is %s', async (_state, capabilities) => { - const eventsCreate = vi.fn(); - const world = createWorld(eventsCreate, capabilities); - const pending = new Map([ - [ - 'hook_with_retention', - { - type: 'hook' as const, - correlationId: 'hook_with_retention', - token: 'order:123', - tokenRetentionUntil: new Date('2026-08-01T00:00:00.000Z'), - }, - ], - ]); - - await expect( - handleSuspension({ - suspension: new WorkflowSuspension(pending, globalThis), - world, - run, - }) - ).rejects.toThrow( - 'The configured World does not support `experimental_minRetention` for Hooks.' - ); - expect(eventsCreate).not.toHaveBeenCalled(); - }); - it('marks hook.getConflict()-awaited creations without converting them into wait timeouts', async () => { const eventsCreate = vi.fn().mockResolvedValue({ event: { diff --git a/packages/core/src/runtime/suspension-handler.ts b/packages/core/src/runtime/suspension-handler.ts index c1c979063f..a9c102caeb 100644 --- a/packages/core/src/runtime/suspension-handler.ts +++ b/packages/core/src/runtime/suspension-handler.ts @@ -260,17 +260,6 @@ export async function handleSuspension({ (item) => !item.hasCreatedEvent ); - if ( - hooksNeedingCreation.some( - (item) => item.tokenRetentionUntil !== undefined - ) && - world.capabilities?.hookRetention?.active !== true - ) { - throw new FatalError( - 'The configured World does not support `experimental_minRetention` for Hooks.' - ); - } - // Group hook items that need work by token, preserving queue-insertion // (workflow code) order within each token. Operations on one token must // apply in code order: a dispose() of an earlier hook releases the token diff --git a/packages/world-vercel/src/events.test.ts b/packages/world-vercel/src/events.test.ts index c9ebe29a4b..f78845fdaf 100644 --- a/packages/world-vercel/src/events.test.ts +++ b/packages/world-vercel/src/events.test.ts @@ -204,20 +204,15 @@ describe('createWorkflowRunEvent stateUpdatedAt wire field', () => { * actually reach the frame meta with the right values and renames. */ describe('splitEventDataForV4 attribute fields', () => { - it.each([ - new Date('2026-07-10T12:00:00.000Z'), - '2026-07-10T12:00:00.000Z', - ])('carries a %s Hook retention deadline in the frame meta', (value) => { - const tokenRetentionUntil = new Date( - typeof value === 'string' ? value : value.getTime() - ); + it('carries a Hook retention deadline in the frame meta', () => { + const tokenRetentionUntil = new Date('2026-07-10T12:00:00.000Z'); const { payload, meta } = splitEventDataForV4({ eventType: 'hook_created', correlationId: 'hook_1', specVersion: 5, eventData: { token: 'order:123', - tokenRetentionUntil: value, + tokenRetentionUntil, }, } as AnyEventRequest); diff --git a/packages/world-vercel/src/events.ts b/packages/world-vercel/src/events.ts index 7f4dbe8e51..6284bb6e8b 100644 --- a/packages/world-vercel/src/events.ts +++ b/packages/world-vercel/src/events.ts @@ -293,11 +293,6 @@ export function splitEventDataForV4(data: AnyEventRequest): SplitEventData { } if (eventData.tokenRetentionUntil instanceof Date) { meta.hookTokenRetentionUntil = eventData.tokenRetentionUntil; - } else if (typeof eventData.tokenRetentionUntil === 'string') { - const parsed = new Date(eventData.tokenRetentionUntil); - if (!Number.isNaN(parsed.getTime())) { - meta.hookTokenRetentionUntil = parsed; - } } if (typeof eventData.isWebhook === 'boolean') { meta.hookIsWebhook = eventData.isWebhook; From 0db7b3fe98ed92c152366941127c69d7b212ed79 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Mon, 13 Jul 2026 15:27:24 -0700 Subject: [PATCH 30/36] feat(world-local): support Hook token expiration --- .changeset/local-hook-token-expiration.md | 5 + packages/core/e2e/e2e.test.ts | 33 ++ packages/world-local/package.json | 2 + packages/world-local/src/storage.test.ts | 173 +++++- .../world-local/src/storage/events-storage.ts | 530 ++++++------------ packages/world-local/src/storage/helpers.ts | 61 +- .../src/storage/hook-token-constraint.ts | 53 ++ .../world-local/src/storage/hooks-storage.ts | 264 ++++----- packages/world-local/src/test-helpers.ts | 7 +- pnpm-lock.yaml | 18 + workbench/example/workflows/99_e2e.ts | 26 + 11 files changed, 616 insertions(+), 556 deletions(-) create mode 100644 .changeset/local-hook-token-expiration.md create mode 100644 packages/world-local/src/storage/hook-token-constraint.ts diff --git a/.changeset/local-hook-token-expiration.md b/.changeset/local-hook-token-expiration.md new file mode 100644 index 0000000000..8e75c2d865 --- /dev/null +++ b/.changeset/local-hook-token-expiration.md @@ -0,0 +1,5 @@ +--- +'@workflow/world-local': minor +--- + +Keep Hook tokens reserved after terminal runs until they expire. diff --git a/packages/core/e2e/e2e.test.ts b/packages/core/e2e/e2e.test.ts index 582240e381..32afcf5b65 100644 --- a/packages/core/e2e/e2e.test.ts +++ b/packages/core/e2e/e2e.test.ts @@ -2047,6 +2047,39 @@ describe('e2e', () => { } ); + test.skipIf( + !isLocalDeployment() || + process.env.WORKFLOW_TARGET_WORLD === '@workflow/world-postgres' + )( + 'hookTokenExpirationWorkflow - terminal Hook cannot resume and its token stays unavailable', + { timeout: 60_000 }, + async () => { + const token = `expires-${Math.random().toString(36).slice(2)}`; + const owner = await start(await e2e('hookTokenExpirationWorkflow'), [ + token, + 60_000, + ]); + + expect(await owner.returnValue).toEqual({ role: 'owner' }); + + const hook = await getHookByToken(token); + expect(hook.runId).toBe(owner.runId); + await expect(resumeHook(hook, { duplicate: true })).rejects.toSatisfy( + (error: unknown) => HookNotFoundError.is(error) + ); + + const duplicate = await start(await e2e('hookTokenExpirationWorkflow'), [ + token, + 60_000, + ]); + expect(await duplicate.returnValue).toEqual({ + role: 'duplicate', + conflictRunId: owner.runId, + conflictStatus: 'completed', + }); + } + ); + test( 'hookAdoptOwnerResultWorkflow - duplicate adopts the owner result via conflict.returnValue', { timeout: 120_000 }, diff --git a/packages/world-local/package.json b/packages/world-local/package.json index 7b2eaf4a47..89dc60332e 100644 --- a/packages/world-local/package.json +++ b/packages/world-local/package.json @@ -35,6 +35,7 @@ "@workflow/utils": "workspace:*", "@workflow/world": "workspace:*", "async-sema": "3.1.1", + "proper-lockfile": "4.1.2", "ulid": "catalog:", "undici": "catalog:", "zod": "catalog:" @@ -43,6 +44,7 @@ "@opentelemetry/api": "1.9.0", "@types/ms": "0.7.34", "@types/node": "catalog:", + "@types/proper-lockfile": "4.1.4", "@workflow/tsconfig": "workspace:*", "ms": "2.1.3", "typescript": "catalog:", diff --git a/packages/world-local/src/storage.test.ts b/packages/world-local/src/storage.test.ts index c942018b3d..50539d2282 100644 --- a/packages/world-local/src/storage.test.ts +++ b/packages/world-local/src/storage.test.ts @@ -6,8 +6,12 @@ import type { Event, Storage } from '@workflow/world'; import { SPEC_VERSION_CURRENT, stripEventDataRefs } from '@workflow/world'; import { monotonicFactory } from 'ulid'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { writeJSON } from './fs.js'; -import { hashToken, hookDisposeLockPath } from './storage/helpers.js'; +import { deleteJSON, writeJSON } from './fs.js'; +import { + hashToken, + hookDisposeLockPath, + hookTokenConstraintPath, +} from './storage/helpers.js'; import { createStorage } from './storage.js'; import { completeWait, @@ -2148,12 +2152,8 @@ describe('Storage', () => { }); it('reaps an unparseable claim of a live hook and recovers the real conflict from the event log', async () => { - // A corrupt claim file whose hook is still live must not be trusted as - // debris and stolen — but it also must not block the token forever. - // The claimant loop reaps the unparseable file after a few observations - // (nothing else deletes it — the releaser can't determine ownership), - // then rebuilds the live hook's claim from the event log, so the - // duplicate still conflicts and now carries the real conflicting run. + // Replace the corrupt constraint under its lock, rebuilding the live + // Hook from the event log so the duplicate still conflicts. const token = 'corrupt-claim-live-hook-token'; await createHook(storage, testRunId, { @@ -2193,11 +2193,7 @@ describe('Storage', () => { }); it('reaps an unparseable orphan claim so a new hook can reuse the token (#2808)', async () => { - // Regression for the "unparseable claim blocks its token indefinitely" - // gap: a corrupt claim with no live hook behind it (genuine debris — - // e.g. a partial/corrupted write) is never reaped by the releaser - // (ownership is undeterminable). The claimant loop must delete it after - // N observations so the token becomes claimable again. + // A corrupt constraint with no event behind it must not block the token. const token = 'corrupt-orphan-claim-token'; const claimPath = path.join( testDir, @@ -2241,6 +2237,7 @@ describe('Storage', () => { const hook1 = await createHook(storage, testRunId, { hookId: 'hook_1', token, + tokenExpiresAt: new Date(Date.now() + 60_000), }); expect(hook1.token).toBe(token); @@ -2271,6 +2268,130 @@ describe('Storage', () => { expect(hook2.hookId).toBe('hook_2'); }); + it('keeps a terminal token unavailable until expiration', async () => { + const token = 'retained-token'; + await createHook(storage, testRunId, { + hookId: 'hook_retained_owner', + token, + tokenExpiresAt: new Date(Date.now() + 60_000), + }); + + await updateRun(storage, testRunId, 'run_completed', { + output: new Uint8Array(), + }); + await expect(storage.hooks.getByToken(token)).resolves.toMatchObject({ + runId: testRunId, + token, + }); + + await deleteJSON(hookTokenConstraintPath(testDir, token)); + + const duplicate = await createRun(storage, { + deploymentId: 'deployment-retained-duplicate', + workflowName: 'retained-duplicate', + input: new Uint8Array(), + }); + const result = await storage.events.create(duplicate.runId, { + eventType: 'hook_created', + correlationId: 'hook_retained_duplicate', + eventData: { token }, + }); + + expect(result.event?.eventType).toBe('hook_conflict'); + expect( + result.event?.eventType === 'hook_conflict' + ? result.event.eventData.conflictingRunId + : undefined + ).toBe(testRunId); + expect(result.hook).toBeUndefined(); + }); + + it('uses expiration only after the owning run is terminal', async () => { + const token = 'expiration-boundary-token'; + const tokenExpiresAt = new Date(Date.now() + 60_000); + const replacement = await createRun(storage, { + deploymentId: 'deployment-expiration-boundary', + workflowName: 'expiration-boundary', + input: new Uint8Array(), + }); + await createHook(storage, testRunId, { + hookId: 'hook_expiration_boundary_owner', + token, + tokenExpiresAt, + }); + + const dateNow = vi + .spyOn(Date, 'now') + .mockReturnValue(tokenExpiresAt.getTime()); + try { + const conflict = await storage.events.create(replacement.runId, { + eventType: 'hook_created', + correlationId: 'hook_expiration_boundary_conflict', + eventData: { token }, + }); + expect(conflict.event.eventType).toBe('hook_conflict'); + + await updateRun(storage, testRunId, 'run_completed', { + output: new Uint8Array(), + }); + await expect(storage.hooks.getByToken(token)).rejects.toMatchObject({ + name: 'HookNotFoundError', + }); + const hook = await createHook(storage, replacement.runId, { + hookId: 'hook_expiration_boundary_replacement', + token, + }); + expect(hook.runId).toBe(replacement.runId); + } finally { + dateNow.mockRestore(); + } + }); + + it('admits only one hook when a token expires concurrently', async () => { + const token = 'concurrent-expiry-token'; + const tokenExpiresAt = new Date(); + const workerA = createStorage(testDir); + const workerB = createStorage(testDir); + const contenderA = await createRun(workerA, { + deploymentId: 'deployment-concurrent-expiry-a', + workflowName: 'concurrent-expiry-a', + input: new Uint8Array(), + }); + const contenderB = await createRun(workerB, { + deploymentId: 'deployment-concurrent-expiry-b', + workflowName: 'concurrent-expiry-b', + input: new Uint8Array(), + }); + await createHook(workerA, testRunId, { + hookId: 'hook_concurrent_expiry_owner', + token, + tokenExpiresAt, + }); + await updateRun(workerA, testRunId, 'run_completed', { + output: new Uint8Array(), + }); + + const results = await Promise.all([ + workerA.events.create(contenderA.runId, { + eventType: 'hook_created', + correlationId: 'hook_concurrent_expiry_a', + eventData: { token }, + }), + workerB.events.create(contenderB.runId, { + eventType: 'hook_created', + correlationId: 'hook_concurrent_expiry_b', + eventData: { token }, + }), + ]); + + expect( + results.filter((result) => result.event.eventType === 'hook_created') + ).toHaveLength(1); + expect( + results.filter((result) => result.event.eventType === 'hook_conflict') + ).toHaveLength(1); + }); + // Regression test for #2778: a claim whose owning run is terminal can // never become live again — a new claimant must treat it as vacant // instead of recording a hook_conflict against a finished run. The @@ -3394,6 +3515,32 @@ describe('Storage', () => { expect(conflicts).toHaveLength(0); }); + it('keeps the original expiration during same-Hook recovery', async () => { + const token = 'orphaned-retention-constraint-token'; + const hookId = 'hook_orphaned_retention_constraint'; + const tokenExpiresAt = new Date('2026-08-01T00:00:00.000Z'); + await writeJSON(hookTokenConstraintPath(testDir, token), { + token, + hookId, + runId: testRunId, + eventId: 'evnt_00000000000000000000000000', + tokenExpiresAt, + }); + + const result = await storage.events.create(testRunId, { + eventType: 'hook_created', + correlationId: hookId, + eventData: { + token, + tokenExpiresAt: new Date('2026-09-01T00:00:00.000Z'), + }, + }); + if (result.event.eventType !== 'hook_created') { + throw new Error(`Expected hook_created, got ${result.event.eventType}`); + } + expect(result.event.eventData.tokenExpiresAt).toEqual(tokenExpiresAt); + }); + it('should recover an orphaned hook entity with no matching hook_created event', async () => { // Crash-recovery regression for the second window pranaygp // flagged on PR #2295: the claim file, the hook entity, and diff --git a/packages/world-local/src/storage/events-storage.ts b/packages/world-local/src/storage/events-storage.ts index 7b3a8ddcc2..8c4675227c 100644 --- a/packages/world-local/src/storage/events-storage.ts +++ b/packages/world-local/src/storage/events-storage.ts @@ -1,3 +1,4 @@ +import assert from 'node:assert'; import fs from 'node:fs/promises'; import path from 'node:path'; import { @@ -63,19 +64,24 @@ import { getObjectCreatedAt, hookDisposeLockPath, hookRecoveryMarkerPath, - hookTokenClaimPath, + hookTokenConstraintPath, isHookDisposalCommitted, monotonicUlid, - releaseHookTokenClaimIfOwnedBy, + withHookTokenConstraintLock, } from './helpers.js'; import { deleteHookByRunMarker, writeHookByRunMarker, writeHookCreatedIndexEntries, } from './hook-index.js'; +import { + type HookTokenConstraint, + hasFutureTokenExpiration, + readHookTokenConstraint, +} from './hook-token-constraint.js'; import { deleteAllHooksForRun, - rebuildLiveHookByTokenFromEventLog, + rebuildHookTokenConstraintFromEventLog, } from './hooks-storage.js'; import { handleLegacyEvent } from './legacy.js'; import { withRunFileLock } from './runs-storage.js'; @@ -103,26 +109,6 @@ import { withRunFileLock } from './runs-storage.js'; // but a shared filesystem), exactly matching the cross-process // semantics without spawning subprocesses. -const HookTokenClaimSchema = z.object({ - // The token-claim writer below has always persisted `hookId`, but - // this read schema previously omitted it, which is the bug fixed - // by https://github.com/vercel/workflow/issues/2283. `optional()` - // is defensive: any claim file that somehow lacks the field still - // parses (yielding `undefined`) and falls through to the cross- - // hook conflict branch, matching pre-fix behavior. - hookId: z.string().optional(), - runId: z.string(), - // `eventId` is the canonical hook_created event ID the claiming - // worker committed to publishing. Persisting it here turns the - // claim file into a durable convergence key for cross-worker / - // cross-process retries (see comment on the hook_created branch). - // `optional()` for backward compatibility: a legacy claim file - // written before this field existed falls through to the recovery- - // marker upgrade path, which atomically pins a canonical eventId - // via a sidecar marker (also a `writeExclusive`). - eventId: z.string().optional(), -}); - /** * Sidecar recovery marker that pins a canonical `hook_created` * eventId for a legacy token claim — one written by a version of @@ -144,58 +130,9 @@ const HookRecoveryMarkerSchema = z.object({ eventId: z.string(), }); -async function readHookTokenClaim( - constraintPath: string -): Promise | null> { - try { - return await readJSON(constraintPath, HookTokenClaimSchema); - } catch (error) { - if (error instanceof SyntaxError || error instanceof z.ZodError) { - return null; - } - throw error; - } -} - -/** - * Whether a token claim held by another `(runId, hookId)` can never become - * live again and may therefore be released by a new claimant: - * - * - the claimed hook's disposal is committed (its dispose lock exists — - * the durable release of the claim file just hasn't landed yet, or was - * lost to a crash between the lock write and the claim delete), or - * - the owning run is terminal (the run-completion cleanup releases all - * of the run's claims, but a claimant can observe the claim in the - * window before that cleanup lands — or the cleanup crashed), or - * - the owning run does not exist (a claim can only be written during a - * suspension of an existing run, so an ownerless claim is debris). - * - * A claim from a mid-creation writer is never releasable: its owning run - * exists and is non-terminal, and its dispose lock does not exist. - */ -async function isHookTokenClaimReleasable( - basedir: string, - claim: z.infer, - tag?: string -): Promise { - if ( - claim.hookId && - (await isHookDisposalCommitted(basedir, claim.hookId, tag)) - ) { - return true; - } - const owningRun = await readJSONWithFallback( - basedir, - 'runs', - claim.runId, - WorkflowRunSchema, - tag - ); - if (!owningRun) { - return true; - } - return isTerminalWorkflowRunStatus(owningRun.status); -} +type HookTokenConstraintResult = + | { type: 'claimed' } + | { type: 'existing'; constraint: HookTokenConstraint }; async function readHookRecoveryMarker( markerPath: string @@ -982,6 +919,13 @@ export function createEventsStorage( isHookEventRequiringExistence(data.eventType) && data.correlationId ) { + if ( + data.eventType === 'hook_received' && + currentRun && + isTerminalWorkflowRunStatus(currentRun.status) + ) { + throw new HookNotFoundError(data.correlationId); + } // A resume must never be journaled after the hook's disposal. // The disposer's durable order is: dispose lock → claim/entity // delete → `hook_disposed` append, so the hook entity can still @@ -1010,8 +954,8 @@ export function createEventsStorage( } } // `event` may be reassigned later in the `hook_created` - // dedup-recovery branch to swap in a canonical eventId / - // createdAt persisted in the durable token claim so + // dedup-recovery branch to swap in a canonical eventId and + // its deterministically derived createdAt so // concurrent / cross-process workers converge on a single // event in the log. let event: Event = { @@ -1607,307 +1551,191 @@ export function createEventsStorage( data.eventType === 'hook_created' && 'eventData' in data ) { - const hookData = data.eventData as { - token: string; - metadata?: any; - isWebhook?: boolean; - isSystem?: boolean; - }; + const hookData = data.eventData; - // Atomically claim the token using an exclusive-create constraint file. - // This avoids the TOCTOU race of the previous read-all-then-check approach. - const constraintPath = hookTokenClaimPath(basedir, hookData.token); - // Persist `eventId` in the claim so concurrent / cross- - // process retries can converge on a single canonical - // `hook_created` event path. See the recovery comment - // below. - const claimContent = JSON.stringify({ + const constraintPath = hookTokenConstraintPath( + basedir, + hookData.token + ); + const nextConstraint: HookTokenConstraint = { + type: 'pinned', token: hookData.token, hookId: data.correlationId, runId: effectiveRunId, eventId, - }); + tokenExpiresAt: hookData.tokenExpiresAt, + }; + const constraintContent = JSON.stringify(nextConstraint); - // Bounded claim loop. A same-token claim can be observed in the - // short window where its release is already committed but the - // claim file itself is still on disk (or was just deleted): - // - // - the claim vanished between the exclusive-create attempt - // and the ownership read → retry immediately; - // - the claim is held by a disposed hook or a terminal/missing - // run (`isHookTokenClaimReleasable`) → wait briefly for the - // in-flight releaser (the hook_disposed handler or the - // run-completion cleanup) to delete it, then reclaim; if it - // never does (the releaser crashed after committing its - // dispose lock / terminal status), release the stale claim - // ourselves and reclaim. - // - // Without this, a run claiming a token right after the previous - // owner disposed it records a durable, spurious hook_conflict - // (issue #2778). A genuinely live claim exits the loop on the - // first iteration and falls through to the dedup / conflict - // handling below. - let tokenClaimed = false; - let existingClaim: z.infer | null = null; - let releasableObservations = 0; - // A claim file that exists (exclusive-create keeps failing) but never - // parses is debris: `writeExclusive` writes atomically, so a live - // claim is always valid JSON — an unparseable one at the canonical - // path is a corrupt/partial leftover. The releaser leaves it alone - // (ownership is undeterminable), so nothing else reaps it and it would - // block the token forever. Force-delete it after a few observations. - let unparseableObservations = 0; - for (let attempt = 0; attempt < 10; attempt++) { - // When the claim is absent, the event log is the only durable - // source that can distinguish a first hook from a crash-lost - // token cache. - if (!(await readHookTokenClaim(constraintPath))) { - await rebuildLiveHookByTokenFromEventLog( - basedir, - hookData.token, - tag - ); - } - tokenClaimed = await writeExclusive(constraintPath, claimContent); - if (tokenClaimed) break; - - existingClaim = await readHookTokenClaim(constraintPath); - if (!existingClaim) { - // The claim either vanished (raced a releaser between the - // exclusive-create attempt and this read → retry resolves it) or - // exists but is unparseable. `readHookTokenClaim` can't tell them - // apart, so count consecutive misses; once we're confident it is - // not a transient race, delete the (presumed corrupt) file so the - // token isn't blocked forever. `deleteJSON` is a no-op if it was - // in fact a vanished-claim race. - unparseableObservations++; - if (unparseableObservations >= 3) { + const constraintResult = await withHookTokenConstraintLock( + constraintPath, + async (): Promise => { + let existing = await readHookTokenConstraint(constraintPath); + if (!existing) { await deleteJSON(constraintPath); + existing = await rebuildHookTokenConstraintFromEventLog( + basedir, + hookData.token, + tag + ); } - continue; - } - if ( - existingClaim.runId === effectiveRunId && - existingClaim.hookId === data.correlationId - ) { - // Our own prior claim — dedup-recovery handling below. - break; - } - if ( - !(await isHookTokenClaimReleasable(basedir, existingClaim, tag)) - ) { - // Genuinely live claim — conflict handling below. - break; - } - releasableObservations++; - if (releasableObservations < 3) { - // Give the in-flight releaser a moment to delete the claim. - await new Promise((resolve) => setTimeout(resolve, 10)); - } else { - // The releaser is not coming. Release the stale claim and - // the hook entity it points at, mirroring the hook_disposed - // cleanup. - await deleteJSON(constraintPath); - if (existingClaim.hookId) { - await deleteJSON( - taggedPath(basedir, 'hooks', existingClaim.hookId, tag) + + if (!existing) { + assert( + await writeExclusive(constraintPath, constraintContent), + 'Hook token constraint must be available while locked' + ); + return { type: 'claimed' }; + } + + if ( + existing.runId === effectiveRunId && + existing.hookId === data.correlationId + ) { + return { type: 'existing', constraint: existing }; + } + + if ( + !(await isHookDisposalCommitted(basedir, existing.hookId, tag)) + ) { + if (hasFutureTokenExpiration(existing)) { + return { type: 'existing', constraint: existing }; + } + const owner = await readJSONWithFallback( + basedir, + 'runs', + existing.runId, + WorkflowRunSchema, + tag ); - await deleteJSON( + if (owner && !isTerminalWorkflowRunStatus(owner.status)) { + return { type: 'existing', constraint: existing }; + } + } + + await deleteJSON(constraintPath); + await Promise.all([ + deleteJSON(taggedPath(basedir, 'hooks', existing.hookId, tag)), + deleteJSON( hookRecoveryMarkerPath( basedir, hookData.token, - existingClaim.runId, - existingClaim.hookId + existing.runId, + existing.hookId ) - ); - } + ), + deleteHookByRunMarker( + basedir, + existing.runId, + existing.hookId, + tag + ), + ]); + assert( + await writeExclusive(constraintPath, constraintContent), + 'Hook token constraint must be available after reuse' + ); + return { type: 'claimed' }; } - existingClaim = null; - } + ); - // Recovery shape: the durable record of a successful hook - // creation is the `hook_created` event in the event log. The - // claim file and hook entity are written before the event, - // and the three writes are NOT atomic, so a crash at any - // point can leave one or two of them on disk without the - // event. Treating those as "completed" would have the - // suspension handler swallow the retry and permanently lose - // the `hook_created` event from the log. - // - // When the dedup branch fires for the same `(runId, hookId)`, - // we converge on the canonical `eventId` persisted in the - // claim file by the original (winning) `writeExclusive`. By - // adopting that eventId for this retry's event write — and - // letting the outer no-overwrite `writeJSON` for the event - // throw `EntityConflictError` on collision — concurrent / - // cross-process workers either: - // - publish the same event at the same path exactly once - // (the loser's `writeJSON` throws EntityConflictError, - // which the runtime's existing concurrent-replay catch - // path at suspension-handler.ts:142 swallows), or - // - converge on a single recovery write when the prior - // claim was orphaned by a crash before the event landed. - // - // The legacy fallback (`existingClaim.eventId` undefined) - // is for claim files written before this field was added — - // those probe the event log directly and fall through to a - // fresh-eventId recovery write. The legacy path does not - // converge across workers but cannot regress behavior for - // freshly-written claims. - // - // The `withHookLock` in-process mutex above keeps two same- - // tick in-process callers from racing into this branch with - // the winner mid-write, but is not sufficient across - // processes — the durable convergence key (`claim.eventId`) - // is what closes the cross-process race. let writeHookEntityWithOverwrite = false; + switch (constraintResult.type) { + case 'claimed': + break; + case 'existing': { + const { constraint } = constraintResult; + if ( + constraint.runId !== effectiveRunId || + constraint.hookId !== data.correlationId + ) { + const conflictEvent: Event = { + eventType: 'hook_conflict', + correlationId: data.correlationId, + eventData: { + token: hookData.token, + conflictingRunId: constraint.runId, + }, + runId: effectiveRunId, + eventId, + createdAt: now, + specVersion: effectiveSpecVersion, + }; + await storeEvent(conflictEvent); + return { + event: stripEventDataRefs( + conflictEvent, + params?.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION + ), + run, + step, + hook: undefined, + }; + } - if (!tokenClaimed) { - if ( - existingClaim?.runId === effectiveRunId && - existingClaim.hookId === data.correlationId - ) { - // Adopt a canonical eventId for the recovery write. The - // outer event publish (`writeExclusive(eventPath)`) - // either succeeds (we publish the canonical event, - // repairing a partial write left by the original - // claimant) or returns `false` and we throw - // `EntityConflictError` (the event was already - // published — a real duplicate). Either way the log - // ends with exactly one `hook_created` event for this - // `(runId, hookId)`. - // - // The canonical eventId comes from one of two places: - // - // - `existingClaim.eventId` for claims written by - // this version (the writer above persists the - // candidate eventId atomically with the claim). - // The eventId is durable, so the outer - // `writeExclusive(eventPath)` alone is enough to - // arbitrate publication: it fails iff the event - // was already published at that exact path. - // - // - The recovery-marker sidecar for legacy claims - // written before `eventId` was persisted inline - // in the claim. The marker is itself a - // `writeExclusive`, so the first retry pins its - // candidate eventId as canonical and subsequent - // retries adopt it. Without this, two processes - // both reading the same legacy claim would each - // generate their own eventId, land their - // `writeExclusive(eventPath)` calls at different - // paths, and append two events. - // - // For legacy claims we also must probe the event - // log for an existing `hook_created` event BEFORE - // pinning a canonical eventId: the pre-upgrade - // writer may have already published the event - // with its own eventId, and the marker has no way - // of knowing that eventId after the fact. Without - // this probe, a post-upgrade retry would pin a - // different eventId, write a hook entity, and - // publish a duplicate event at the marker's path. - let canonicalEventId: string; - if (existingClaim.eventId) { - canonicalEventId = existingClaim.eventId; - } else { - const alreadyPublishedEventId = - await findExistingHookCreatedEventId( + switch (constraint.type) { + case 'pinned': + eventId = constraint.eventId; + break; + case 'legacy': { + const existingEventId = await findExistingHookCreatedEventId( basedir, effectiveRunId, data.correlationId ); - if (alreadyPublishedEventId !== null) { - // The pre-upgrade writer may have crashed between - // its event publish and its hook entity write — - // repair the entity from the persisted event's - // payload before surfacing the benign duplicate. - await repairHookEntityFromPersistedEvent( + if (existingEventId) { + await repairHookEntityFromPersistedEvent( + basedir, + effectiveRunId, + data.correlationId, + existingEventId, + tag + ); + throw new EntityConflictError( + `Hook "${data.correlationId}" already created` + ); + } + const pinned = await pinCanonicalEventIdForLegacyClaim( basedir, + hookData.token, effectiveRunId, data.correlationId, - alreadyPublishedEventId, - tag + eventId ); - throw new EntityConflictError( - `Hook "${data.correlationId}" already created` + assert( + pinned, + 'Legacy Hook constraint must have an event ID' ); + eventId = pinned; + break; } - const pinned = await pinCanonicalEventIdForLegacyClaim( - basedir, - hookData.token, - effectiveRunId, - data.correlationId, - eventId - ); - if (pinned === null) { - // Lost the marker race and the marker file is - // unreadable (extremely rare; corrupted disk). - // Treat as a real duplicate so the runtime's - // concurrent-replay catch path swallows this - // attempt instead of risking divergent - // publication. - throw new EntityConflictError( - `Hook "${data.correlationId}" already created` + default: { + const unknownConstraint: never = constraint; + throw new Error( + `Unknown Hook token constraint: ${JSON.stringify(unknownConstraint)}` ); } - canonicalEventId = pinned; } - // Rebuild `event` with the canonical eventId and a - // deterministic `createdAt` derived from the eventId - // (a ULID) so two workers writing the same event - // produce byte-identical content. - eventId = canonicalEventId; - const canonicalCreatedAt = - ulidToDate(eventId.replace(/^evnt_/, '')) ?? now; event = { ...data, - runId: effectiveRunId, - eventId, - createdAt: canonicalCreatedAt, - specVersion: effectiveSpecVersion, - }; - writeHookEntityWithOverwrite = true; - } else { - // Cross-hook / cross-run conflict: a different - // (runId, hookId) holds this token. Create a - // hook_conflict event so the workflow can fail - // gracefully when the hook is awaited. - const conflictEvent: Event = { - eventType: 'hook_conflict', - correlationId: data.correlationId, eventData: { - token: hookData.token, - ...(existingClaim - ? { conflictingRunId: existingClaim.runId } - : {}), + ...data.eventData, + tokenExpiresAt: constraint.tokenExpiresAt, }, runId: effectiveRunId, eventId, - createdAt: now, + createdAt: ulidToDate(eventId.replace(/^evnt_/, '')) ?? now, specVersion: effectiveSpecVersion, }; - - // Persist and cache the conflict event (create-only, - // same path the read cache keys on) so an immediate - // replay can serve it without rereading from disk. - await storeEvent(conflictEvent); - - const resolveData = - params?.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION; - const filteredEvent = stripEventDataRefs( - conflictEvent, - resolveData + writeHookEntityWithOverwrite = true; + break; + } + default: { + const unknownResult: never = constraintResult; + throw new Error( + `Unknown Hook token constraint result: ${JSON.stringify(unknownResult)}` ); - - // Return EventResult with conflict event (no hook entity created) - return { - event: filteredEvent, - run, - step, - hook: undefined, - }; } } @@ -1994,23 +1822,19 @@ export function createEventsStorage( tag ); if (existingHook) { - // Release the token claim to free up the token for reuse — - // but only if it still points at this hook. A claimant that - // force-released this hook's stale claim (see - // `isHookTokenClaimReleasable`) may already hold a fresh - // claim for the token; deleting unconditionally here would - // destroy that live claim and transiently break token - // uniqueness. Also delete this hook's recovery marker (if - // any) for disk hygiene. The marker's filename hash - // includes `(token, runId, hookId)` so different - // lifetimes never collide, but cleaning up reduces disk - // leak for hooks that go through the recovery path. - await releaseHookTokenClaimIfOwnedBy( + const constraintPath = hookTokenConstraintPath( basedir, - existingHook.token, - existingHook.runId, - existingHook.hookId + existingHook.token ); + await withHookTokenConstraintLock(constraintPath, async () => { + const constraint = await readHookTokenConstraint(constraintPath); + if ( + constraint?.runId === existingHook.runId && + constraint.hookId === existingHook.hookId + ) { + await deleteJSON(constraintPath); + } + }); await deleteJSON( hookRecoveryMarkerPath( basedir, diff --git a/packages/world-local/src/storage/helpers.ts b/packages/world-local/src/storage/helpers.ts index 93534710dd..27c089d9cd 100644 --- a/packages/world-local/src/storage/helpers.ts +++ b/packages/world-local/src/storage/helpers.ts @@ -1,6 +1,7 @@ import { createHash } from 'node:crypto'; import fs from 'node:fs/promises'; import path from 'node:path'; +import { lock } from 'proper-lockfile'; import { monotonicFactory } from 'ulid'; import { resolveWithinBase, stripTag, ulidToDate } from '../fs.js'; @@ -14,7 +15,7 @@ export function hashToken(token: string): string { /** * Path of the exclusive-create lock file that commits a hook's disposal. * The `hook_disposed` handler writes this lock BEFORE deleting the token - * claim and hook entity (and before appending the event to the log), so + * constraint and hook entity (and before appending the event to the log), so * its existence is the earliest durable evidence that the hook can never * be live again. */ @@ -53,47 +54,29 @@ export async function isHookDisposalCommitted( } /** - * Path of the exclusive-create claim file that reserves a hook token. + * Path of the file that reserves a hook token. */ -export function hookTokenClaimPath(basedir: string, token: string): string { +export function hookTokenConstraintPath( + basedir: string, + token: string +): string { return path.join(basedir, 'hooks', 'tokens', `${hashToken(token)}.json`); } -/** - * Release (delete) a token claim only if it still points at the releasing - * hook's own `(runId, hookId)`. - * - * Both in-flight releasers — the `hook_disposed` handler and the - * terminal-run `deleteAllHooksForRun` cleanup — read the hook entity and - * then delete the claim file. Deleting unconditionally is unsafe across - * processes: a releaser that stalls between those two operations can - * outlive a force-release of its stale claim (see - * `isHookTokenClaimReleasable`) and then delete the NEXT claimant's live - * claim, transiently breaking token uniqueness. Re-reading the claim and - * matching its identity here shrinks that window from "a stall of any - * length" to the adjacent read/delete file ops. - * - * A claim that is missing, unreadable, or owned by someone else is left - * alone — if it is genuinely stale debris, the claimant-side force-release - * path reaps it. - */ -export async function releaseHookTokenClaimIfOwnedBy( - basedir: string, - token: string, - runId: string, - hookId: string -): Promise { - const claimPath = hookTokenClaimPath(basedir, token); - let claim: { runId?: unknown; hookId?: unknown }; +export async function withHookTokenConstraintLock( + constraintPath: string, + fn: () => Promise +): Promise { + await fs.mkdir(path.dirname(constraintPath), { recursive: true }); + const release = await lock(constraintPath, { + realpath: false, + retries: { forever: true, minTimeout: 10, maxTimeout: 100 }, + }); try { - claim = JSON.parse(await fs.readFile(claimPath, 'utf8')); - } catch { - return; - } - if (claim.runId !== runId || claim.hookId !== hookId) { - return; + return await fn(); + } finally { + await release(); } - await fs.unlink(claimPath).catch(() => {}); } /** @@ -114,12 +97,10 @@ export function hookRecoveryMarkerPath( runId: string, hookId: string ): string { - // Distinct from `hashToken(token)` so a token's claim file and + // Distinct from `hashToken(token)` so a token's constraint file and // its recovery marker live at different paths AND a different // lifetime's recovery marker never collides with this one. - const key = createHash('sha256') - .update(`${token}\x00${runId}\x00${hookId}`) - .digest('hex'); + const key = hashToken(`${token}\x00${runId}\x00${hookId}`); return path.join(basedir, 'hooks', 'tokens', `${key}.recovery.json`); } diff --git a/packages/world-local/src/storage/hook-token-constraint.ts b/packages/world-local/src/storage/hook-token-constraint.ts new file mode 100644 index 0000000000..c25fdbbf0b --- /dev/null +++ b/packages/world-local/src/storage/hook-token-constraint.ts @@ -0,0 +1,53 @@ +import { z } from 'zod'; +import { readJSON } from '../fs.js'; + +const HookTokenConstraintFields = { + token: z.string(), + runId: z.string(), + hookId: z.string(), + tokenExpiresAt: z.coerce.date().optional(), +}; + +const HookTokenConstraintSchema = z + .union([ + z.object({ + ...HookTokenConstraintFields, + type: z.literal('pinned'), + eventId: z.string(), + }), + z.object({ + ...HookTokenConstraintFields, + type: z.undefined().optional(), + eventId: z.string().optional(), + }), + ]) + .transform((constraint) => { + if (constraint.type === 'pinned') return constraint; + const { eventId, ...legacy } = constraint; + return eventId + ? { ...legacy, type: 'pinned' as const, eventId } + : { ...legacy, type: 'legacy' as const }; + }); + +export type HookTokenConstraint = z.infer; + +export async function readHookTokenConstraint( + path: string +): Promise { + try { + return await readJSON(path, HookTokenConstraintSchema); + } catch (error) { + if (error instanceof SyntaxError || error instanceof z.ZodError) + return null; + throw error; + } +} + +export function hasFutureTokenExpiration( + constraint: Pick +): boolean { + return ( + constraint.tokenExpiresAt !== undefined && + Date.now() < constraint.tokenExpiresAt.getTime() + ); +} diff --git a/packages/world-local/src/storage/hooks-storage.ts b/packages/world-local/src/storage/hooks-storage.ts index ea4a18619d..aeb7f49ede 100644 --- a/packages/world-local/src/storage/hooks-storage.ts +++ b/packages/world-local/src/storage/hooks-storage.ts @@ -14,27 +14,23 @@ import { isTerminalWorkflowRunStatus, WorkflowRunSchema, } from '@workflow/world'; -import { z } from 'zod'; import { DEFAULT_RESOLVE_DATA_OPTION } from '../config.js'; import { assertSafeEntityId, deleteJSON, jsonReplacer, - listJSONFiles, paginatedFileSystemQuery, readJSON, readJSONWithFallback, taggedPath, - UnsafeEntityIdError, writeExclusive, } from '../fs.js'; import { filterHookData } from './filters.js'; import { - hashToken, hookRecoveryMarkerPath, - hookTokenClaimPath, + hookTokenConstraintPath, isHookDisposalCommitted, - releaseHookTokenClaimIfOwnedBy, + withHookTokenConstraintLock, } from './helpers.js'; import { deleteHookByRunMarkerFile, @@ -43,11 +39,20 @@ import { listHookByRunMarkers, writeHookByRunMarker, } from './hook-index.js'; +import { + type HookTokenConstraint, + hasFutureTokenExpiration, + readHookTokenConstraint, +} from './hook-token-constraint.js'; -function getHookCreatedToken(event: Event): string | undefined { - if (event.eventType !== 'hook_created') return undefined; - const token = (event.eventData as { token?: unknown }).token; - return typeof token === 'string' ? token : undefined; +function isMatchingHookCreatedEvent( + event: Event, + index: { kind: 'token'; token: string } | { kind: 'id'; hookId: string } +): event is HookCreatedEvent { + if (event.eventType !== 'hook_created') return false; + return index.kind === 'token' + ? event.eventData.token === index.token + : event.correlationId === index.hookId; } function hookFromCreatedEvent(event: HookCreatedEvent): Hook { @@ -67,18 +72,7 @@ function hookFromCreatedEvent(event: HookCreatedEvent): Hook { }; } -function isMatchingHookCreatedEvent( - event: Event, - matches: (event: Event) => boolean -): event is HookCreatedEvent { - return ( - event.eventType === 'hook_created' && - typeof event.correlationId === 'string' && - matches(event) - ); -} - -async function isTerminalRunCache( +async function isRunActive( basedir: string, runId: string, tag?: string @@ -90,7 +84,7 @@ async function isTerminalRunCache( WorkflowRunSchema, tag ); - return run ? isTerminalWorkflowRunStatus(run.status) : false; + return run !== null && !isTerminalWorkflowRunStatus(run.status); } /** @@ -106,57 +100,51 @@ async function isTerminalRunCache( async function findLiveHookCreatedEvent( basedir: string, index: { kind: 'token'; token: string } | { kind: 'id'; hookId: string }, - matches: (event: Event) => boolean, tag?: string ): Promise { - const newest = await findNewestIndexedHookCreatedEvent( + const event = await findNewestIndexedHookCreatedEvent( basedir, index, - (event) => isMatchingHookCreatedEvent(event, matches), + (candidate) => isMatchingHookCreatedEvent(candidate, index), tag ); - if (!newest || !isMatchingHookCreatedEvent(newest, matches)) { - return null; - } + if (!event || !isMatchingHookCreatedEvent(event, index)) return null; - if (await isTerminalRunCache(basedir, newest.runId, tag)) { + if (!(await isRunActive(basedir, event.runId, tag))) { return null; } - // A committed disposal (dispose lock on disk) closes the hook even when - // its `hook_disposed` event has not landed in the log yet — the disposer - // writes the lock, releases the token claim and hook entity, and only - // then appends the event. Rebuilding the caches from the log in that - // window would resurrect a claim for a hook that is being torn down. - if (await isHookDisposalCommitted(basedir, newest.correlationId, tag)) { + // A committed disposal closes the hook before its event is appended. + if (await isHookDisposalCommitted(basedir, event.correlationId, tag)) { return null; } - return newest; + return event; +} + +function hookTokenConstraintFromEvent( + event: HookCreatedEvent +): HookTokenConstraint { + return { + type: 'pinned', + token: event.eventData.token, + hookId: event.correlationId, + runId: event.runId, + eventId: event.eventId, + tokenExpiresAt: event.eventData.tokenExpiresAt, + }; } async function restoreHookCachesFromEvent( basedir: string, event: HookCreatedEvent, - tag?: string + tag: string | undefined ): Promise { - const hook = hookFromCreatedEvent(event); - - const claimPath = path.join( - basedir, - 'hooks', - 'tokens', - `${hashToken(hook.token)}.json` - ); await writeExclusive( - claimPath, - JSON.stringify({ - token: hook.token, - hookId: hook.hookId, - runId: hook.runId, - eventId: event.eventId, - }) + hookTokenConstraintPath(basedir, event.eventData.token), + JSON.stringify(hookTokenConstraintFromEvent(event)) ); + const hook = hookFromCreatedEvent(event); // Marker before entity (see hook-index.ts crash-ordering invariant). await writeHookByRunMarker(basedir, hook.runId, hook.hookId, tag); await writeExclusive( @@ -167,6 +155,34 @@ async function restoreHookCachesFromEvent( return hook; } +export async function rebuildHookTokenConstraintFromEventLog( + basedir: string, + token: string, + tag?: string +): Promise { + const event = await findNewestIndexedHookCreatedEvent( + basedir, + { kind: 'token', token }, + (candidate) => + candidate.eventType === 'hook_created' && + candidate.eventData.token === token, + tag + ); + if (!event || event.eventType !== 'hook_created') return null; + if (await isHookDisposalCommitted(basedir, event.correlationId, tag)) { + return null; + } + + if ( + !(await isRunActive(basedir, event.runId, tag)) && + !hasFutureTokenExpiration(event.eventData) + ) { + return null; + } + await restoreHookCachesFromEvent(basedir, event, tag); + return hookTokenConstraintFromEvent(event); +} + export async function rebuildLiveHookByTokenFromEventLog( basedir: string, token: string, @@ -175,7 +191,6 @@ export async function rebuildLiveHookByTokenFromEventLog( const event = await findLiveHookCreatedEvent( basedir, { kind: 'token', token }, - (candidate) => getHookCreatedToken(candidate) === token, tag ); return event ? restoreHookCachesFromEvent(basedir, event, tag) : null; @@ -189,7 +204,6 @@ async function rebuildLiveHookByIdFromEventLog( const event = await findLiveHookCreatedEvent( basedir, { kind: 'id', hookId }, - (candidate) => candidate.correlationId === hookId, tag ); return event ? restoreHookCachesFromEvent(basedir, event, tag) : null; @@ -203,81 +217,35 @@ export function createHooksStorage( basedir: string, tag?: string ): Storage['hooks'] { - const TokenClaimPointerSchema = z.object({ - hookId: z.string().optional(), - }); - async function findHookByToken(token: string): Promise { - // Fast path: the token claim file points at the owning hookId. - let claim: z.infer | null = null; - try { - claim = await readJSON( - hookTokenClaimPath(basedir, token), - TokenClaimPointerSchema - ); - } catch (error) { - if (!(error instanceof SyntaxError || error instanceof z.ZodError)) { - throw error; - } - } - if (claim?.hookId) { - try { - const hook = await readJSONWithFallback( - basedir, - 'hooks', - claim.hookId, - HookSchema, - tag - ); - if (hook && hook.token === token) { - return { ...hook, isWebhook: hook.isWebhook ?? true }; - } - } catch (error) { - if (!UnsafeEntityIdError.is(error)) { - throw error; - } - } - } - - // Slow path for legacy states (e.g. a lost claim file while the - // entity is still on disk). - const hooksDir = path.join(basedir, 'hooks'); - const files = await listJSONFiles(hooksDir); - - for (const file of files) { - const hookPath = path.join(hooksDir, `${file}.json`); - const hook = await readJSON(hookPath, HookSchema); - if (hook && hook.token === token) { - return { ...hook, isWebhook: hook.isWebhook ?? true }; - } - } - - return null; - } - - async function get(hookId: string, params?: GetHookParams): Promise { - assertSafeEntityId('hookId', hookId); + const constraint = await readHookTokenConstraint( + hookTokenConstraintPath(basedir, token) + ); + if (!constraint) return null; const hook = await readJSONWithFallback( basedir, 'hooks', - hookId, + constraint.hookId, HookSchema, tag ); + if (!hook || hook.token !== token) return null; + if ( + !(await isRunActive(basedir, hook.runId, tag)) && + !hasFutureTokenExpiration(constraint) + ) { + return null; + } + return { ...hook, isWebhook: hook.isWebhook ?? true }; + } + + async function get(hookId: string, params?: GetHookParams): Promise { + assertSafeEntityId('hookId', hookId); + const hook = + (await readJSONWithFallback(basedir, 'hooks', hookId, HookSchema, tag)) ?? + (await rebuildLiveHookByIdFromEventLog(basedir, hookId, tag)); if (!hook) { - const rebuilt = await rebuildLiveHookByIdFromEventLog( - basedir, - hookId, - tag - ); - if (!rebuilt) { - throw new HookNotFoundError(hookId); - } - const resolveData = params?.resolveData || DEFAULT_RESOLVE_DATA_OPTION; - return filterHookData( - { ...rebuilt, isWebhook: rebuilt.isWebhook ?? true }, - resolveData - ); + throw new HookNotFoundError(hookId); } const resolveData = params?.resolveData || DEFAULT_RESOLVE_DATA_OPTION; return filterHookData( @@ -290,9 +258,7 @@ export function createHooksStorage( const hook = (await findHookByToken(token)) ?? (await rebuildLiveHookByTokenFromEventLog(basedir, token, tag)); - if (!hook) { - throw new HookNotFoundError(token); - } + if (!hook) throw new HookNotFoundError(token); return hook; } @@ -348,36 +314,36 @@ export async function deleteAllHooksForRun( await ensureHookIndexes(basedir); for (const marker of await listHookByRunMarkers(basedir, runId)) { - if (marker.hookId) { - let hook: Hook | null = null; - let hookPath: string | null = null; - try { - hookPath = taggedPath(basedir, 'hooks', marker.hookId, marker.tag); - hook = await readJSON(hookPath, HookSchema); - } catch (error) { - if ( - !UnsafeEntityIdError.is(error) && - !(error instanceof SyntaxError || error instanceof z.ZodError) - ) { - throw error; + if (!marker.hookId) { + await deleteHookByRunMarkerFile(basedir, marker.fileId); + continue; + } + const hookPath = taggedPath(basedir, 'hooks', marker.hookId, marker.tag); + const hook = await readJSON(hookPath, HookSchema); + if (hook?.runId === runId) { + const constraintPath = hookTokenConstraintPath(basedir, hook.token); + const keepHook = await withHookTokenConstraintLock( + constraintPath, + async () => { + const constraint = await readHookTokenConstraint(constraintPath); + const owned = + constraint?.runId === hook.runId && + constraint.hookId === hook.hookId; + if (owned && hasFutureTokenExpiration(constraint)) return true; + if (owned) await deleteJSON(constraintPath); + return false; } - } - if (hook && hookPath && hook.runId === runId) { - // Release the claim only if it still points at this hook — a - // claimant may already hold a fresh claim for the token (see - // `isHookTokenClaimReleasable`). - await releaseHookTokenClaimIfOwnedBy( - basedir, - hook.token, - hook.runId, - hook.hookId - ); + ); + + if (!keepHook) { await deleteJSON( hookRecoveryMarkerPath(basedir, hook.token, hook.runId, hook.hookId) ); await deleteJSON(hookPath); + await deleteHookByRunMarkerFile(basedir, marker.fileId); } + } else { + await deleteHookByRunMarkerFile(basedir, marker.fileId); } - await deleteHookByRunMarkerFile(basedir, marker.fileId); } } diff --git a/packages/world-local/src/test-helpers.ts b/packages/world-local/src/test-helpers.ts index 7af454bebb..bf52a549ad 100644 --- a/packages/world-local/src/test-helpers.ts +++ b/packages/world-local/src/test-helpers.ts @@ -116,6 +116,7 @@ export async function createHook( data: { hookId: string; token: string; + tokenExpiresAt?: Date; metadata?: SerializedData; } ): Promise { @@ -123,7 +124,11 @@ export async function createHook( eventType: 'hook_created', specVersion: SPEC_VERSION_CURRENT, correlationId: data.hookId, - eventData: { token: data.token, metadata: data.metadata }, + eventData: { + token: data.token, + tokenExpiresAt: data.tokenExpiresAt, + metadata: data.metadata, + }, }); if (!result.hook) { throw new Error('Expected hook to be created'); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 203cf11198..7fd5abc7de 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1373,6 +1373,9 @@ importers: async-sema: specifier: 3.1.1 version: 3.1.1 + proper-lockfile: + specifier: 4.1.2 + version: 4.1.2 ulid: specifier: 'catalog:' version: 3.0.1 @@ -1392,6 +1395,9 @@ importers: '@types/node': specifier: 'catalog:' version: 22.19.0 + '@types/proper-lockfile': + specifier: 4.1.4 + version: 4.1.4 '@workflow/tsconfig': specifier: workspace:* version: link:../tsconfig @@ -8937,9 +8943,15 @@ packages: '@types/pg@8.20.0': resolution: {integrity: sha512-bEPFOaMAHTEP1EzpvHTbmwR8UsFyHSKsRisLIHVMXnpNefSbGA1bD6CVy+qKjGSqmZqNqBDV2azOBo8TgkcVow==} + '@types/proper-lockfile@4.1.4': + resolution: {integrity: sha512-uo2ABllncSqg9F1D4nugVl9v93RmjxF6LJzQLMLDdPaXCUIDPeOJ21Gbqi43xNKzBi/WQ0Q0dICqufzQbMjipQ==} + '@types/qs@6.14.0': resolution: {integrity: sha512-eOunJqu0K1923aExK6y8p6fsihYEn/BYuQ4g0CxAAgFc4b/ZLN4CrsRZ55srTdqoiLzU2B2evC+apEIxprEzkQ==} + '@types/retry@0.12.5': + resolution: {integrity: sha512-3xSjTp3v03X/lSQLkczaN9UIEwJMoMCA1+Nb5HfbJEQWogdeQIyVtTvxPXDQjZ5zws8rFQfVfRdz03ARihPJgw==} + '@types/range-parser@1.2.7': resolution: {integrity: sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==} @@ -24652,8 +24664,14 @@ snapshots: pg-protocol: 1.10.3 pg-types: 2.2.0 + '@types/proper-lockfile@4.1.4': + dependencies: + '@types/retry': 0.12.5 + '@types/qs@6.14.0': {} + '@types/retry@0.12.5': {} + '@types/range-parser@1.2.7': {} '@types/react-dom@19.1.9(@types/react@19.1.13)': diff --git a/workbench/example/workflows/99_e2e.ts b/workbench/example/workflows/99_e2e.ts index d9df9bc703..3d93cfa5fd 100644 --- a/workbench/example/workflows/99_e2e.ts +++ b/workbench/example/workflows/99_e2e.ts @@ -746,6 +746,32 @@ export async function hookGetConflictThenStepParallelWorkflow( // docs/content/docs/*/foundations/idempotency.mdx. ////////////////////////////////////////////////////////// +/** + * Keeps its token reserved after this run ends. The Hook is intentionally left + * undisposed so duplicates continue to receive a conflict until it expires. + */ +export async function hookTokenExpirationWorkflow( + token: string, + expiresInMs: number +) { + 'use workflow'; + + const hook = createHook({ + token, + experimental_expires: expiresInMs, + }); + const conflict = await hook.getConflict(); + if (conflict) { + return { + role: 'duplicate' as const, + conflictRunId: conflict.runId, + conflictStatus: await conflict.status, + }; + } + + return { role: 'owner' as const }; +} + /** * Claim-only run mutex: the hook is used purely for run idempotency — * the workflow claims the token, holds it while doing unrelated work, From 2d20344db0b72bfdd37139e83ed3900aa303abe0 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Tue, 14 Jul 2026 14:06:08 -0700 Subject: [PATCH 31/36] fix(world-local): make hook recovery atomic --- packages/world-local/src/storage.test.ts | 157 +++---- .../world-local/src/storage/events-storage.ts | 396 ++++++++---------- packages/world-local/src/storage/helpers.ts | 38 +- .../world-local/src/storage/hook-index.ts | 59 ++- .../src/storage/hook-token-constraint.ts | 33 +- .../world-local/src/storage/hooks-storage.ts | 282 ++++++++----- packages/world-local/src/test-helpers.ts | 9 +- 7 files changed, 497 insertions(+), 477 deletions(-) diff --git a/packages/world-local/src/storage.test.ts b/packages/world-local/src/storage.test.ts index 50539d2282..26f97daf9f 100644 --- a/packages/world-local/src/storage.test.ts +++ b/packages/world-local/src/storage.test.ts @@ -6,12 +6,9 @@ import type { Event, Storage } from '@workflow/world'; import { SPEC_VERSION_CURRENT, stripEventDataRefs } from '@workflow/world'; import { monotonicFactory } from 'ulid'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { deleteJSON, writeJSON } from './fs.js'; -import { - hashToken, - hookDisposeLockPath, - hookTokenConstraintPath, -} from './storage/helpers.js'; +import { writeJSON } from './fs.js'; +import { hashToken, hookDisposeLockPath } from './storage/helpers.js'; +import { hookTokenConstraintPath } from './storage/hook-token-constraint.js'; import { createStorage } from './storage.js'; import { completeWait, @@ -2268,7 +2265,7 @@ describe('Storage', () => { expect(hook2.hookId).toBe('hook_2'); }); - it('keeps a terminal token unavailable until expiration', async () => { + it('rebuilds a retained Hook and keeps its token unavailable', async () => { const token = 'retained-token'; await createHook(storage, testRunId, { hookId: 'hook_retained_owner', @@ -2279,13 +2276,13 @@ describe('Storage', () => { await updateRun(storage, testRunId, 'run_completed', { output: new Uint8Array(), }); + // Simulate cache loss: the retained event must rebuild the reservation. + await fs.rm(hookTokenConstraintPath(testDir, token)); await expect(storage.hooks.getByToken(token)).resolves.toMatchObject({ runId: testRunId, token, }); - await deleteJSON(hookTokenConstraintPath(testDir, token)); - const duplicate = await createRun(storage, { deploymentId: 'deployment-retained-duplicate', workflowName: 'retained-duplicate', @@ -2297,13 +2294,13 @@ describe('Storage', () => { eventData: { token }, }); - expect(result.event?.eventType).toBe('hook_conflict'); - expect( - result.event?.eventType === 'hook_conflict' - ? result.event.eventData.conflictingRunId - : undefined - ).toBe(testRunId); - expect(result.hook).toBeUndefined(); + expect(result).toMatchObject({ + event: { + eventType: 'hook_conflict', + eventData: { conflictingRunId: testRunId }, + }, + hook: undefined, + }); }); it('uses expiration only after the owning run is terminal', async () => { @@ -2320,31 +2317,25 @@ describe('Storage', () => { tokenExpiresAt, }); - const dateNow = vi - .spyOn(Date, 'now') - .mockReturnValue(tokenExpiresAt.getTime()); - try { - const conflict = await storage.events.create(replacement.runId, { - eventType: 'hook_created', - correlationId: 'hook_expiration_boundary_conflict', - eventData: { token }, - }); - expect(conflict.event.eventType).toBe('hook_conflict'); + vi.spyOn(Date, 'now').mockReturnValue(tokenExpiresAt.getTime()); + const conflict = await storage.events.create(replacement.runId, { + eventType: 'hook_created', + correlationId: 'hook_expiration_boundary_conflict', + eventData: { token }, + }); + expect(conflict.event.eventType).toBe('hook_conflict'); - await updateRun(storage, testRunId, 'run_completed', { - output: new Uint8Array(), - }); - await expect(storage.hooks.getByToken(token)).rejects.toMatchObject({ - name: 'HookNotFoundError', - }); - const hook = await createHook(storage, replacement.runId, { - hookId: 'hook_expiration_boundary_replacement', - token, - }); - expect(hook.runId).toBe(replacement.runId); - } finally { - dateNow.mockRestore(); - } + await updateRun(storage, testRunId, 'run_completed', { + output: new Uint8Array(), + }); + await expect(storage.hooks.getByToken(token)).rejects.toMatchObject({ + name: 'HookNotFoundError', + }); + const hook = await createHook(storage, replacement.runId, { + hookId: 'hook_expiration_boundary_replacement', + token, + }); + expect(hook.runId).toBe(replacement.runId); }); it('admits only one hook when a token expires concurrently', async () => { @@ -2384,12 +2375,10 @@ describe('Storage', () => { }), ]); - expect( - results.filter((result) => result.event.eventType === 'hook_created') - ).toHaveLength(1); - expect( - results.filter((result) => result.event.eventType === 'hook_conflict') - ).toHaveLength(1); + expect(results.map(({ event }) => event.eventType).sort()).toEqual([ + 'hook_conflict', + 'hook_created', + ]); }); // Regression test for #2778: a claim whose owning run is terminal can @@ -3459,48 +3448,35 @@ describe('Storage', () => { }); it('should recover an orphaned hook token claim with no matching hook entity', async () => { - // Crash-recovery regression: if a prior in-flight `hook_created` - // wrote the token-claim file but exited before the hook entity - // was written, the same-`(runId, hookId)` retry must not be - // treated as a "real duplicate" — that would throw - // EntityConflictError, which the runtime's concurrent-replay - // catch path would swallow, permanently leaving the run with no - // hook and no `hook_created` event in the log. - // - // The recovery path detects the missing hook entity and - // completes the partial write: it (re-)writes the hook entity - // and the outer code path emits the `hook_created` event. + // Simulate a crash after reserving the token but before persisting the + // Hook and event. The same owner must finish creation, not conflict. const token = 'orphaned-claim-token'; const hookId = 'hook_orphan_1'; + const tokenExpiresAt = new Date('2026-08-01T00:00:00.000Z'); - // Pre-seed an orphaned token claim — same shape as one written - // by `events.create` but with no corresponding hook entity on - // disk. This simulates a crash between `writeExclusive(claim)` - // and the hook entity write. - const tokensDir = path.join(testDir, 'hooks', 'tokens'); - await fs.mkdir(tokensDir, { recursive: true }); - await fs.writeFile( - path.join(tokensDir, `${hashToken(token)}.json`), - JSON.stringify({ token, hookId, runId: testRunId, foo: 'bar' }) - ); + await writeJSON(hookTokenConstraintPath(testDir, token), { + token, + hookId, + runId: testRunId, + tokenExpiresAt, + }); - // Sanity: the hook entity is not on disk yet. await expect(storage.hooks.get(hookId)).rejects.toThrow( /not found|HookNotFoundError/i ); - // Retry: must succeed, write the hook entity, and emit a - // hook_created event. - const hook = await createHook(storage, testRunId, { hookId, token }); + // The retry cannot extend the expiration stored by the first attempt. + const hook = await createHook(storage, testRunId, { + hookId, + token, + tokenExpiresAt: new Date('2026-09-01T00:00:00.000Z'), + }); expect(hook.hookId).toBe(hookId); expect(hook.token).toBe(token); - // The hook entity is now durable. const retrieved = await storage.hooks.get(hookId); expect(retrieved.hookId).toBe(hookId); - // The event log contains a hook_created event for this hookId - // (and no hook_conflict event). const events = await storage.events.list({ runId: testRunId, pagination: {}, @@ -3511,36 +3487,14 @@ describe('Storage', () => { const conflicts = events.data.filter( (e) => e.eventType === 'hook_conflict' ); - expect(created).toHaveLength(1); + expect(created).toEqual([ + expect.objectContaining({ + eventData: expect.objectContaining({ tokenExpiresAt }), + }), + ]); expect(conflicts).toHaveLength(0); }); - it('keeps the original expiration during same-Hook recovery', async () => { - const token = 'orphaned-retention-constraint-token'; - const hookId = 'hook_orphaned_retention_constraint'; - const tokenExpiresAt = new Date('2026-08-01T00:00:00.000Z'); - await writeJSON(hookTokenConstraintPath(testDir, token), { - token, - hookId, - runId: testRunId, - eventId: 'evnt_00000000000000000000000000', - tokenExpiresAt, - }); - - const result = await storage.events.create(testRunId, { - eventType: 'hook_created', - correlationId: hookId, - eventData: { - token, - tokenExpiresAt: new Date('2026-09-01T00:00:00.000Z'), - }, - }); - if (result.event.eventType !== 'hook_created') { - throw new Error(`Expected hook_created, got ${result.event.eventType}`); - } - expect(result.event.eventData.tokenExpiresAt).toEqual(tokenExpiresAt); - }); - it('should recover an orphaned hook entity with no matching hook_created event', async () => { // Crash-recovery regression for the second window pranaygp // flagged on PR #2295: the claim file, the hook entity, and @@ -3780,7 +3734,6 @@ describe('Storage', () => { testRunId ); - await fs.unlink(hookPath); await fs.unlink(tokenClaimPath); await expect(storage.hooks.get(hookId)).resolves.toMatchObject({ diff --git a/packages/world-local/src/storage/events-storage.ts b/packages/world-local/src/storage/events-storage.ts index 8c4675227c..f776e2ed49 100644 --- a/packages/world-local/src/storage/events-storage.ts +++ b/packages/world-local/src/storage/events-storage.ts @@ -1,4 +1,3 @@ -import assert from 'node:assert'; import fs from 'node:fs/promises'; import path from 'node:path'; import { @@ -54,6 +53,7 @@ import { readJSON, readJSONWithFallback, resolveWithinBase, + stripTag, taggedPath, write, writeExclusive, @@ -64,10 +64,10 @@ import { getObjectCreatedAt, hookDisposeLockPath, hookRecoveryMarkerPath, - hookTokenConstraintPath, isHookDisposalCommitted, monotonicUlid, - withHookTokenConstraintLock, + withFileLock, + withRunEventLock, } from './helpers.js'; import { deleteHookByRunMarker, @@ -75,8 +75,10 @@ import { writeHookCreatedIndexEntries, } from './hook-index.js'; import { + type CurrentHookTokenConstraint, type HookTokenConstraint, hasFutureTokenExpiration, + hookTokenConstraintPath, readHookTokenConstraint, } from './hook-token-constraint.js'; import { @@ -98,17 +100,6 @@ import { withRunFileLock } from './runs-storage.js'; * (retries legitimately re-start a step), only writes to an already-terminal * step are rejected. */ -// `stepLocks` and `hookLocks` are now instantiated per -// `createEventsStorage` call (see inside the function) rather than -// being module-level. The on-disk constraint / claim files remain -// the durable source of truth across processes; the in-process -// mutex is a per-instance optimization that closes a short race -// window in the dedup-recovery path. Per-instance scoping lets -// tests simulate cross-process behavior with two storage instances -// sharing one data directory (each instance has independent locks -// but a shared filesystem), exactly matching the cross-process -// semantics without spawning subprocesses. - /** * Sidecar recovery marker that pins a canonical `hook_created` * eventId for a legacy token claim — one written by a version of @@ -130,9 +121,13 @@ const HookRecoveryMarkerSchema = z.object({ eventId: z.string(), }); -type HookTokenConstraintResult = +type HookTokenReservation = | { type: 'claimed' } - | { type: 'existing'; constraint: HookTokenConstraint }; + | { type: 'owned'; constraint: HookTokenConstraint } + | { type: 'conflict'; runId: string }; +type HookTokenRelease = + | { type: 'retain' } + | { type: 'release'; ownerTag: string | undefined }; async function readHookRecoveryMarker( markerPath: string @@ -401,6 +396,125 @@ async function writeRunUnderLifecycleLock( }); } +// Legacy reservations do not record their tag. Prefer an active matching run +// so it can never be mistaken for terminal data and released. +async function findRunAcrossTags( + basedir: string, + runId: string +): Promise<{ run: WorkflowRun; tag: string | undefined } | null> { + let terminal: { run: WorkflowRun; tag: string | undefined } | null = null; + for (const fileId of await listJSONFiles(path.join(basedir, 'runs'))) { + if (stripTag(fileId) !== runId) continue; + const tag = fileId === runId ? undefined : fileId.slice(runId.length + 1); + const run = await readJSON( + taggedPath(basedir, 'runs', runId, tag), + WorkflowRunSchema + ); + if (!run) continue; + if (!isTerminalWorkflowRunStatus(run.status)) return { run, tag }; + terminal ??= { run, tag }; + } + return terminal; +} + +/** + * A disposed Hook releases its token immediately. Otherwise the token stays + * reserved while its run is active or its configured expiration is future. + */ +async function getHookTokenRelease( + basedir: string, + constraint: HookTokenConstraint +): Promise { + let ownerTag: string | undefined; + let owningRun: WorkflowRun | null = null; + switch (constraint.type) { + case 'current': + ownerTag = constraint.tag ?? undefined; + break; + case 'legacy': + case 'legacy-pinned': { + const owner = await findRunAcrossTags(basedir, constraint.runId); + ownerTag = owner?.tag; + owningRun = owner?.run ?? null; + break; + } + default: { + const unknownConstraint: never = constraint; + throw new Error( + `Unknown Hook token constraint: ${JSON.stringify(unknownConstraint)}` + ); + } + } + + if (await isHookDisposalCommitted(basedir, constraint.hookId, ownerTag)) { + return { type: 'release', ownerTag }; + } + if (hasFutureTokenExpiration(constraint)) return { type: 'retain' }; + + owningRun ??= await readJSONWithFallback( + basedir, + 'runs', + constraint.runId, + WorkflowRunSchema, + ownerTag + ); + if (owningRun && !isTerminalWorkflowRunStatus(owningRun.status)) { + return { type: 'retain' }; + } + return { type: 'release', ownerTag }; +} + +/** + * Atomically reserves a token. Retries by the same Hook recover its canonical + * event; other Hooks conflict until the previous reservation can be released. + */ +async function reserveHookToken( + basedir: string, + next: CurrentHookTokenConstraint +): Promise { + const constraintPath = hookTokenConstraintPath(basedir, next.token); + return withFileLock(constraintPath, async () => { + const existing = + (await readHookTokenConstraint(constraintPath)) ?? + (await rebuildHookTokenConstraintFromEventLog(basedir, next.token)); + + if (!existing) { + await writeJSON(constraintPath, next, { overwrite: true }); + return { type: 'claimed' }; + } + if (existing.runId === next.runId && existing.hookId === next.hookId) { + return { type: 'owned', constraint: existing }; + } + + const release = await getHookTokenRelease(basedir, existing); + if (release.type === 'retain') { + return { type: 'conflict', runId: existing.runId }; + } + const { ownerTag } = release; + + await Promise.all([ + deleteJSON(taggedPath(basedir, 'hooks', existing.hookId, ownerTag)), + deleteJSON( + hookRecoveryMarkerPath( + basedir, + existing.token, + existing.runId, + existing.hookId + ) + ), + deleteHookByRunMarker(basedir, existing.runId, existing.hookId, ownerTag), + ]); + await writeJSON(constraintPath, next, { overwrite: true }); + return { type: 'claimed' }; + }); +} + +function shouldLockRunEvent(eventType: Event['eventType']): boolean { + return ( + isTerminalRunEventType(eventType) || isHookLifecycleEventType(eventType) + ); +} + /** * Creates the events storage implementation using the filesystem. * Implements the Storage['events'] interface with create, list, and listByCorrelationId operations. @@ -522,24 +636,11 @@ export function createEventsStorage( rememberStoredEvent(event, eventPath, serializedEvent); } - // Per-instance in-process mutexes. Two storage instances sharing - // one data directory get independent lock maps, which makes them - // behave like two separate OS processes from the locking - // standpoint — cross-instance arbitration relies on the on-disk - // `writeExclusive` constraint / claim files instead. Tests use - // this to exercise cross-process convergence without spawning - // subprocesses. - // - // `stepLocks` serializes step lifecycle events for the same - // (runId, correlationId): see comment further down in the - // `isStepEvent` branch. - // - // `hookLocks` serializes `hook_created` calls for the same - // (runId, correlationId) so the "claim token, then write hook - // entity + event" sequence runs to completion before another - // in-process invocation enters the dedup branch. + // Step lifecycle events use an in-process lock. Hook lifecycle and terminal + // events share a filesystem run lock, so separate storage instances cannot + // create a Hook after cleanup or resume one after its run becomes terminal. + // It also keeps same-run Hook retries and resume-vs-dispose ordering atomic. const stepLocks = new Map>(); - const hookLocks = new Map>(); return { clearCache, @@ -573,34 +674,10 @@ export function createEventsStorage( : `${runId}-${data.correlationId}`; return withInProcessLock(stepLocks, lockKey, () => createImpl()); } - // `hook_created` is serialized per-(runId, hookId) so the - // "claim token, write hook entity, write event" sequence runs to - // completion before another in-process invocation enters the - // same-hook dedup branch. Without this, two same-tick concurrent - // callers can race between the winner's `writeExclusive(claim)` - // and `writeJSON(hook)`, making the second caller momentarily - // observe a claim with no matching hook entity — which the - // crash-recovery path below would misinterpret as a prior crash - // and incorrectly fall through to a second hook entity write. - // - // `hook_received` and `hook_disposed` share the same per-hook lock - // so a resume's "hook exists and is not disposed, then append" - // sequence is atomic with respect to the disposer's "write dispose - // lock, delete entity, then append" sequence. Without this, a - // resume that passed its existence check before the disposal began - // could append its `hook_received` AFTER `hook_disposed` — an - // ordering that is journaled durably and makes every subsequent - // replay of the owning run diverge at that event - // (https://github.com/vercel/workflow/issues/2781). - if ( - isHookLifecycleEventType(data.eventType) && - runId && - data.correlationId - ) { - const lockKey = tag - ? `${runId}-${data.correlationId}.hook.${tag}` - : `${runId}-${data.correlationId}.hook`; - return withInProcessLock(hookLocks, lockKey, () => createImpl()); + // Token locks are acquired inside this lock, keeping the global order + // run -> token for Hook creation, disposal, and terminal cleanup. + if (runId && shouldLockRunEvent(data.eventType)) { + return withRunEventLock(basedir, runId, () => createImpl()); } return createImpl(); @@ -926,15 +1003,9 @@ export function createEventsStorage( ) { throw new HookNotFoundError(data.correlationId); } - // A resume must never be journaled after the hook's disposal. - // The disposer's durable order is: dispose lock → claim/entity - // delete → `hook_disposed` append, so the hook entity can still - // exist (or the disposer may have crashed mid-teardown) while - // disposal is already committed. Re-validate the dispose lock - // here — under the per-hook in-process lock taken above — so - // acceptance observes the same order replay will: once disposal - // has committed, the resume is rejected exactly like one that - // arrived after teardown finished. + // Disposal commits its lock before deleting the Hook or appending + // the event. A crash can therefore leave an entity that must not be + // resumable; the durable disposal lock is the source of truth. if ( data.eventType === 'hook_received' && (await isHookDisposalCommitted(basedir, data.correlationId, tag)) @@ -999,7 +1070,7 @@ export function createEventsStorage( // chosen by the dedup-recovery branch above (undefined for // first writers, `{ overwrite: true }` for retries that may // be repairing an orphaned partial write). - let hookEntityWriteOptions: { overwrite: boolean } | undefined; + let hookEntityWriteOptions: { overwrite: true } | undefined; // Create/update entity based on event type (event-sourced architecture) // Run lifecycle events @@ -1553,128 +1624,49 @@ export function createEventsStorage( ) { const hookData = data.eventData; - const constraintPath = hookTokenConstraintPath( - basedir, - hookData.token - ); - const nextConstraint: HookTokenConstraint = { - type: 'pinned', + const nextConstraint: CurrentHookTokenConstraint = { + type: 'current', token: hookData.token, hookId: data.correlationId, runId: effectiveRunId, + tag: tag ?? null, eventId, tokenExpiresAt: hookData.tokenExpiresAt, }; - const constraintContent = JSON.stringify(nextConstraint); - - const constraintResult = await withHookTokenConstraintLock( - constraintPath, - async (): Promise => { - let existing = await readHookTokenConstraint(constraintPath); - if (!existing) { - await deleteJSON(constraintPath); - existing = await rebuildHookTokenConstraintFromEventLog( - basedir, - hookData.token, - tag - ); - } - - if (!existing) { - assert( - await writeExclusive(constraintPath, constraintContent), - 'Hook token constraint must be available while locked' - ); - return { type: 'claimed' }; - } - - if ( - existing.runId === effectiveRunId && - existing.hookId === data.correlationId - ) { - return { type: 'existing', constraint: existing }; - } - - if ( - !(await isHookDisposalCommitted(basedir, existing.hookId, tag)) - ) { - if (hasFutureTokenExpiration(existing)) { - return { type: 'existing', constraint: existing }; - } - const owner = await readJSONWithFallback( - basedir, - 'runs', - existing.runId, - WorkflowRunSchema, - tag - ); - if (owner && !isTerminalWorkflowRunStatus(owner.status)) { - return { type: 'existing', constraint: existing }; - } - } - - await deleteJSON(constraintPath); - await Promise.all([ - deleteJSON(taggedPath(basedir, 'hooks', existing.hookId, tag)), - deleteJSON( - hookRecoveryMarkerPath( - basedir, - hookData.token, - existing.runId, - existing.hookId - ) - ), - deleteHookByRunMarker( - basedir, - existing.runId, - existing.hookId, - tag - ), - ]); - assert( - await writeExclusive(constraintPath, constraintContent), - 'Hook token constraint must be available after reuse' - ); - return { type: 'claimed' }; - } - ); + const reservation = await reserveHookToken(basedir, nextConstraint); - let writeHookEntityWithOverwrite = false; - switch (constraintResult.type) { + switch (reservation.type) { case 'claimed': break; - case 'existing': { - const { constraint } = constraintResult; - if ( - constraint.runId !== effectiveRunId || - constraint.hookId !== data.correlationId - ) { - const conflictEvent: Event = { - eventType: 'hook_conflict', - correlationId: data.correlationId, - eventData: { - token: hookData.token, - conflictingRunId: constraint.runId, - }, - runId: effectiveRunId, - eventId, - createdAt: now, - specVersion: effectiveSpecVersion, - }; - await storeEvent(conflictEvent); - return { - event: stripEventDataRefs( - conflictEvent, - params?.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION - ), - run, - step, - hook: undefined, - }; - } - + case 'conflict': { + const conflictEvent: Event = { + eventType: 'hook_conflict', + correlationId: data.correlationId, + eventData: { + token: hookData.token, + conflictingRunId: reservation.runId, + }, + runId: effectiveRunId, + eventId, + createdAt: now, + specVersion: effectiveSpecVersion, + }; + await storeEvent(conflictEvent); + return { + event: stripEventDataRefs( + conflictEvent, + params?.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION + ), + run, + step, + hook: undefined, + }; + } + case 'owned': { + const { constraint } = reservation; switch (constraint.type) { - case 'pinned': + case 'current': + case 'legacy-pinned': eventId = constraint.eventId; break; case 'legacy': { @@ -1702,10 +1694,11 @@ export function createEventsStorage( data.correlationId, eventId ); - assert( - pinned, - 'Legacy Hook constraint must have an event ID' - ); + if (!pinned) { + throw new EntityConflictError( + `Hook "${data.correlationId}" already created` + ); + } eventId = pinned; break; } @@ -1728,13 +1721,13 @@ export function createEventsStorage( createdAt: ulidToDate(eventId.replace(/^evnt_/, '')) ?? now, specVersion: effectiveSpecVersion, }; - writeHookEntityWithOverwrite = true; + hookEntityWriteOptions = { overwrite: true }; break; } default: { - const unknownResult: never = constraintResult; + const unknownReservation: never = reservation; throw new Error( - `Unknown Hook token constraint result: ${JSON.stringify(unknownResult)}` + `Unknown Hook token reservation: ${JSON.stringify(unknownReservation)}` ); } } @@ -1772,10 +1765,6 @@ export function createEventsStorage( isWebhook: hookData.isWebhook ?? false, isSystem: hookData.isSystem ?? false, }; - hookEntityWriteOptions = writeHookEntityWithOverwrite - ? { overwrite: true } - : undefined; - // Index entries before the event publish (see hook-index.ts // crash-ordering invariant). `eventId` is final here — the // dedup-recovery branch above already reassigned it to the @@ -1826,7 +1815,7 @@ export function createEventsStorage( basedir, existingHook.token ); - await withHookTokenConstraintLock(constraintPath, async () => { + await withFileLock(constraintPath, async () => { const constraint = await readHookTokenConstraint(constraintPath); if ( constraint?.runId === existingHook.runId && @@ -1959,21 +1948,8 @@ export function createEventsStorage( // collision indicates a real bug and EntityConflictError is // also the right surface — same shape as step_created's // claim-file behavior. - // Last-instant re-validation for `hook_received` (see the acceptance - // check above). The per-hook in-process lock already serializes - // resume vs. dispose within one storage instance; this second check - // narrows the cross-instance window (independent lock maps, shared - // filesystem) to the single event write below, matching the - // module's convention that the on-disk lock file — not the - // in-process mutex — is the durable source of truth. - if ( - data.eventType === 'hook_received' && - data.correlationId && - (await isHookDisposalCommitted(basedir, data.correlationId, tag)) - ) { - throw new HookNotFoundError(data.correlationId); - } - + // Hook events still hold the run lock here, so resume, disposal, and + // terminal cleanup cannot interleave between validation and publish. const compositeKey = `${effectiveRunId}-${eventId}`; const eventPath = taggedPath(basedir, 'events', compositeKey, tag); // Capture the serialized payload before the write's `await` so the diff --git a/packages/world-local/src/storage/helpers.ts b/packages/world-local/src/storage/helpers.ts index 27c089d9cd..0a3a41c0dd 100644 --- a/packages/world-local/src/storage/helpers.ts +++ b/packages/world-local/src/storage/helpers.ts @@ -53,24 +53,21 @@ export async function isHookDisposalCommitted( return false; } -/** - * Path of the file that reserves a hook token. - */ -export function hookTokenConstraintPath( - basedir: string, - token: string -): string { - return path.join(basedir, 'hooks', 'tokens', `${hashToken(token)}.json`); -} - -export async function withHookTokenConstraintLock( - constraintPath: string, +/** Cross-process lock for Hook token and run-event transactions. */ +export async function withFileLock( + filePath: string, fn: () => Promise ): Promise { - await fs.mkdir(path.dirname(constraintPath), { recursive: true }); - const release = await lock(constraintPath, { + await fs.mkdir(path.dirname(filePath), { recursive: true }); + const release = await lock(filePath, { realpath: false, - retries: { forever: true, minTimeout: 10, maxTimeout: 100 }, + stale: 30_000, + retries: { + forever: true, + maxRetryTime: 15_000, + minTimeout: 10, + maxTimeout: 100, + }, }); try { return await fn(); @@ -79,6 +76,17 @@ export async function withHookTokenConstraintLock( } } +export function withRunEventLock( + basedir: string, + runId: string, + fn: () => Promise +): Promise { + return withFileLock( + resolveWithinBase(basedir, '.locks', 'runs', `${runId}.events`), + fn + ); +} + /** * Compute the path of the recovery-marker sidecar for a specific * `(token, runId, hookId)` triple. Identity is encoded in the diff --git a/packages/world-local/src/storage/hook-index.ts b/packages/world-local/src/storage/hook-index.ts index 7658a95cb1..dab9f5cee6 100644 --- a/packages/world-local/src/storage/hook-index.ts +++ b/packages/world-local/src/storage/hook-index.ts @@ -1,6 +1,6 @@ import fs from 'node:fs/promises'; import path from 'node:path'; -import type { Event } from '@workflow/world'; +import type { Event, HookCreatedEvent } from '@workflow/world'; import { EventSchema, HookSchema } from '@workflow/world'; import { z } from 'zod'; import { @@ -295,30 +295,47 @@ async function ensureHookIndexesImpl(basedir: string): Promise { await writeExclusive(markerPath, ''); } +// Token reservations are global; Hook ID reads retain normal tag visibility. +export type HookIndexLookup = + | { kind: 'token'; token: string } + | { kind: 'id'; hookId: string; tag: string | undefined }; + /** - * Find the newest visible `hook_created` event for a token or hookId. + * Find the newest matching `hook_created` event for a token or hookId. * Entries are iterated newest-first (eventIds are ULIDs); dangling or * non-matching entries are skipped. Liveness is the caller's job. */ export async function findNewestIndexedHookCreatedEvent( basedir: string, - index: { kind: 'token'; token: string } | { kind: 'id'; hookId: string }, - matches: (event: Event) => boolean, - tag?: string -): Promise { + index: HookIndexLookup +): Promise<{ event: HookCreatedEvent; tag: string | undefined } | null> { await ensureHookIndexes(basedir); let dir: string; - try { - dir = - index.kind === 'token' - ? tokenIndexDir(basedir, index.token) - : idIndexDir(basedir, index.hookId); - } catch { - return null; + let isVisible: (fileId: string) => boolean; + let matches: (event: HookCreatedEvent) => boolean; + switch (index.kind) { + case 'token': + dir = tokenIndexDir(basedir, index.token); + isVisible = () => true; + matches = (event) => event.eventData.token === index.token; + break; + case 'id': + try { + dir = idIndexDir(basedir, index.hookId); + } catch { + return null; + } + isVisible = (fileId) => isVisibleToTag(fileId, index.tag); + matches = (event) => event.correlationId === index.hookId; + break; + default: { + const unknownIndex: never = index; + throw new Error(`Unknown Hook index: ${JSON.stringify(unknownIndex)}`); + } } const entryIds = (await listJSONFiles(dir)) - .filter((fileId) => isVisibleToTag(fileId, tag)) + .filter(isVisible) .sort((a, b) => stripTag(b).localeCompare(stripTag(a))); for (const entryId of entryIds) { @@ -335,24 +352,22 @@ export async function findNewestIndexedHookCreatedEvent( } if (!entry) continue; - const eventId = stripTag(entryId); + const tag = tagOf(entryId); let eventPath: string; try { eventPath = taggedPath( basedir, 'events', - `${entry.runId}-${eventId}`, - tagOf(entryId) + `${entry.runId}-${stripTag(entryId)}`, + tag ); } catch { continue; } const event = await readEventLenient(eventPath); - if (!event) continue; - if (event.eventType !== 'hook_created') continue; - if (typeof event.correlationId !== 'string') continue; - if (!matches(event)) continue; - return event; + if (event?.eventType === 'hook_created' && matches(event)) { + return { event, tag }; + } } return null; } diff --git a/packages/world-local/src/storage/hook-token-constraint.ts b/packages/world-local/src/storage/hook-token-constraint.ts index c25fdbbf0b..9834a7917a 100644 --- a/packages/world-local/src/storage/hook-token-constraint.ts +++ b/packages/world-local/src/storage/hook-token-constraint.ts @@ -1,5 +1,15 @@ +import path from 'node:path'; import { z } from 'zod'; import { readJSON } from '../fs.js'; +import { hashToken } from './helpers.js'; + +/** Path of the file that reserves a Hook token across processes. */ +export function hookTokenConstraintPath( + basedir: string, + token: string +): string { + return path.join(basedir, 'hooks', 'tokens', `${hashToken(token)}.json`); +} const HookTokenConstraintFields = { token: z.string(), @@ -8,28 +18,39 @@ const HookTokenConstraintFields = { tokenExpiresAt: z.coerce.date().optional(), }; +// Current files include both the canonical event and owner tag. Normalize +// older shapes at this boundary so callers handle migration states explicitly. const HookTokenConstraintSchema = z .union([ z.object({ ...HookTokenConstraintFields, - type: z.literal('pinned'), + type: z.enum(['current', 'pinned']), + tag: z.string().nullable().optional(), eventId: z.string(), }), z.object({ ...HookTokenConstraintFields, type: z.undefined().optional(), + tag: z.string().nullable().optional(), eventId: z.string().optional(), }), ]) .transform((constraint) => { - if (constraint.type === 'pinned') return constraint; - const { eventId, ...legacy } = constraint; - return eventId - ? { ...legacy, type: 'pinned' as const, eventId } - : { ...legacy, type: 'legacy' as const }; + const { eventId, tag, type: _storedType, ...fields } = constraint; + if (eventId && tag !== undefined) { + return { ...fields, type: 'current' as const, eventId, tag }; + } + if (eventId) { + return { ...fields, type: 'legacy-pinned' as const, eventId }; + } + return { ...fields, type: 'legacy' as const }; }); export type HookTokenConstraint = z.infer; +export type CurrentHookTokenConstraint = Extract< + HookTokenConstraint, + { type: 'current' } +>; export async function readHookTokenConstraint( path: string diff --git a/packages/world-local/src/storage/hooks-storage.ts b/packages/world-local/src/storage/hooks-storage.ts index aeb7f49ede..3ceacb08ff 100644 --- a/packages/world-local/src/storage/hooks-storage.ts +++ b/packages/world-local/src/storage/hooks-storage.ts @@ -1,7 +1,6 @@ import path from 'node:path'; import { HookNotFoundError } from '@workflow/errors'; import type { - Event, GetHookParams, Hook, HookCreatedEvent, @@ -14,6 +13,7 @@ import { isTerminalWorkflowRunStatus, WorkflowRunSchema, } from '@workflow/world'; +import { z } from 'zod'; import { DEFAULT_RESOLVE_DATA_OPTION } from '../config.js'; import { assertSafeEntityId, @@ -23,37 +23,35 @@ import { readJSON, readJSONWithFallback, taggedPath, + UnsafeEntityIdError, writeExclusive, + writeJSON, } from '../fs.js'; import { filterHookData } from './filters.js'; import { hookRecoveryMarkerPath, - hookTokenConstraintPath, isHookDisposalCommitted, - withHookTokenConstraintLock, + withFileLock, } from './helpers.js'; import { deleteHookByRunMarkerFile, ensureHookIndexes, findNewestIndexedHookCreatedEvent, + type HookIndexLookup, listHookByRunMarkers, writeHookByRunMarker, } from './hook-index.js'; import { + type CurrentHookTokenConstraint, type HookTokenConstraint, hasFutureTokenExpiration, + hookTokenConstraintPath, readHookTokenConstraint, } from './hook-token-constraint.js'; -function isMatchingHookCreatedEvent( - event: Event, - index: { kind: 'token'; token: string } | { kind: 'id'; hookId: string } -): event is HookCreatedEvent { - if (event.eventType !== 'hook_created') return false; - return index.kind === 'token' - ? event.eventData.token === index.token - : event.correlationId === index.hookId; -} +type IndexedHookCreatedEvent = NonNullable< + Awaited> +>; function hookFromCreatedEvent(event: HookCreatedEvent): Hook { const { token, metadata, isWebhook, isSystem } = event.eventData; @@ -88,62 +86,60 @@ async function isRunActive( } /** - * Find the live `hook_created` event for a token or hookId via the + * Find the available `hook_created` event for a token or hookId via the * durable hook indexes (instead of scanning the whole event log). * - * The liveness checks below subsume the old scan's in-log closure + * The availability checks below subsume the old scan's in-log closure * replay: the dispose lock is written before `hook_disposed` is * appended, the run entity is terminal before any terminal run event * is appended, and neither is ever deleted — so any closure visible * in the log is also visible to these checks. */ -async function findLiveHookCreatedEvent( +async function isHookCreatedEventAvailable( basedir: string, - index: { kind: 'token'; token: string } | { kind: 'id'; hookId: string }, - tag?: string -): Promise { - const event = await findNewestIndexedHookCreatedEvent( - basedir, - index, - (candidate) => isMatchingHookCreatedEvent(candidate, index), - tag - ); - if (!event || !isMatchingHookCreatedEvent(event, index)) return null; - - if (!(await isRunActive(basedir, event.runId, tag))) { - return null; - } - - // A committed disposal closes the hook before its event is appended. - if (await isHookDisposalCommitted(basedir, event.correlationId, tag)) { - return null; + { event, tag }: IndexedHookCreatedEvent +): Promise { + if ( + !(await isRunActive(basedir, event.runId, tag)) && + !hasFutureTokenExpiration(event.eventData) + ) { + return false; } + // Disposal is committed before its event, so the lock closes the Hook even + // when a crash leaves the entity and event-log cleanup incomplete. + return !(await isHookDisposalCommitted(basedir, event.correlationId, tag)); +} - return event; +async function findHookCreatedEvent( + basedir: string, + index: HookIndexLookup +): Promise { + const indexed = await findNewestIndexedHookCreatedEvent(basedir, index); + if (!indexed) return null; + if (!(await isHookCreatedEventAvailable(basedir, indexed))) return null; + return indexed; } function hookTokenConstraintFromEvent( - event: HookCreatedEvent -): HookTokenConstraint { + event: HookCreatedEvent, + tag: string | undefined +): CurrentHookTokenConstraint { return { - type: 'pinned', + type: 'current', token: event.eventData.token, hookId: event.correlationId, runId: event.runId, + tag: tag ?? null, eventId: event.eventId, tokenExpiresAt: event.eventData.tokenExpiresAt, }; } -async function restoreHookCachesFromEvent( +async function restoreHookEntityFromEvent( basedir: string, event: HookCreatedEvent, tag: string | undefined ): Promise { - await writeExclusive( - hookTokenConstraintPath(basedir, event.eventData.token), - JSON.stringify(hookTokenConstraintFromEvent(event)) - ); const hook = hookFromCreatedEvent(event); // Marker before entity (see hook-index.ts crash-ordering invariant). await writeHookByRunMarker(basedir, hook.runId, hook.hookId, tag); @@ -155,60 +151,84 @@ async function restoreHookCachesFromEvent( return hook; } +// Admission repairs only the reservation. Hook reads restore the entity and +// by-run marker, keeping conflict checks free of unrelated filesystem writes. export async function rebuildHookTokenConstraintFromEventLog( basedir: string, - token: string, - tag?: string + token: string ): Promise { - const event = await findNewestIndexedHookCreatedEvent( - basedir, - { kind: 'token', token }, - (candidate) => - candidate.eventType === 'hook_created' && - candidate.eventData.token === token, - tag - ); - if (!event || event.eventType !== 'hook_created') return null; - if (await isHookDisposalCommitted(basedir, event.correlationId, tag)) { - return null; - } - - if ( - !(await isRunActive(basedir, event.runId, tag)) && - !hasFutureTokenExpiration(event.eventData) - ) { - return null; - } - await restoreHookCachesFromEvent(basedir, event, tag); - return hookTokenConstraintFromEvent(event); + const indexed = await findHookCreatedEvent(basedir, { kind: 'token', token }); + if (!indexed) return null; + const constraint = hookTokenConstraintFromEvent(indexed.event, indexed.tag); + await writeJSON(hookTokenConstraintPath(basedir, token), constraint, { + overwrite: true, + }); + return constraint; } -export async function rebuildLiveHookByTokenFromEventLog( +async function rebuildHookByTokenFromEventLog( basedir: string, - token: string, - tag?: string + token: string ): Promise { - const event = await findLiveHookCreatedEvent( - basedir, - { kind: 'token', token }, - tag - ); - return event ? restoreHookCachesFromEvent(basedir, event, tag) : null; + const constraintPath = hookTokenConstraintPath(basedir, token); + return withFileLock(constraintPath, async () => { + const indexed = await findHookCreatedEvent(basedir, { + kind: 'token', + token, + }); + if (!indexed) return null; + await writeJSON( + constraintPath, + hookTokenConstraintFromEvent(indexed.event, indexed.tag), + { overwrite: true } + ); + return restoreHookEntityFromEvent(basedir, indexed.event, indexed.tag); + }); } -async function rebuildLiveHookByIdFromEventLog( +async function rebuildHookByIdFromEventLog( basedir: string, hookId: string, tag?: string ): Promise { - const event = await findLiveHookCreatedEvent( + const indexed = await findHookCreatedEvent(basedir, { + kind: 'id', + hookId, + tag, + }); + if (!indexed) return null; + + const constraintPath = hookTokenConstraintPath( basedir, - { kind: 'id', hookId }, - tag + indexed.event.eventData.token ); - return event ? restoreHookCachesFromEvent(basedir, event, tag) : null; + return withFileLock(constraintPath, async () => { + if (!(await isHookCreatedEventAvailable(basedir, indexed))) return null; + // An ID-based repair must not replace a newer owner of the same token. + const constraint = await readHookTokenConstraint(constraintPath); + if ( + constraint && + (constraint.runId !== indexed.event.runId || + constraint.hookId !== indexed.event.correlationId) + ) { + return null; + } + if (!constraint) { + await writeJSON( + constraintPath, + hookTokenConstraintFromEvent(indexed.event, indexed.tag), + { overwrite: true } + ); + } + return restoreHookEntityFromEvent(basedir, indexed.event, indexed.tag); + }); } +type HookByTokenLookup = + | { type: 'found'; hook: Hook } + | { type: 'recover' } + | { type: 'unavailable' }; + /** * Creates a hooks storage implementation using the filesystem. * Implements the Storage['hooks'] interface with hook CRUD operations. @@ -217,33 +237,39 @@ export function createHooksStorage( basedir: string, tag?: string ): Storage['hooks'] { - async function findHookByToken(token: string): Promise { + async function findHookByToken(token: string): Promise { const constraint = await readHookTokenConstraint( hookTokenConstraintPath(basedir, token) ); - if (!constraint) return null; + if (!constraint || constraint.type !== 'current') { + return { type: 'recover' }; + } + const ownerTag = constraint.tag ?? undefined; const hook = await readJSONWithFallback( basedir, 'hooks', constraint.hookId, HookSchema, - tag + ownerTag ); - if (!hook || hook.token !== token) return null; + if (!hook || hook.token !== token) return { type: 'recover' }; if ( - !(await isRunActive(basedir, hook.runId, tag)) && + !(await isRunActive(basedir, hook.runId, ownerTag)) && !hasFutureTokenExpiration(constraint) ) { - return null; + return { type: 'unavailable' }; } - return { ...hook, isWebhook: hook.isWebhook ?? true }; + return { + type: 'found', + hook: { ...hook, isWebhook: hook.isWebhook ?? true }, + }; } async function get(hookId: string, params?: GetHookParams): Promise { assertSafeEntityId('hookId', hookId); const hook = (await readJSONWithFallback(basedir, 'hooks', hookId, HookSchema, tag)) ?? - (await rebuildLiveHookByIdFromEventLog(basedir, hookId, tag)); + (await rebuildHookByIdFromEventLog(basedir, hookId, tag)); if (!hook) { throw new HookNotFoundError(hookId); } @@ -255,11 +281,24 @@ export function createHooksStorage( } async function getByToken(token: string): Promise { - const hook = - (await findHookByToken(token)) ?? - (await rebuildLiveHookByTokenFromEventLog(basedir, token, tag)); - if (!hook) throw new HookNotFoundError(token); - return hook; + const lookup = await findHookByToken(token); + switch (lookup.type) { + case 'found': + return lookup.hook; + case 'recover': { + const hook = await rebuildHookByTokenFromEventLog(basedir, token); + if (hook) return hook; + throw new HookNotFoundError(token); + } + case 'unavailable': + throw new HookNotFoundError(token); + default: { + const unknownLookup: never = lookup; + throw new Error( + `Unknown Hook token lookup: ${JSON.stringify(unknownLookup)}` + ); + } + } } async function list( @@ -318,32 +357,43 @@ export async function deleteAllHooksForRun( await deleteHookByRunMarkerFile(basedir, marker.fileId); continue; } - const hookPath = taggedPath(basedir, 'hooks', marker.hookId, marker.tag); - const hook = await readJSON(hookPath, HookSchema); - if (hook?.runId === runId) { - const constraintPath = hookTokenConstraintPath(basedir, hook.token); - const keepHook = await withHookTokenConstraintLock( - constraintPath, - async () => { - const constraint = await readHookTokenConstraint(constraintPath); - const owned = - constraint?.runId === hook.runId && - constraint.hookId === hook.hookId; - if (owned && hasFutureTokenExpiration(constraint)) return true; - if (owned) await deleteJSON(constraintPath); - return false; - } - ); - - if (!keepHook) { - await deleteJSON( - hookRecoveryMarkerPath(basedir, hook.token, hook.runId, hook.hookId) - ); - await deleteJSON(hookPath); - await deleteHookByRunMarkerFile(basedir, marker.fileId); + let hookPath: string; + let hook: Hook | null; + try { + hookPath = taggedPath(basedir, 'hooks', marker.hookId, marker.tag); + hook = await readJSON(hookPath, HookSchema); + } catch (error) { + if ( + !UnsafeEntityIdError.is(error) && + !(error instanceof SyntaxError || error instanceof z.ZodError) + ) { + throw error; } - } else { await deleteHookByRunMarkerFile(basedir, marker.fileId); + continue; + } + if (!hook || hook.runId !== runId) { + await deleteHookByRunMarkerFile(basedir, marker.fileId); + continue; } + + const constraintPath = hookTokenConstraintPath(basedir, hook.token); + const keepHook = await withFileLock(constraintPath, async () => { + const constraint = await readHookTokenConstraint(constraintPath); + const owned = + constraint?.runId === hook.runId && constraint.hookId === hook.hookId; + if (owned && hasFutureTokenExpiration(constraint)) return true; + if (owned) await deleteJSON(constraintPath); + return false; + }); + if (keepHook) continue; + + await Promise.all([ + deleteJSON( + hookRecoveryMarkerPath(basedir, hook.token, hook.runId, hook.hookId) + ), + deleteJSON(hookPath), + deleteHookByRunMarkerFile(basedir, marker.fileId), + ]); } } diff --git a/packages/world-local/src/test-helpers.ts b/packages/world-local/src/test-helpers.ts index bf52a549ad..d570b5175d 100644 --- a/packages/world-local/src/test-helpers.ts +++ b/packages/world-local/src/test-helpers.ts @@ -120,15 +120,12 @@ export async function createHook( metadata?: SerializedData; } ): Promise { + const { hookId, ...eventData } = data; const result = await storage.events.create(runId, { eventType: 'hook_created', specVersion: SPEC_VERSION_CURRENT, - correlationId: data.hookId, - eventData: { - token: data.token, - tokenExpiresAt: data.tokenExpiresAt, - metadata: data.metadata, - }, + correlationId: hookId, + eventData, }); if (!result.hook) { throw new Error('Expected hook to be created'); From 22c7f4007cf8976068f9e7157671a9dcfb201345 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Thu, 16 Jul 2026 13:30:25 -0700 Subject: [PATCH 32/36] refactor(world-local): align Hook minimum retention --- .changeset/local-hook-min-retention.md | 5 ++ .changeset/local-hook-token-expiration.md | 5 -- packages/core/e2e/e2e.test.ts | 8 +-- packages/world-local/src/index.ts | 1 + packages/world-local/src/storage.test.ts | 56 +++++++++---------- .../world-local/src/storage/events-storage.ts | 10 ++-- .../src/storage/hook-token-constraint.ts | 52 ++++++++--------- .../world-local/src/storage/hooks-storage.ts | 10 ++-- packages/world-local/src/test-helpers.ts | 2 +- workbench/example/workflows/99_e2e.ts | 8 +-- 10 files changed, 75 insertions(+), 82 deletions(-) create mode 100644 .changeset/local-hook-min-retention.md delete mode 100644 .changeset/local-hook-token-expiration.md diff --git a/.changeset/local-hook-min-retention.md b/.changeset/local-hook-min-retention.md new file mode 100644 index 0000000000..096592c076 --- /dev/null +++ b/.changeset/local-hook-min-retention.md @@ -0,0 +1,5 @@ +--- +'@workflow/world-local': minor +--- + +Keep Hook tokens reserved through their configured minimum retention. diff --git a/.changeset/local-hook-token-expiration.md b/.changeset/local-hook-token-expiration.md deleted file mode 100644 index 8e75c2d865..0000000000 --- a/.changeset/local-hook-token-expiration.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@workflow/world-local': minor ---- - -Keep Hook tokens reserved after terminal runs until they expire. diff --git a/packages/core/e2e/e2e.test.ts b/packages/core/e2e/e2e.test.ts index 32afcf5b65..2b2c9ddbb0 100644 --- a/packages/core/e2e/e2e.test.ts +++ b/packages/core/e2e/e2e.test.ts @@ -2051,11 +2051,11 @@ describe('e2e', () => { !isLocalDeployment() || process.env.WORKFLOW_TARGET_WORLD === '@workflow/world-postgres' )( - 'hookTokenExpirationWorkflow - terminal Hook cannot resume and its token stays unavailable', + 'hookMinRetentionWorkflow - terminal Hook cannot resume and its token stays unavailable', { timeout: 60_000 }, async () => { - const token = `expires-${Math.random().toString(36).slice(2)}`; - const owner = await start(await e2e('hookTokenExpirationWorkflow'), [ + const token = `retained-${Math.random().toString(36).slice(2)}`; + const owner = await start(await e2e('hookMinRetentionWorkflow'), [ token, 60_000, ]); @@ -2068,7 +2068,7 @@ describe('e2e', () => { (error: unknown) => HookNotFoundError.is(error) ); - const duplicate = await start(await e2e('hookTokenExpirationWorkflow'), [ + const duplicate = await start(await e2e('hookMinRetentionWorkflow'), [ token, 60_000, ]); diff --git a/packages/world-local/src/index.ts b/packages/world-local/src/index.ts index b7807f112c..7f31f3cd2d 100644 --- a/packages/world-local/src/index.ts +++ b/packages/world-local/src/index.ts @@ -70,6 +70,7 @@ export function createWorld(args?: Partial): LocalWorld { const recoverActiveRuns = resolveRecoverActiveRuns(mergedConfig); return { specVersion: SPEC_VERSION_CURRENT, + capabilities: { hookRetention: { active: true } }, ...queue, ...storage, ...instrumentObject('world.streams', { diff --git a/packages/world-local/src/storage.test.ts b/packages/world-local/src/storage.test.ts index 26f97daf9f..fb99b28acf 100644 --- a/packages/world-local/src/storage.test.ts +++ b/packages/world-local/src/storage.test.ts @@ -2234,7 +2234,7 @@ describe('Storage', () => { const hook1 = await createHook(storage, testRunId, { hookId: 'hook_1', token, - tokenExpiresAt: new Date(Date.now() + 60_000), + tokenRetentionUntil: new Date(Date.now() + 60_000), }); expect(hook1.token).toBe(token); @@ -2270,7 +2270,7 @@ describe('Storage', () => { await createHook(storage, testRunId, { hookId: 'hook_retained_owner', token, - tokenExpiresAt: new Date(Date.now() + 60_000), + tokenRetentionUntil: new Date(Date.now() + 60_000), }); await updateRun(storage, testRunId, 'run_completed', { @@ -2303,24 +2303,24 @@ describe('Storage', () => { }); }); - it('uses expiration only after the owning run is terminal', async () => { - const token = 'expiration-boundary-token'; - const tokenExpiresAt = new Date(Date.now() + 60_000); + it('uses retention only after the owning run is terminal', async () => { + const token = 'retention-boundary-token'; + const tokenRetentionUntil = new Date(Date.now() + 60_000); const replacement = await createRun(storage, { - deploymentId: 'deployment-expiration-boundary', - workflowName: 'expiration-boundary', + deploymentId: 'deployment-retention-boundary', + workflowName: 'retention-boundary', input: new Uint8Array(), }); await createHook(storage, testRunId, { - hookId: 'hook_expiration_boundary_owner', + hookId: 'hook_retention_boundary_owner', token, - tokenExpiresAt, + tokenRetentionUntil, }); - vi.spyOn(Date, 'now').mockReturnValue(tokenExpiresAt.getTime()); + vi.spyOn(Date, 'now').mockReturnValue(tokenRetentionUntil.getTime()); const conflict = await storage.events.create(replacement.runId, { eventType: 'hook_created', - correlationId: 'hook_expiration_boundary_conflict', + correlationId: 'hook_retention_boundary_conflict', eventData: { token }, }); expect(conflict.event.eventType).toBe('hook_conflict'); @@ -2332,31 +2332,31 @@ describe('Storage', () => { name: 'HookNotFoundError', }); const hook = await createHook(storage, replacement.runId, { - hookId: 'hook_expiration_boundary_replacement', + hookId: 'hook_retention_boundary_replacement', token, }); expect(hook.runId).toBe(replacement.runId); }); - it('admits only one hook when a token expires concurrently', async () => { - const token = 'concurrent-expiry-token'; - const tokenExpiresAt = new Date(); + it('admits one Hook across workers after retention ends', async () => { + const token = 'concurrent-retention-token'; + const tokenRetentionUntil = new Date(); const workerA = createStorage(testDir); const workerB = createStorage(testDir); const contenderA = await createRun(workerA, { - deploymentId: 'deployment-concurrent-expiry-a', - workflowName: 'concurrent-expiry-a', + deploymentId: 'deployment-concurrent-retention-a', + workflowName: 'concurrent-retention-a', input: new Uint8Array(), }); const contenderB = await createRun(workerB, { - deploymentId: 'deployment-concurrent-expiry-b', - workflowName: 'concurrent-expiry-b', + deploymentId: 'deployment-concurrent-retention-b', + workflowName: 'concurrent-retention-b', input: new Uint8Array(), }); await createHook(workerA, testRunId, { - hookId: 'hook_concurrent_expiry_owner', + hookId: 'hook_concurrent_retention_owner', token, - tokenExpiresAt, + tokenRetentionUntil, }); await updateRun(workerA, testRunId, 'run_completed', { output: new Uint8Array(), @@ -2365,12 +2365,12 @@ describe('Storage', () => { const results = await Promise.all([ workerA.events.create(contenderA.runId, { eventType: 'hook_created', - correlationId: 'hook_concurrent_expiry_a', + correlationId: 'hook_concurrent_retention_a', eventData: { token }, }), workerB.events.create(contenderB.runId, { eventType: 'hook_created', - correlationId: 'hook_concurrent_expiry_b', + correlationId: 'hook_concurrent_retention_b', eventData: { token }, }), ]); @@ -3452,24 +3452,24 @@ describe('Storage', () => { // Hook and event. The same owner must finish creation, not conflict. const token = 'orphaned-claim-token'; const hookId = 'hook_orphan_1'; - const tokenExpiresAt = new Date('2026-08-01T00:00:00.000Z'); + const tokenRetentionUntil = new Date(Date.now() + 60_000); await writeJSON(hookTokenConstraintPath(testDir, token), { token, hookId, runId: testRunId, - tokenExpiresAt, + tokenRetentionUntil, }); await expect(storage.hooks.get(hookId)).rejects.toThrow( /not found|HookNotFoundError/i ); - // The retry cannot extend the expiration stored by the first attempt. + // A retry cannot extend the retention stored by the first attempt. const hook = await createHook(storage, testRunId, { hookId, token, - tokenExpiresAt: new Date('2026-09-01T00:00:00.000Z'), + tokenRetentionUntil: new Date(Date.now() + 120_000), }); expect(hook.hookId).toBe(hookId); expect(hook.token).toBe(token); @@ -3489,7 +3489,7 @@ describe('Storage', () => { ); expect(created).toEqual([ expect.objectContaining({ - eventData: expect.objectContaining({ tokenExpiresAt }), + eventData: expect.objectContaining({ tokenRetentionUntil }), }), ]); expect(conflicts).toHaveLength(0); diff --git a/packages/world-local/src/storage/events-storage.ts b/packages/world-local/src/storage/events-storage.ts index f776e2ed49..99efbed14e 100644 --- a/packages/world-local/src/storage/events-storage.ts +++ b/packages/world-local/src/storage/events-storage.ts @@ -77,7 +77,7 @@ import { import { type CurrentHookTokenConstraint, type HookTokenConstraint, - hasFutureTokenExpiration, + hasRemainingTokenRetention, hookTokenConstraintPath, readHookTokenConstraint, } from './hook-token-constraint.js'; @@ -419,7 +419,7 @@ async function findRunAcrossTags( /** * A disposed Hook releases its token immediately. Otherwise the token stays - * reserved while its run is active or its configured expiration is future. + * reserved while its run is active or its minimum retention has not elapsed. */ async function getHookTokenRelease( basedir: string, @@ -449,7 +449,7 @@ async function getHookTokenRelease( if (await isHookDisposalCommitted(basedir, constraint.hookId, ownerTag)) { return { type: 'release', ownerTag }; } - if (hasFutureTokenExpiration(constraint)) return { type: 'retain' }; + if (hasRemainingTokenRetention(constraint)) return { type: 'retain' }; owningRun ??= await readJSONWithFallback( basedir, @@ -1631,7 +1631,7 @@ export function createEventsStorage( runId: effectiveRunId, tag: tag ?? null, eventId, - tokenExpiresAt: hookData.tokenExpiresAt, + tokenRetentionUntil: hookData.tokenRetentionUntil, }; const reservation = await reserveHookToken(basedir, nextConstraint); @@ -1714,7 +1714,7 @@ export function createEventsStorage( ...data, eventData: { ...data.eventData, - tokenExpiresAt: constraint.tokenExpiresAt, + tokenRetentionUntil: constraint.tokenRetentionUntil, }, runId: effectiveRunId, eventId, diff --git a/packages/world-local/src/storage/hook-token-constraint.ts b/packages/world-local/src/storage/hook-token-constraint.ts index 9834a7917a..3363870d1d 100644 --- a/packages/world-local/src/storage/hook-token-constraint.ts +++ b/packages/world-local/src/storage/hook-token-constraint.ts @@ -15,36 +15,28 @@ const HookTokenConstraintFields = { token: z.string(), runId: z.string(), hookId: z.string(), - tokenExpiresAt: z.coerce.date().optional(), + tokenRetentionUntil: z.coerce.date().optional(), }; -// Current files include both the canonical event and owner tag. Normalize -// older shapes at this boundary so callers handle migration states explicitly. -const HookTokenConstraintSchema = z - .union([ - z.object({ +// Normalize pre-retention constraints so migration states remain explicit. +const HookTokenConstraintSchema = z.union([ + z.object({ + ...HookTokenConstraintFields, + type: z.literal('current'), + tag: z.string().nullable(), + eventId: z.string(), + }), + z + .object({ ...HookTokenConstraintFields, - type: z.enum(['current', 'pinned']), - tag: z.string().nullable().optional(), - eventId: z.string(), - }), - z.object({ - ...HookTokenConstraintFields, - type: z.undefined().optional(), - tag: z.string().nullable().optional(), eventId: z.string().optional(), - }), - ]) - .transform((constraint) => { - const { eventId, tag, type: _storedType, ...fields } = constraint; - if (eventId && tag !== undefined) { - return { ...fields, type: 'current' as const, eventId, tag }; - } - if (eventId) { - return { ...fields, type: 'legacy-pinned' as const, eventId }; - } - return { ...fields, type: 'legacy' as const }; - }); + }) + .transform(({ eventId, ...constraint }) => + eventId + ? { ...constraint, type: 'legacy-pinned' as const, eventId } + : { ...constraint, type: 'legacy' as const } + ), +]); export type HookTokenConstraint = z.infer; export type CurrentHookTokenConstraint = Extract< @@ -64,11 +56,11 @@ export async function readHookTokenConstraint( } } -export function hasFutureTokenExpiration( - constraint: Pick +export function hasRemainingTokenRetention( + constraint: Pick ): boolean { return ( - constraint.tokenExpiresAt !== undefined && - Date.now() < constraint.tokenExpiresAt.getTime() + constraint.tokenRetentionUntil !== undefined && + Date.now() < constraint.tokenRetentionUntil.getTime() ); } diff --git a/packages/world-local/src/storage/hooks-storage.ts b/packages/world-local/src/storage/hooks-storage.ts index 3ceacb08ff..3197533f66 100644 --- a/packages/world-local/src/storage/hooks-storage.ts +++ b/packages/world-local/src/storage/hooks-storage.ts @@ -44,7 +44,7 @@ import { import { type CurrentHookTokenConstraint, type HookTokenConstraint, - hasFutureTokenExpiration, + hasRemainingTokenRetention, hookTokenConstraintPath, readHookTokenConstraint, } from './hook-token-constraint.js'; @@ -101,7 +101,7 @@ async function isHookCreatedEventAvailable( ): Promise { if ( !(await isRunActive(basedir, event.runId, tag)) && - !hasFutureTokenExpiration(event.eventData) + !hasRemainingTokenRetention(event.eventData) ) { return false; } @@ -131,7 +131,7 @@ function hookTokenConstraintFromEvent( runId: event.runId, tag: tag ?? null, eventId: event.eventId, - tokenExpiresAt: event.eventData.tokenExpiresAt, + tokenRetentionUntil: event.eventData.tokenRetentionUntil, }; } @@ -255,7 +255,7 @@ export function createHooksStorage( if (!hook || hook.token !== token) return { type: 'recover' }; if ( !(await isRunActive(basedir, hook.runId, ownerTag)) && - !hasFutureTokenExpiration(constraint) + !hasRemainingTokenRetention(constraint) ) { return { type: 'unavailable' }; } @@ -382,7 +382,7 @@ export async function deleteAllHooksForRun( const constraint = await readHookTokenConstraint(constraintPath); const owned = constraint?.runId === hook.runId && constraint.hookId === hook.hookId; - if (owned && hasFutureTokenExpiration(constraint)) return true; + if (owned && hasRemainingTokenRetention(constraint)) return true; if (owned) await deleteJSON(constraintPath); return false; }); diff --git a/packages/world-local/src/test-helpers.ts b/packages/world-local/src/test-helpers.ts index d570b5175d..fa4ada687f 100644 --- a/packages/world-local/src/test-helpers.ts +++ b/packages/world-local/src/test-helpers.ts @@ -116,7 +116,7 @@ export async function createHook( data: { hookId: string; token: string; - tokenExpiresAt?: Date; + tokenRetentionUntil?: Date; metadata?: SerializedData; } ): Promise { diff --git a/workbench/example/workflows/99_e2e.ts b/workbench/example/workflows/99_e2e.ts index 3d93cfa5fd..69e7835a34 100644 --- a/workbench/example/workflows/99_e2e.ts +++ b/workbench/example/workflows/99_e2e.ts @@ -748,17 +748,17 @@ export async function hookGetConflictThenStepParallelWorkflow( /** * Keeps its token reserved after this run ends. The Hook is intentionally left - * undisposed so duplicates continue to receive a conflict until it expires. + * undisposed so duplicates continue to receive a conflict during retention. */ -export async function hookTokenExpirationWorkflow( +export async function hookMinRetentionWorkflow( token: string, - expiresInMs: number + minRetentionMs: number ) { 'use workflow'; const hook = createHook({ token, - experimental_expires: expiresInMs, + experimental_minRetention: minRetentionMs, }); const conflict = await hook.getConflict(); if (conflict) { From d9fa89c0d6e143a8297a28d943fc4b65e222256d Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Thu, 16 Jul 2026 14:06:43 -0700 Subject: [PATCH 33/36] fix(world-local): preserve Hook creation order --- packages/world-local/src/storage/events-storage.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/packages/world-local/src/storage/events-storage.ts b/packages/world-local/src/storage/events-storage.ts index 99efbed14e..f6d8e8a392 100644 --- a/packages/world-local/src/storage/events-storage.ts +++ b/packages/world-local/src/storage/events-storage.ts @@ -636,11 +636,10 @@ export function createEventsStorage( rememberStoredEvent(event, eventPath, serializedEvent); } - // Step lifecycle events use an in-process lock. Hook lifecycle and terminal - // events share a filesystem run lock, so separate storage instances cannot - // create a Hook after cleanup or resume one after its run becomes terminal. - // It also keeps same-run Hook retries and resume-vs-dispose ordering atomic. + // In-process queues preserve call order. Hook lifecycle and terminal events + // also share a filesystem run lock for cross-process atomicity. const stepLocks = new Map>(); + const runEventLocks = new Map>(); return { clearCache, @@ -677,7 +676,10 @@ export function createEventsStorage( // Token locks are acquired inside this lock, keeping the global order // run -> token for Hook creation, disposal, and terminal cleanup. if (runId && shouldLockRunEvent(data.eventType)) { - return withRunEventLock(basedir, runId, () => createImpl()); + const lockKey = tag ? `${runId}.${tag}` : runId; + return withInProcessLock(runEventLocks, lockKey, () => + withRunEventLock(basedir, runId, () => createImpl()) + ); } return createImpl(); From b257f4687f3ae5c607734b67263576f2bbb3f558 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Thu, 16 Jul 2026 15:36:09 -0700 Subject: [PATCH 34/36] fix(world-local): expose retained Hooks consistently --- packages/world-local/src/storage.test.ts | 34 ++++++++++++++++--- .../world-local/src/storage/hooks-storage.ts | 28 +++++++++++---- 2 files changed, 51 insertions(+), 11 deletions(-) diff --git a/packages/world-local/src/storage.test.ts b/packages/world-local/src/storage.test.ts index fb99b28acf..e317261bf7 100644 --- a/packages/world-local/src/storage.test.ts +++ b/packages/world-local/src/storage.test.ts @@ -2267,21 +2267,28 @@ describe('Storage', () => { it('rebuilds a retained Hook and keeps its token unavailable', async () => { const token = 'retained-token'; + const hookId = 'hook_retained_owner'; + const tokenRetentionUntil = new Date(Date.now() + 60_000); await createHook(storage, testRunId, { - hookId: 'hook_retained_owner', + hookId, token, - tokenRetentionUntil: new Date(Date.now() + 60_000), + tokenRetentionUntil, }); - await updateRun(storage, testRunId, 'run_completed', { - output: new Uint8Array(), - }); + await storage.events.create(testRunId, { eventType: 'run_cancelled' }); // Simulate cache loss: the retained event must rebuild the reservation. await fs.rm(hookTokenConstraintPath(testDir, token)); await expect(storage.hooks.getByToken(token)).resolves.toMatchObject({ runId: testRunId, token, }); + await expect(storage.hooks.get(hookId)).resolves.toMatchObject({ + runId: testRunId, + token, + }); + await expect(storage.hooks.list({})).resolves.toMatchObject({ + data: [{ runId: testRunId, token }], + }); const duplicate = await createRun(storage, { deploymentId: 'deployment-retained-duplicate', @@ -2301,6 +2308,23 @@ describe('Storage', () => { }, hook: undefined, }); + + vi.spyOn(Date, 'now').mockReturnValue(tokenRetentionUntil.getTime()); + await expect(storage.hooks.getByToken(token)).rejects.toMatchObject({ + name: 'HookNotFoundError', + }); + await expect(storage.hooks.get(hookId)).rejects.toMatchObject({ + name: 'HookNotFoundError', + }); + await expect(storage.hooks.list({})).resolves.toMatchObject({ + data: [], + }); + await expect( + createHook(storage, duplicate.runId, { + hookId: 'hook_retained_replacement', + token, + }) + ).resolves.toMatchObject({ runId: duplicate.runId, token }); }); it('uses retention only after the owning run is terminal', async () => { diff --git a/packages/world-local/src/storage/hooks-storage.ts b/packages/world-local/src/storage/hooks-storage.ts index 3197533f66..d7b5cf72a3 100644 --- a/packages/world-local/src/storage/hooks-storage.ts +++ b/packages/world-local/src/storage/hooks-storage.ts @@ -273,11 +273,19 @@ export function createHooksStorage( if (!hook) { throw new HookNotFoundError(hookId); } + if (await isHookDisposalCommitted(basedir, hookId, tag)) { + throw new HookNotFoundError(hookId); + } const resolveData = params?.resolveData || DEFAULT_RESOLVE_DATA_OPTION; - return filterHookData( - { ...hook, isWebhook: hook.isWebhook ?? true }, - resolveData - ); + if (await isRunActive(basedir, hook.runId, tag)) { + return filterHookData( + { ...hook, isWebhook: hook.isWebhook ?? true }, + resolveData + ); + } + const retained = await getByToken(hook.token); + if (retained.hookId !== hookId) throw new HookNotFoundError(hookId); + return filterHookData(retained, resolveData); } async function getByToken(token: string): Promise { @@ -330,10 +338,18 @@ export function createHooksStorage( getId: (hook) => hook.hookId, }); - // Transform the data after pagination + const data: Hook[] = []; + for (const hook of result.data) { + try { + data.push(await get(hook.hookId, { resolveData })); + } catch (error) { + if (!HookNotFoundError.is(error)) throw error; + } + } + return { ...result, - data: result.data.map((hook) => filterHookData(hook, resolveData)), + data, }; } From a30a3d79fcd20895380cc509305e59d55e1b3e7a Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Thu, 16 Jul 2026 15:59:16 -0700 Subject: [PATCH 35/36] refactor(world-local): simplify retained hook storage --- packages/world-local/src/fs.ts | 18 +++++++- packages/world-local/src/storage.test.ts | 19 ++++++-- .../world-local/src/storage/events-storage.ts | 22 +-------- .../world-local/src/storage/hook-index.ts | 21 +++------ .../src/storage/hook-token-constraint.ts | 10 +---- .../world-local/src/storage/hooks-storage.ts | 45 +++++++++---------- 6 files changed, 62 insertions(+), 73 deletions(-) diff --git a/packages/world-local/src/fs.ts b/packages/world-local/src/fs.ts index a1d6fc5bfc..9dac03527b 100644 --- a/packages/world-local/src/fs.ts +++ b/packages/world-local/src/fs.ts @@ -413,6 +413,20 @@ export async function readJSON( } } +/** Read JSON, treating malformed persistence data as absent. */ +export async function readJSONLenient( + filePath: string, + decoder: z.ZodType +): Promise { + try { + return await readJSON(filePath, decoder); + } catch (error) { + if (error instanceof SyntaxError || error instanceof z.ZodError) + return null; + throw error; + } +} + export async function readBuffer(filePath: string): Promise { const content = await fs.readFile(filePath); return content; @@ -507,7 +521,7 @@ interface PaginatedFileSystemQueryConfig { cachedItems?: ReadonlyMap; filePrefix?: string; fileIdFilter?: (fileId: string) => boolean; - filter?: (item: T) => boolean; + filter?: (item: T) => boolean | Promise; sortOrder?: 'asc' | 'desc'; limit?: number; cursor?: string; @@ -647,7 +661,7 @@ export async function paginatedFileSystemQuery( for (const item of loadedBatch) { if (!item) continue; // Apply custom filter early if provided - if (filter && !filter(item)) continue; + if (filter && !(await filter(item))) continue; // Double-check cursor filtering with actual createdAt from JSON // (in case ULID timestamp differs from stored createdAt) diff --git a/packages/world-local/src/storage.test.ts b/packages/world-local/src/storage.test.ts index e317261bf7..83a3577f3b 100644 --- a/packages/world-local/src/storage.test.ts +++ b/packages/world-local/src/storage.test.ts @@ -2267,7 +2267,7 @@ describe('Storage', () => { it('rebuilds a retained Hook and keeps its token unavailable', async () => { const token = 'retained-token'; - const hookId = 'hook_retained_owner'; + const hookId = 'hook_0_retained_owner'; const tokenRetentionUntil = new Date(Date.now() + 60_000); await createHook(storage, testRunId, { hookId, @@ -2309,6 +2309,11 @@ describe('Storage', () => { hook: undefined, }); + await createHook(storage, duplicate.runId, { + hookId: 'hook_1_live', + token: 'live-token', + }); + vi.spyOn(Date, 'now').mockReturnValue(tokenRetentionUntil.getTime()); await expect(storage.hooks.getByToken(token)).rejects.toMatchObject({ name: 'HookNotFoundError', @@ -2316,8 +2321,11 @@ describe('Storage', () => { await expect(storage.hooks.get(hookId)).rejects.toMatchObject({ name: 'HookNotFoundError', }); - await expect(storage.hooks.list({})).resolves.toMatchObject({ - data: [], + await expect( + storage.hooks.list({ pagination: { limit: 1 } }) + ).resolves.toMatchObject({ + data: [{ hookId: 'hook_1_live' }], + hasMore: false, }); await expect( createHook(storage, duplicate.runId, { @@ -2467,6 +2475,9 @@ describe('Storage', () => { const lockPath = hookDisposeLockPath(testDir, 'hook_1'); await fs.mkdir(path.dirname(lockPath), { recursive: true }); await fs.writeFile(lockPath, ''); + await expect(storage.hooks.getByToken(token)).rejects.toMatchObject({ + name: 'HookNotFoundError', + }); const run2 = await createRun(storage, { deploymentId: 'deployment-456', @@ -3471,7 +3482,7 @@ describe('Storage', () => { expect(result.hook).toBeUndefined(); }); - it('should recover an orphaned hook token claim with no matching hook entity', async () => { + it('recovers an orphaned Hook constraint without extending retention', async () => { // Simulate a crash after reserving the token but before persisting the // Hook and event. The same owner must finish creation, not conflict. const token = 'orphaned-claim-token'; diff --git a/packages/world-local/src/storage/events-storage.ts b/packages/world-local/src/storage/events-storage.ts index f6d8e8a392..31b1e6366f 100644 --- a/packages/world-local/src/storage/events-storage.ts +++ b/packages/world-local/src/storage/events-storage.ts @@ -51,6 +51,7 @@ import { listJSONFiles, paginatedFileSystemQuery, readJSON, + readJSONLenient, readJSONWithFallback, resolveWithinBase, stripTag, @@ -88,18 +89,6 @@ import { import { handleLegacyEvent } from './legacy.js'; import { withRunFileLock } from './runs-storage.js'; -/** - * Per-step in-process async mutex. Serializes concurrent `events.create` calls - * that target the same step, so that the "check terminal state, then write step - * entity + event" sequence is atomic. Without this, two concurrent step_started - * calls can both pass the not-terminal check and both write step_started events - * — or a step_started can land in the log after step_completed has already - * written, producing unconsumed events on replay. - * - * Duplicate step_started events for a non-terminal step are still allowed - * (retries legitimately re-start a step), only writes to an already-terminal - * step are rejected. - */ /** * Sidecar recovery marker that pins a canonical `hook_created` * eventId for a legacy token claim — one written by a version of @@ -132,14 +121,7 @@ type HookTokenRelease = async function readHookRecoveryMarker( markerPath: string ): Promise | null> { - try { - return await readJSON(markerPath, HookRecoveryMarkerSchema); - } catch (error) { - if (error instanceof SyntaxError || error instanceof z.ZodError) { - return null; - } - throw error; - } + return readJSONLenient(markerPath, HookRecoveryMarkerSchema); } /** diff --git a/packages/world-local/src/storage/hook-index.ts b/packages/world-local/src/storage/hook-index.ts index dab9f5cee6..3cf9aab01a 100644 --- a/packages/world-local/src/storage/hook-index.ts +++ b/packages/world-local/src/storage/hook-index.ts @@ -1,6 +1,6 @@ import fs from 'node:fs/promises'; import path from 'node:path'; -import type { Event, HookCreatedEvent } from '@workflow/world'; +import type { HookCreatedEvent } from '@workflow/world'; import { EventSchema, HookSchema } from '@workflow/world'; import { z } from 'zod'; import { @@ -10,6 +10,7 @@ import { isUntagged, listJSONFiles, readJSON, + readJSONLenient, resolveWithinBase, stripTag, taggedPath, @@ -76,17 +77,6 @@ export function isVisibleToTag( return tag ? isUntagged(fileId) || hasTag(fileId, tag) : isUntagged(fileId); } -async function readEventLenient(filePath: string): Promise { - try { - return await readJSON(filePath, EventSchema); - } catch (error) { - if (error instanceof SyntaxError || error instanceof z.ZodError) { - return null; - } - throw error; - } -} - /** * Record a `hook_created` event in the token- and id-indexes. Must be * called before the event is published. Idempotent (`writeExclusive`). @@ -247,8 +237,9 @@ async function ensureHookIndexesImpl(basedir: string): Promise { await listJSONFiles(eventsDir), 32, async (fileId) => { - const event = await readEventLenient( - path.join(eventsDir, `${fileId}.json`) + const event = await readJSONLenient( + path.join(eventsDir, `${fileId}.json`), + EventSchema ); if (!event || event.eventType !== 'hook_created') return; if (typeof event.correlationId !== 'string') return; @@ -364,7 +355,7 @@ export async function findNewestIndexedHookCreatedEvent( } catch { continue; } - const event = await readEventLenient(eventPath); + const event = await readJSONLenient(eventPath, EventSchema); if (event?.eventType === 'hook_created' && matches(event)) { return { event, tag }; } diff --git a/packages/world-local/src/storage/hook-token-constraint.ts b/packages/world-local/src/storage/hook-token-constraint.ts index 3363870d1d..b33f320db3 100644 --- a/packages/world-local/src/storage/hook-token-constraint.ts +++ b/packages/world-local/src/storage/hook-token-constraint.ts @@ -1,6 +1,6 @@ import path from 'node:path'; import { z } from 'zod'; -import { readJSON } from '../fs.js'; +import { readJSONLenient } from '../fs.js'; import { hashToken } from './helpers.js'; /** Path of the file that reserves a Hook token across processes. */ @@ -47,13 +47,7 @@ export type CurrentHookTokenConstraint = Extract< export async function readHookTokenConstraint( path: string ): Promise { - try { - return await readJSON(path, HookTokenConstraintSchema); - } catch (error) { - if (error instanceof SyntaxError || error instanceof z.ZodError) - return null; - throw error; - } + return readJSONLenient(path, HookTokenConstraintSchema); } export function hasRemainingTokenRetention( diff --git a/packages/world-local/src/storage/hooks-storage.ts b/packages/world-local/src/storage/hooks-storage.ts index d7b5cf72a3..cc91778a0f 100644 --- a/packages/world-local/src/storage/hooks-storage.ts +++ b/packages/world-local/src/storage/hooks-storage.ts @@ -99,15 +99,15 @@ async function isHookCreatedEventAvailable( basedir: string, { event, tag }: IndexedHookCreatedEvent ): Promise { - if ( - !(await isRunActive(basedir, event.runId, tag)) && - !hasRemainingTokenRetention(event.eventData) - ) { - return false; - } // Disposal is committed before its event, so the lock closes the Hook even // when a crash leaves the entity and event-log cleanup incomplete. - return !(await isHookDisposalCommitted(basedir, event.correlationId, tag)); + if (await isHookDisposalCommitted(basedir, event.correlationId, tag)) { + return false; + } + return ( + hasRemainingTokenRetention(event.eventData) || + (await isRunActive(basedir, event.runId, tag)) + ); } async function findHookCreatedEvent( @@ -253,9 +253,12 @@ export function createHooksStorage( ownerTag ); if (!hook || hook.token !== token) return { type: 'recover' }; + if (await isHookDisposalCommitted(basedir, hook.hookId, ownerTag)) { + return { type: 'unavailable' }; + } if ( - !(await isRunActive(basedir, hook.runId, ownerTag)) && - !hasRemainingTokenRetention(constraint) + !hasRemainingTokenRetention(constraint) && + !(await isRunActive(basedir, hook.runId, ownerTag)) ) { return { type: 'unavailable' }; } @@ -322,12 +325,15 @@ export function createHooksStorage( limit: params.pagination?.limit, cursor: params.pagination?.cursor, filePrefix: undefined, // Hooks don't have ULIDs, so we can't optimize by filename - filter: (hook) => { - // Filter by runId if provided - if (params.runId && hook.runId !== params.runId) { - return false; + filter: async (hook) => { + if (params.runId && hook.runId !== params.runId) return false; + try { + await get(hook.hookId); + return true; + } catch (error) { + if (HookNotFoundError.is(error)) return false; + throw error; } - return true; }, getCreatedAt: () => { // Hook files don't have ULID timestamps in filename, so return null @@ -338,18 +344,9 @@ export function createHooksStorage( getId: (hook) => hook.hookId, }); - const data: Hook[] = []; - for (const hook of result.data) { - try { - data.push(await get(hook.hookId, { resolveData })); - } catch (error) { - if (!HookNotFoundError.is(error)) throw error; - } - } - return { ...result, - data, + data: result.data.map((hook) => filterHookData(hook, resolveData)), }; } From f37fa91fa9ea5ee89f5cca350b0ab9ace9b4b3b6 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Thu, 16 Jul 2026 16:18:15 -0700 Subject: [PATCH 36/36] fix(world-local): allow stale lock recovery --- packages/world-local/src/storage/helpers.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/world-local/src/storage/helpers.ts b/packages/world-local/src/storage/helpers.ts index 0a3a41c0dd..c307a9c762 100644 --- a/packages/world-local/src/storage/helpers.ts +++ b/packages/world-local/src/storage/helpers.ts @@ -64,7 +64,6 @@ export async function withFileLock( stale: 30_000, retries: { forever: true, - maxRetryTime: 15_000, minTimeout: 10, maxTimeout: 100, },