diff --git a/.changeset/inline-claim-reclaim.md b/.changeset/inline-claim-reclaim.md new file mode 100644 index 0000000000..9d57541efc --- /dev/null +++ b/.changeset/inline-claim-reclaim.md @@ -0,0 +1,5 @@ +--- +'@workflow/core': patch +--- + +Keep a batch of inline steps together when one of its event writes loses a race, instead of discarding the batch diff --git a/.changeset/late-hook-delivery-divergence.md b/.changeset/late-hook-delivery-divergence.md new file mode 100644 index 0000000000..ab62353ca8 --- /dev/null +++ b/.changeset/late-hook-delivery-divergence.md @@ -0,0 +1,6 @@ +--- +'@workflow/core': patch +'workflow': patch +--- + +Stop failing runs with a corrupted-event-log error when a hook delivery arrives after the hook was disposed, or when a step result is still being fetched diff --git a/.changeset/local-slot-order.md b/.changeset/local-slot-order.md new file mode 100644 index 0000000000..148f7df032 --- /dev/null +++ b/.changeset/local-slot-order.md @@ -0,0 +1,5 @@ +--- +'@workflow/world-local': patch +--- + +Read a position-numbered event log in position order, so a replay sees the log the order it was written diff --git a/.changeset/slot-correlation-ids.md b/.changeset/slot-correlation-ids.md new file mode 100644 index 0000000000..e99c6c6578 --- /dev/null +++ b/.changeset/slot-correlation-ids.md @@ -0,0 +1,6 @@ +--- +'@workflow/core': patch +'@workflow/world': patch +--- + +Scope every queue idempotency key to the run, and number step and wait correlation IDs per kind so that inserting one kind no longer renumbers the others. diff --git a/.changeset/slot-event-identity-client.md b/.changeset/slot-event-identity-client.md new file mode 100644 index 0000000000..e6d2c5feee --- /dev/null +++ b/.changeset/slot-event-identity-client.md @@ -0,0 +1,9 @@ +--- +'@workflow/world-vercel': minor +'@workflow/core': minor +'@workflow/errors': minor +'@workflow/world': minor +'workflow': minor +--- + +Event creations on runs that number events by slot now claim their own event id and merge, replay and re-claim when a `SlotConflictError` shows another writer took it first. diff --git a/.changeset/slot-event-identity-worlds.md b/.changeset/slot-event-identity-worlds.md new file mode 100644 index 0000000000..863a2f860b --- /dev/null +++ b/.changeset/slot-event-identity-worlds.md @@ -0,0 +1,8 @@ +--- +'@workflow/world': minor +'@workflow/world-local': minor +'@workflow/world-postgres': minor +'@workflow/core': minor +--- + +Number a run's events by position in the Local and Postgres Worlds when `WORKFLOW_SLOT_IDENTITY` is set, so a reader can prove its copy of an event log is complete. diff --git a/.changeset/slot-identity-default-on.md b/.changeset/slot-identity-default-on.md new file mode 100644 index 0000000000..55e3973586 --- /dev/null +++ b/.changeset/slot-identity-default-on.md @@ -0,0 +1,8 @@ +--- +'@workflow/world': minor +'@workflow/world-vercel': minor +'@workflow/world-local': minor +'@workflow/world-postgres': minor +--- + +Number new runs' events by position by default, in every World including the Vercel one. Set `WORKFLOW_SLOT_IDENTITY=0` to keep minting ULID event ids. diff --git a/.changeset/tidy-moons-observe.md b/.changeset/tidy-moons-observe.md new file mode 100644 index 0000000000..9c6c5b7692 --- /dev/null +++ b/.changeset/tidy-moons-observe.md @@ -0,0 +1,5 @@ +--- +'@workflow/core': patch +--- + +Name the divergent event's pending invocations and the fenced member of an inline step batch in replay-divergence logs diff --git a/.changeset/turbo-run-started-occurred-at.md b/.changeset/turbo-run-started-occurred-at.md new file mode 100644 index 0000000000..bfbfbaf2d6 --- /dev/null +++ b/.changeset/turbo-run-started-occurred-at.md @@ -0,0 +1,5 @@ +--- +'@workflow/core': patch +--- + +Report the same workflow start time on an optimistically started run's first pass and its replays diff --git a/.gitignore b/.gitignore index 8b7641f7d7..28b5575028 100644 --- a/.gitignore +++ b/.gitignore @@ -42,3 +42,6 @@ packages/swc-plugin-workflow/build-hash.json workbench/nextjs-*/public/.well-known/workflow workbench/sveltekit/static/.well-known/workflow + +# Local e2e diagnostics dumps +e2e-diagnostics-*.json diff --git a/CHANGELOG-5.0.0-draft.md b/CHANGELOG-5.0.0-draft.md new file mode 100644 index 0000000000..1c7cc5d05c --- /dev/null +++ b/CHANGELOG-5.0.0-draft.md @@ -0,0 +1,267 @@ +# Workflow SDK 5.0.0 — Aggregate Changelog + +Everything released across `workflow@5.0.0-beta.0` → `beta.36` (2026-04-08 → 2026-07-22), plus changes merged to `main` but not yet released, **excluding anything also shipped in a `4.x` release** (4.2.0 → 4.6.2). + +Matching is by PR number across all release bodies — the same PR number appears in both lines when a change was cherry-picked, even though the commit SHA (and sometimes the wording) differs. + +| | Count | +|---|---| +| PRs in the 5.x line (beta.0 → beta.36) | 348 | +| Also shipped in 4.x (backported — excluded) | 108 | +| **Exclusive to 5.x, released** | **240** | +| **Exclusive to 5.x, merged but unreleased** | **12** | +| **Total in this document** | **252** | + +Organized by change severity, then thematically — not by subpackage. + +--- + +## 🔴 Breaking Changes + +- **`getWorld()` / `createWorld()` are now async**, to support ESM dynamic imports for custom world modules. All callers must `await getWorld()`. ([#942](https://github.com/vercel/workflow/pull/942)) +- **World stream methods moved into a `world.streams.*` namespace**, with `runId` promoted to the first parameter: `writeToStream(name, runId, chunk)` → `streams.write(runId, name, chunk)`, `writeToStreamMulti` → `streams.writeMulti`, `closeStream` → `streams.close`, `readFromStream` → `streams.get(runId, name, startIndex?)`, `listStreamsByRunId` → `streams.list(runId)`. `world.steps.get` now requires `runId`. ([#1293](https://github.com/vercel/workflow/pull/1293)) +- **Removed the `@workflow/core/private` and `workflow/internal/private` subpath exports.** Step registrations are now inlined as self-contained IIFEs (with closure variables inlined) immediately after each function definition, instead of a batched import. This also lets third-party packages define step functions without depending on `workflow` in `node_modules`. ([#1632](https://github.com/vercel/workflow/pull/1632)) +- **Removed path-based `isWorkflowSdkFile` serde exclusion** — serde discovery is now AST-level (SWC detect mode) across all integration paths. ([#1662](https://github.com/vercel/workflow/pull/1662)) +- **Removed the `client` SWC transform mode**, merged into `step` mode, which now absorbs hoisted variable references and DCE. Integrations passing `mode: 'client'` must move to `mode: 'step'`. ([#1686](https://github.com/vercel/workflow/pull/1686)) +- **Run and step errors now serialize through the full workflow serialization pipeline**, preserving class identity and cause chains on `WorkflowRunFailedError.cause`. Failed runs created before the upgrade, stored in `world-postgres`'s legacy `error` text column, read back as `error: undefined` (the payload is still recoverable from `errorJson`). ([#1851](https://github.com/vercel/workflow/pull/1851)) +- **The suspension/dispatch contract for World implementers changed.** The old asymmetric `{ timeoutSeconds }` return contract for waits is gone — waits are ordinary queue continuations with `delaySeconds`, and wait + step dispatch is unified into one parallel batch per suspension. Custom World packages relying on the old wait-return contract need updating. ([#1925](https://github.com/vercel/workflow/pull/1925)) +- **Duplicate step/workflow IDs across non-exported workspace files now fail the build** instead of silently colliding last-write-wins. ([#2018](https://github.com/vercel/workflow/pull/2018)) +- **Deterministic RNG/clock seed derivation changed** to `runId:workflowName:deploymentId` (dropping `startedAt`), with the initial clock seeded from the runId's ULID timestamp. This changes the seed-derived value sequence — step/hook correlation IDs, nanoids, random values — for a given run. **Runs started before this change must not be replayed across the upgrade.** ([#2525](https://github.com/vercel/workflow/pull/2525)) +- **Worlds are statically injected into host bundles at build time** instead of being selected dynamically at runtime, and first-party World packages standardized on a `createWorld()` factory. Custom and community world consumers should verify resolution still works — several follow-ups were needed ([#2799](https://github.com/vercel/workflow/pull/2799), [#2802](https://github.com/vercel/workflow/pull/2802), [#2804](https://github.com/vercel/workflow/pull/2804), [#2806](https://github.com/vercel/workflow/pull/2806)). ([#2752](https://github.com/vercel/workflow/pull/2752), [#2468](https://github.com/vercel/workflow/pull/2468)) +- **`NestLocalBuilder` moved out of the `@workflow/nest` package root** to the `@workflow/nest/builder` subpath, so importing `WorkflowModule` no longer pulls the build toolchain into the runtime bundle. Shipped alongside the new `workflow-nest build --vercel` command and `NestVercelBuilder` at `@workflow/nest/vercel-builder`. ([#2988](https://github.com/vercel/workflow/pull/2988)) +- **`experimental_setAttributes` renamed to `setAttributes`** now that attributes are no longer experimental. The old name remains as a deprecated alias. ([#2882](https://github.com/vercel/workflow/pull/2882)) + +### Default-behavior changes worth auditing + +- **The event-creation optimistic-concurrency guard is now on by default** (`WORKFLOW_PRECONDITION_GUARD=0` to opt out). ([#2946](https://github.com/vercel/workflow/pull/2946)) +- **Turbo mode is on by default** (`WORKFLOW_TURBO=0` to disable). ([#2526](https://github.com/vercel/workflow/pull/2526)) +- **`WORKFLOW_TRACE_MODE=linked` is the new default** — each invocation is its own trace root with span links rather than one deep trace. ([#2363](https://github.com/vercel/workflow/pull/2363), [#2527](https://github.com/vercel/workflow/pull/2527)) +- **`lazyDiscovery: true` is the default** for `withWorkflow` on Next.js ≥ 16.2.0-canary.48; older versions fall back to eager discovery automatically. ([#1805](https://github.com/vercel/workflow/pull/1805)) +- **Build output moved from CJS to ESM** for step/workflow/webhook bundles (with a `createRequire` banner for CJS deps). The VM-executed workflow bundle stays CJS. ([#1562](https://github.com/vercel/workflow/pull/1562)) +- **A server-supplied per-run event limit is now enforced** (default 25K). ([#2986](https://github.com/vercel/workflow/pull/2986)) +- **Stream writes now dispatch the first chunk of an idle stream immediately** (flush window default 0 instead of 10ms); opt back into a windowed leading edge via `streamFlushIntervalMs` / `WORKFLOW_STREAM_FLUSH_INTERVAL_MS`. *(unreleased)* ([#3088](https://github.com/vercel/workflow/pull/3088)) + +--- + +## 🟢 New Features & Enhancements + +### Runtime & execution model + +- Turbo mode: first delivery backgrounds `run_started`, skips the initial event-log load, and forces optimistic inline start, so first steps execute with no preceding round-trips. ([#2526](https://github.com/vercel/workflow/pull/2526), perf follow-up [#2569](https://github.com/vercel/workflow/pull/2569)) +- Parallel inline step execution up to `WORKFLOW_MAX_INLINE_STEPS` (default 3), each lazily created; opt-in `WORKFLOW_OPTIMISTIC_INLINE_START` begins step bodies before `step_started` is confirmed. ([#2516](https://github.com/vercel/workflow/pull/2516)) +- Lazy inline step start — a single `step_started` call now carries the step input, saving a round-trip per inline step. ([#2478](https://github.com/vercel/workflow/pull/2478)) +- Skip the per-step incremental `events.list` round-trip in the inline sequential loop by consuming the event-log delta from the step's terminal write. ([#2475](https://github.com/vercel/workflow/pull/2475)) +- Inline steps excluded from the replay timeout, with a `WORKFLOW_REPLAY_TIMEOUT_MS` override and a `World.processExitTriggersQueueRedelivery` capability flag. ([#2013](https://github.com/vercel/workflow/pull/2013)) +- Replays exceeding 240s are retried up to 3× instead of failing immediately. ([#1740](https://github.com/vercel/workflow/pull/1740)) +- Skip replay entirely when a refreshed event log already contains a terminal run event. ([#2215](https://github.com/vercel/workflow/pull/2215)) +- Encrypted replay payloads are prepared concurrently, and the decrypted/decompressed representation is cached for reuse across inline replay iterations. ([#2980](https://github.com/vercel/workflow/pull/2980)) +- Runtime tuning constants (timeouts, retry counts, stream buffering/reconnect) are configurable via `WORKFLOW_*` env vars; a `WORKFLOW_TEST_LIMIT_OVERRIDES` header lets a deployment tighten server-side limits for testing. ([#2718](https://github.com/vercel/workflow/pull/2718)) +- `start()` can be called directly inside workflow functions. ([#1491](https://github.com/vercel/workflow/pull/1491)) +- `WORKFLOW_SEQUENTIAL_REPLAYS` (opt-in, also enabled by the `WORKFLOW_SAFE_MODE=1` umbrella flag): flow/orchestrator routes are limited to one invocation per run via a per-run queue topic and `maxConcurrency: 1`. Step routes are unaffected. ([#2193](https://github.com/vercel/workflow/pull/2193)) +- Optimistic-concurrency guard for event creation: replay-context event creations send a `stateUpdatedAt` snapshot, and the runtime reloads the event log and retries — then falls back to a queue re-invocation — when the backend reports a newer out-of-band event with a 412 `PreconditionFailedError`. ([#2266](https://github.com/vercel/workflow/pull/2266)) +- `capabilities?: WorldCapabilities` on the World interface lets implementations declare backend feature support (`preconditionGuard`, `maxConcurrency`) instead of the runtime inferring it from env vars. The inline event-log delta fast path stays active with open hooks when the guard is on and declared; the lazy inline `step_started` claim carries the guard snapshot so a stale replay's claim is fenced. ([#2970](https://github.com/vercel/workflow/pull/2970)) +- Compression pipeline: step/workflow payloads, errors, and hook payloads are compressed before encryption (zstd preferred via `node:zlib`, gzip portable fallback), gated on run `specVersion` 5, with `WORKFLOW_DISABLE_COMPRESSION` / `WORKFLOW_COMPRESSION_CODEC` overrides and OTEL attributes for compression ratio and sizes. ([#2394](https://github.com/vercel/workflow/pull/2394), cutoff fix [#2470](https://github.com/vercel/workflow/pull/2470)) +- `specVersion` added to the World interface so `start()` uses the safe baseline (v2) for worlds that don't declare a supported version. ([#1658](https://github.com/vercel/workflow/pull/1658)) +- `World.createRunId(options?)` and `region` on `QueueOptions`: worlds can mint custom run IDs and route messages to a specific region. World-vercel mints region-tagged ULIDs preferring `options.region`, then `VERCEL_REGION`, then `iad1`, and the queue routes each message to the region encoded in the tagged run ID. ([#1981](https://github.com/vercel/workflow/pull/1981)) +- Cross-run lineage: runs started from inside a workflow or step are tagged with `$parentRunId` and inherit the parent's `$rootRunId`, so fan-outs and daisy chains of any depth group under one root. ([#2153](https://github.com/vercel/workflow/pull/2153)) +- `runs.getMany()` retrieves ordered run snapshots in one storage operation. ([#2915](https://github.com/vercel/workflow/pull/2915)) +- `experimental_minRetention` keeps a Hook token unavailable after its run ends; supporting Worlds must advertise the `hookRetention` capability. ([#2865](https://github.com/vercel/workflow/pull/2865)) +- `maxRetries` is enforced for inline and backgrounded steps that time out. ([#3035](https://github.com/vercel/workflow/pull/3035)) +- Auto-reconnecting object streams when the server connection times out ([#2318](https://github.com/vercel/workflow/pull/2318)), plus a v3 stream-read endpoint on world-vercel supporting transparent reconnects ([#2424](https://github.com/vercel/workflow/pull/2424)). +- `features.encryption` exposed on `getWorkflowMetadata()`. ([#1652](https://github.com/vercel/workflow/pull/1652)) + +### Run attributes & metadata + +- `experimental_setAttributes()` ([#2134](https://github.com/vercel/workflow/pull/2134)), callable from steps ([#2157](https://github.com/vercel/workflow/pull/2157)), initial attributes via `start()` with native-event recording ([#2226](https://github.com/vercel/workflow/pull/2226)), and `allowReservedAttributes` for framework-level `$`-prefixed keys ([#2385](https://github.com/vercel/workflow/pull/2385)). +- Optional `cancelReason` on `run.cancel()`. ([#2840](https://github.com/vercel/workflow/pull/2840)) +- `replayedFromRunId` stamped by `recreateRunFromExisting` / `start`. ([#2872](https://github.com/vercel/workflow/pull/2872)) +- `namespace` option for `start()`, `recreateRunFromExisting()`, `reenqueueRun()` and `wakeUpRun()`, plus a `healthCheck()` timeout fix. ([#2874](https://github.com/vercel/workflow/pull/2874)) + +### Errors & serialization + +- Friendlier runtime errors: new `SerializationError`, `WorkflowBuildError`, and structured context-violation classes (e.g. `NotInWorkflowContextError`) with docs links; `errorAttribution` (`user` vs `sdk`); namespaced `[workflow-sdk]` logs with a color-coded formatter. ([#1849](https://github.com/vercel/workflow/pull/1849)) +- `AbortController` / `AbortSignal` are serializable across workflow and step boundaries; pending queue items are drained rather than only warned about, and unaborted system hooks are disposed at workflow completion. ([#1301](https://github.com/vercel/workflow/pull/1301)) +- Built-in Error subclasses (`TypeError`, `RangeError`, `SyntaxError`, …) serialize with `cause` preserved ([#1511](https://github.com/vercel/workflow/pull/1511)); `FatalError` / `RetryableError` round-trip with class identity, including from non-SWC environments ([#1513](https://github.com/vercel/workflow/pull/1513)); `Run` instances get custom serialization with e2e boundary coverage ([#1616](https://github.com/vercel/workflow/pull/1616)); workflow function references are serializable ([#1677](https://github.com/vercel/workflow/pull/1677)). + +### Observability, tracing & analytics + +- `WORKFLOW_TRACE_MODE=linked`: each invocation is its own trace root with span links, plus W3C `traceparent` / `tracestate` / `baggage` injection. ([#2363](https://github.com/vercel/workflow/pull/2363), refined [#2527](https://github.com/vercel/workflow/pull/2527)) +- OTEL spans and turbo tagging around `/flow` init. ([#2592](https://github.com/vercel/workflow/pull/2592)) +- Time-to-first-step and step-to-step overhead latency telemetry ([#2833](https://github.com/vercel/workflow/pull/2833)); step progress reporting ([#2850](https://github.com/vercel/workflow/pull/2850)); stream latency spans ([#2857](https://github.com/vercel/workflow/pull/2857)). +- Run-started-to-first-step (`rsfs`) and final-scheduling-replay latency reported on step completion events. ([#2929](https://github.com/vercel/workflow/pull/2929)), with a follow-up fixing TTFS telemetry reporting 0 for runs with region-tagged run IDs ([#2943](https://github.com/vercel/workflow/pull/2943)) +- Client-observed stream telemetry: a `workflow.stream.flush` span per write batch with `buffer_dwell_ms` separating client batching cost from network time ([#2891](https://github.com/vercel/workflow/pull/2891)), later moved into core with `chunk_rtt`, `connect_ms`, and new `workflow.stream.close` / `workflow.stream.read.complete` spans, deduping `@opentelemetry/api` to one workspace instance ([#2901](https://github.com/vercel/workflow/pull/2901)). +- `world.analytics`: an optional metadata-only namespace for observability reads (runs/steps/events/hooks/waits), implemented by world-vercel ([#2234](https://github.com/vercel/workflow/pull/2234)), consumed by the web run/step/event list views ([#2647](https://github.com/vercel/workflow/pull/2647)), CLI `inspect` list views with `--withData` deprecated for lists ([#2648](https://github.com/vercel/workflow/pull/2648)), and the hooks list, where secret tokens are no longer included in list rows ([#2652](https://github.com/vercel/workflow/pull/2652)). +- `analytics.attributes.list()` for attribute key discovery, plus an `attributes` key=value filter on `analytics.runs.list()`. ([#2903](https://github.com/vercel/workflow/pull/2903)) +- `World.describeRun` hook surfaces world-specific run fields (e.g. region on Vercel) in `workflow inspect`. ([#2896](https://github.com/vercel/workflow/pull/2896)) +- CLI `inspect runs --since/--until`; `start` and bulk `cancel` name lookups search past the default 24h window; `world.analytics.runs.list` gained `startTime`/`endTime`; the runs list UI gained infinite scroll with SWR caching and a time-window/status picker. ([#2812](https://github.com/vercel/workflow/pull/2812)) + +### Trace viewer (`@workflow/web`, `@workflow/web-shared`) + +A ground-up replacement of the observability trace UI, delivered incrementally: + +- Clickable Run references ([#1681](https://github.com/vercel/workflow/pull/1681)); encrypted-marker and inline decryption support ([#1716](https://github.com/vercel/workflow/pull/1716), [#1722](https://github.com/vercel/workflow/pull/1722)); UTF-8 stream chunk decoding ([#1852](https://github.com/vercel/workflow/pull/1852)). +- Detail pane, middle-truncate and timeline polish ([#1883](https://github.com/vercel/workflow/pull/1883)); virtualization for large runs ([#2205](https://github.com/vercel/workflow/pull/2205)). +- `attr_set` event rendering with a dedicated Attributes card ([#2393](https://github.com/vercel/workflow/pull/2393), [#2327](https://github.com/vercel/workflow/pull/2327)). +- Reworked JSON data inspector with bracket notation, disclosure icons and `serializeForClipboard` ([#2434](https://github.com/vercel/workflow/pull/2434)); point-in-time event markers ([#2452](https://github.com/vercel/workflow/pull/2452)); resizable, user-draggable detail panel ([#2773](https://github.com/vercel/workflow/pull/2773)); minimap with pan/zoom/brush ([#2800](https://github.com/vercel/workflow/pull/2800)). +- Keyboard navigation: J/K auto-scroll ([#2366](https://github.com/vercel/workflow/pull/2366)), arrow keys ([#2694](https://github.com/vercel/workflow/pull/2694)), shortcut tooltips ([#2163](https://github.com/vercel/workflow/pull/2163)) and helper text ([#2582](https://github.com/vercel/workflow/pull/2582)). +- Alt-held span-to-span time measurement, Figma-style, with the no-selection gap overlay restyled to match. ([#2985](https://github.com/vercel/workflow/pull/2985)) +- Split-pane divider drag matched to the detail panel: wider hit target, hover/focus/drag highlight, double-click reset, keyboard/ARIA resize. ([#2838](https://github.com/vercel/workflow/pull/2838)) +- Loading skeleton exported as `TraceViewerSkeleton` ([#2164](https://github.com/vercel/workflow/pull/2164)); precise sub-second durations ([#2335](https://github.com/vercel/workflow/pull/2335)); `getModuleSourceUrl` resolver for linking to source ([#2817](https://github.com/vercel/workflow/pull/2817)); HookConflictError hydration ([#2249](https://github.com/vercel/workflow/pull/2249)); search matching/highlighting and a reusable icon button ([#2144](https://github.com/vercel/workflow/pull/2144)). +- gzip/zstd decompression for OSS web hydration via a WASM-backed zstd decoder. ([#2680](https://github.com/vercel/workflow/pull/2680)) +- Metadata Token and Hook ID rows gained copy buttons with middle truncation ([#2947](https://github.com/vercel/workflow/pull/2947)); Input/Output sections stay open after decrypting run data ([#2826](https://github.com/vercel/workflow/pull/2826)); detail-panel top info rows grouped into a collapsible monospace Metadata section ([#2604](https://github.com/vercel/workflow/pull/2604)); delayed hover tooltips on event-list row icons ([#2962](https://github.com/vercel/workflow/pull/2962)). +- Queued span time renders as a lead-in connector into the active bar instead of a filled gray box ([#2381](https://github.com/vercel/workflow/pull/2381)); in-progress segments animate ([#2383](https://github.com/vercel/workflow/pull/2383)); hook bars recolored gray instead of amber ([#2950](https://github.com/vercel/workflow/pull/2950)). +- Additional visual polish: [#1973](https://github.com/vercel/workflow/pull/1973), [#2006](https://github.com/vercel/workflow/pull/2006), [#2022](https://github.com/vercel/workflow/pull/2022), [#2036](https://github.com/vercel/workflow/pull/2036), [#2041](https://github.com/vercel/workflow/pull/2041), [#2045](https://github.com/vercel/workflow/pull/2045), [#2209](https://github.com/vercel/workflow/pull/2209), [#2459](https://github.com/vercel/workflow/pull/2459), [#2483](https://github.com/vercel/workflow/pull/2483), [#2520](https://github.com/vercel/workflow/pull/2520), [#2595](https://github.com/vercel/workflow/pull/2595), [#2614](https://github.com/vercel/workflow/pull/2614), [#2695](https://github.com/vercel/workflow/pull/2695), [#2832](https://github.com/vercel/workflow/pull/2832), [#2835](https://github.com/vercel/workflow/pull/2835), [#2837](https://github.com/vercel/workflow/pull/2837), [#2864](https://github.com/vercel/workflow/pull/2864), [#2968](https://github.com/vercel/workflow/pull/2968). + +### Build & framework integrations + +- A combined flow+step route handler executes steps inline where possible, cutting function invocations and queue overhead; default concurrency raised to 50. ([#1338](https://github.com/vercel/workflow/pull/1338)) +- Synchronous functions and getters can use `"use step"` ([#1633](https://github.com/vercel/workflow/pull/1633), [#1630](https://github.com/vercel/workflow/pull/1630)); SWC gained a `detect` mode used during discovery to filter false-positive directive detections ([#1641](https://github.com/vercel/workflow/pull/1641)). +- New `sourcemap` builder option mirroring esbuild's values, plus a `WORKFLOW_SOURCEMAP` env var ([#1842](https://github.com/vercel/workflow/pull/1842)); sourcemaps default to `'inline'` in dev and off in production, with faster stack-trace remapping when absent ([#2529](https://github.com/vercel/workflow/pull/2529)); source content embedded in published sourcemaps ([#1769](https://github.com/vercel/workflow/pull/1769)). +- Vercel Deployment Protection bypass switched to OIDC Trusted Sources via `getVercelOidcToken()`, superseding the `VERCEL_WORKFLOW_SERVER_PROTECTION_BYPASS` env var added a few betas earlier. ([#1882](https://github.com/vercel/workflow/pull/1882), env var introduced in [#1824](https://github.com/vercel/workflow/pull/1824)) +- Deferred Next.js builds stop eager input-graph directive discovery and rely on loader/socket-driven discovery with `onBeforeDeferredEntries`. ([#1646](https://github.com/vercel/workflow/pull/1646)) +- Next.js workflow diagnostics manifests are written inside the Next.js dist directory; `.vercel/output/diagnostics` is reserved for the Build Output API builder. ([#1857](https://github.com/vercel/workflow/pull/1857)) +- `workflow-nest build --vercel` emits a Vercel Build Output API directory, enabling NestJS deployment on Vercel. ([#2988](https://github.com/vercel/workflow/pull/2988)) +- Filesystem polling in `world-local` for cross-process streaming in local dev. ([#1739](https://github.com/vercel/workflow/pull/1739)) +- Standalone `workflow web` deploy-to-Vercel support, using `vercelPreset()` from `@vercel/react-router/vite` for per-route splitting. ([#1732](https://github.com/vercel/workflow/pull/1732), [#1815](https://github.com/vercel/workflow/pull/1815)) +- A default request timeout on world-vercel HTTP calls, so hanging responses retry sooner instead of running until function timeout. ([#1807](https://github.com/vercel/workflow/pull/1807)) +- `@workflow/ai`: forwards `strict` / `inputExamples` / `providerOptions` tool properties and handles `type: 'dynamic'` tools ([#1544](https://github.com/vercel/workflow/pull/1544)); preserves malformed streamed tool-call input for repair hooks ([#1707](https://github.com/vercel/workflow/pull/1707)); `DurableAgent.stream()` exposes `totalUsage` and `finishReason` ([#1863](https://github.com/vercel/workflow/pull/1863)). + +### Performance + +- Skip transferring event payload bytes when listing events with `resolveData: 'none'` on the v4 API. ([#2415](https://github.com/vercel/workflow/pull/2415)) +- Honor the server's explicit pagination flag when listing run events, avoiding one extra empty-page request per event-log load on replay. ([#2486](https://github.com/vercel/workflow/pull/2486)) +- Reduce local sequential-step replay I/O with bounded recent-event and storage-directory caches. ([#2152](https://github.com/vercel/workflow/pull/2152)) +- Shard local stream chunks into a directory per stream, so a tail reader's poll no longer lists every chunk in the world on each tick, and reliably release emitter listeners and the poll timer on cancellation. Steps receiving an `AbortSignal` no longer pay a per-step queue round-trip: the real-time abort-stream reader is released when the step finishes, letting it complete inline. **Note:** chunks now live at `streams/chunks//`; files written in the old flat layout are not read back, and stale flat files are left in place. ([#2807](https://github.com/vercel/workflow/pull/2807)) +- Hook operations in `world-local` no longer scale with total event history: hook creation, cache rebuilds and token lookups use durable per-token/per-hookId indexes instead of scanning the global event log; run-termination cleanup uses per-run markers; directory listings read concurrently; `runs.list` defaults to a page size of 200. ([#2830](https://github.com/vercel/workflow/pull/2830)) +- Avoid a separate esbuild steps bundle in Next.js lazy discovery mode. ([#2263](https://github.com/vercel/workflow/pull/2263)) +- Avoid resolving run data before background step execution. ([#2993](https://github.com/vercel/workflow/pull/2993)) + +--- + +## 🟡 Fixes + +### Runtime & core + +- `Promise.race(step, sleep)` no longer always blocks until the step completes. ([#1924](https://github.com/vercel/workflow/pull/1924)) +- Fixed a race where an `AbortController` aborted from a step wasn't reflected in a subsequent step's `signal`. ([#2412](https://github.com/vercel/workflow/pull/2412)) +- Fixed duplicate inline step execution when a hook or wait wakes a run while the step is still running: the lazy `step_started` now records the owning queue message ID with a delayed backstop (`WORKFLOW_INLINE_OWNERSHIP=0` to disable). ([#2848](https://github.com/vercel/workflow/pull/2848)) +- Fixed a turbo-mode race where a fire-and-forget hook, wait or attribute write could reach the server before the run was created. ([#2685](https://github.com/vercel/workflow/pull/2685)) +- Retry inline step completion persistence failures instead of recording them as step failures. ([#2666](https://github.com/vercel/workflow/pull/2666)) +- Fixed a `CorruptedEventLogError` false positive for `Promise.race([hook, sleep])` — branch-deciding deliveries are ordered by event position, not microtask timing. ([#2185](https://github.com/vercel/workflow/pull/2185)) +- Prevented replayed workflows from advancing their deterministic clock before the matching operation is invoked. ([#2211](https://github.com/vercel/workflow/pull/2211)) +- Retry transient replay divergence before classifying it as a corrupted event log. ([#2212](https://github.com/vercel/workflow/pull/2212)) +- Hardened event-log pagination against rejected, repeated and overlapping cursor responses. ([#2180](https://github.com/vercel/workflow/pull/2180)) +- Fixed false-positive unconsumed `step_created` errors when replay resumes a `for await` hook loop. ([#1778](https://github.com/vercel/workflow/pull/1778)) +- `getWritable()` no longer returns a new `TransformStream` per call, which reordered racing pipes; it now shares one pipe per `(runId, namespace)`. ([#2086](https://github.com/vercel/workflow/pull/2086)) +- Replaced `eval` with `JSON.parse` in the `revive()` deserialization helper. ([#1848](https://github.com/vercel/workflow/pull/1848)) +- Replaced the `chalk` import in `@workflow/errors/ansi` with an inline ANSI shim — `chalk` → `supports-color` called `require('os')` at load time, crashing every workflow in the sandboxed VM. ([#1915](https://github.com/vercel/workflow/pull/1915)) +- Fixed missing serialization revivers for `FatalError` / `RetryableError` / built-in Error subclasses / `AggregateError` / `DOMException` in the web UI. ([#1942](https://github.com/vercel/workflow/pull/1942)) +- Fixed `world.ts` being tree-shaken out of the bundle. ([#1951](https://github.com/vercel/workflow/pull/1951)) +- Fixed Zod 4.4.x compatibility in `WorkflowRunSchema`. ([#1939](https://github.com/vercel/workflow/pull/1939)) +- `AbortError` step failures, including cross-realm and serialized ones, are treated as fatal cancellations. ([#2150](https://github.com/vercel/workflow/pull/2150)) +- Reject an explicit empty-string `token` in `createHook()`. ([#2490](https://github.com/vercel/workflow/pull/2490)) +- Refresh workflow events after completing elapsed waits so concurrent hook events preserve deterministic replay order. ([#2038](https://github.com/vercel/workflow/pull/2038)) +- Removed the redundant `hc_` prefix from health-check `correlationId`, which produced a doubled `hc_hc_` in the derived runId and stream name. ([#1678](https://github.com/vercel/workflow/pull/1678)) +- Fixed the false "data expired" CLI warning for runs with a future `expiredAt`. ([#1736](https://github.com/vercel/workflow/pull/1736)) + +### Streams + +- Fixed stream writes never batching: `flushablePipe` awaited each `writer.write()` and the sink serialized chunks one at a time, so the server writable's buffer never held more than one chunk and its `writeMulti` path never engaged — every chunk became its own round trip. Coalesced batches split at `WORKFLOW_STREAM_MAX_CHUNKS_PER_BATCH` and `WORKFLOW_STREAM_MAX_BYTES_PER_BATCH`, independent of the `WORKFLOW_STREAM_MAX_INFLIGHT_CHUNKS` backpressure bound. ([#2995](https://github.com/vercel/workflow/pull/2995)) +- Use a custom stream close control frame to decide whether to reconnect. ([#1742](https://github.com/vercel/workflow/pull/1742)) +- Retry stream close on retriable 5xx — close is idempotent on the server, unlike chunk appends, and the server may return 503s expecting the writer to close again. ([#3038](https://github.com/vercel/workflow/pull/3038)) +- Stream write batching moved into the server writable itself (group commit), so raw `ReadableStream`s piped across workflow/step boundaries batch the same as `getWritable()` instead of sending one request per chunk. *(unreleased)* ([#3078](https://github.com/vercel/workflow/pull/3078)) + +### Worlds — local & postgres + +- Fixed a race where concurrent `step_created` / `hook_created` / `wait_created` writes with the same `correlationId` both succeeded instead of one losing with `EntityConflictError`; a unique partial index was added on postgres. ([#1877](https://github.com/vercel/workflow/pull/1877), [#1878](https://github.com/vercel/workflow/pull/1878)) +- `run_failed` against a nonexistent run throws `WorkflowRunNotFoundError`, matching postgres and vercel. ([#1894](https://github.com/vercel/workflow/pull/1894)) +- `world-postgres` throws `EntityConflictError` when a `run_created` event targets an existing run instead of resolving with no run, matching the other worlds and stopping `start()` from throwing `Missing 'run' in server response for 'run_created' event` when the resilient start path wins the race. ([#2983](https://github.com/vercel/workflow/pull/2983)) +- Reject `hook_received` on terminal runs, including when the termination commits concurrently cross-process, and for legacy pre-event-sourcing runs. ([#2987](https://github.com/vercel/workflow/pull/2987)) +- Fixed local-world recovery isolation in Vitest and support for custom test directories. ([#1895](https://github.com/vercel/workflow/pull/1895)) +- Retry local queue deliveries that fail at the transport (`fetch failed` / `ETIMEDOUT`) instead of dropping the message. ([#2679](https://github.com/vercel/workflow/pull/2679)) +- Rejected dots and empty `correlationId` values in entity ID validation. ([#2097](https://github.com/vercel/workflow/pull/2097)) +- Fixed a stalled hook token claim release deleting the next claimant's live claim, and a `resumeHook` / `dispose()` race that corrupted the receiving run's replay. ([#2808](https://github.com/vercel/workflow/pull/2808)) +- Fixed `createHook()` conflicting with the run's own disposed hook on token reuse, and claims not reaching the next claimant. ([#2779](https://github.com/vercel/workflow/pull/2779)) +- Use the active queue namespace when re-enqueuing runs during world startup recovery. ([#2888](https://github.com/vercel/workflow/pull/2888)) +- `WORKFLOW_LOCAL_RECOVER_ACTIVE_RUNS` env var as a fallback for the `recoverActiveRuns` option, so startup re-enqueueing can be disabled without a custom world module. ([#2914](https://github.com/vercel/workflow/pull/2914)) +- On shutdown, abort stalled workflow and step HTTP deliveries after Graphile Worker's grace period so Postgres job rows unlock through normal failure handling instead of waiting for stale-lock recovery; aborted deliveries consume an attempt and retry only when budget remains. Adds opt-in application-managed shutdown via `applicationManagedShutdown` / `WORKFLOW_POSTGRES_APPLICATION_MANAGED_SHUTDOWN=1`. *(unreleased)* ([#3064](https://github.com/vercel/workflow/pull/3064)) + +### World — Vercel + +- `streams.get()` includes `runId` in the request URL. ([#1676](https://github.com/vercel/workflow/pull/1676)) +- Injected W3C trace context on v4 event requests, restoring backend span correlation for flow-route traffic. ([#2533](https://github.com/vercel/workflow/pull/2533)) +- Routed v4 event requests through global `fetch` so they appear in the Vercel observability outgoing-requests view. ([#2514](https://github.com/vercel/workflow/pull/2514)) +- Cancel the v4 event frame stream when a reader stops early, returning the undici connection to the pool instead of leaking it. ([#2873](https://github.com/vercel/workflow/pull/2873)) +- Send `x-vercel-queue-region` on proxy-mode queue sends so they route to the correct region's dataplane. ([#2789](https://github.com/vercel/workflow/pull/2789)) +- Decode stable-line CBOR structured errors when reading v4 workflow events, while preserving current serialized error payloads. ([#2951](https://github.com/vercel/workflow/pull/2951)) +- Fixed observability run/event pages hanging (~16s) with no data in bundled server builds, caused by HTTP/2 requests failing to reach `node:http2`. ([#2632](https://github.com/vercel/workflow/pull/2632)) + +### Build & framework integrations + +- Node.js builtin imports are no longer relativized in step bundles ([#1644](https://github.com/vercel/workflow/pull/1644)); fixed bare-specifier resolution in lazy-discovery step-file copies ([#1670](https://github.com/vercel/workflow/pull/1670)); fixed step bundle discovery and externalization for SDK serde classes ([#1669](https://github.com/vercel/workflow/pull/1669)); fixed a discovery WeakMap cache miss causing duplicate esbuild passes on dev rebuilds ([#1699](https://github.com/vercel/workflow/pull/1699)); fixed `next/package.json` resolution in npm workspaces ([#1701](https://github.com/vercel/workflow/pull/1701)). +- Restored export validation for file-level `"use step"` files ([#1664](https://github.com/vercel/workflow/pull/1664)); eliminated unreferenced private class members after `"use step"` stripping ([#1671](https://github.com/vercel/workflow/pull/1671)); preserved original step function names in stack traces ([#1743](https://github.com/vercel/workflow/pull/1743)). +- Fixed eager Next.js workflow builds with lazy discovery disabled. ([#1747](https://github.com/vercel/workflow/pull/1747)) +- Fixed a false-positive `workflow-node-module-error` for step-only Node.js usage in shared modules. ([#1821](https://github.com/vercel/workflow/pull/1821)) +- Made the TypeScript peer dependency optional, with a clearer error when unavailable. ([#1830](https://github.com/vercel/workflow/pull/1830)) +- Forwarded Nitro `externals.external` string entries to the builder's esbuild config ([#1844](https://github.com/vercel/workflow/pull/1844)); matched the Nitro v3 webhook `functionRules` key to the real handler route ([#1575](https://github.com/vercel/workflow/pull/1575)); externalized the optional `@opentelemetry/api` peer only when not installed, across Rollup and Vite framework builds ([#1947](https://github.com/vercel/workflow/pull/1947)). +- Fixed `Package subpath … is not defined by "exports"` errors when step files reach project-local helpers via tsconfig paths or esbuild aliases. ([#1885](https://github.com/vercel/workflow/pull/1885)) +- Fixed duplicate Workflow queue consumers in SvelteKit deployments ([#1995](https://github.com/vercel/workflow/pull/1995)); fixed a SvelteKit production boot crash when a world package pulled `cosmiconfig` into the server bundle ([#2799](https://github.com/vercel/workflow/pull/2799)); avoided recursively loading Vite config while resolving SvelteKit routes ([#2802](https://github.com/vercel/workflow/pull/2802)). +- Fixed `workflow web` for local and postgres backends after the static world-injection change ([#2804](https://github.com/vercel/workflow/pull/2804)); the CLI resolves community world packages when not statically injected ([#2806](https://github.com/vercel/workflow/pull/2806)). +- Fixed detect-mode discovery for object-property step handlers ([#2484](https://github.com/vercel/workflow/pull/2484)); stopped warning on direct workflow calls from workflow code ([#2769](https://github.com/vercel/workflow/pull/2769)); rebuilt deferred Next.js entries on dev recompile ([#2438](https://github.com/vercel/workflow/pull/2438)); added a Windows-safe generated-file writer for step registration output ([#2853](https://github.com/vercel/workflow/pull/2853)). +- Sped up Next.js dev rebuilds, ignored commented imports during HMR discovery, avoided Turbopack resolving custom-world dynamic imports, and filtered Windows `netstat` output by PID when detecting local ports. ([#2678](https://github.com/vercel/workflow/pull/2678)) +- `@workflow/ai` preserves provider tool identity across step boundaries. ([#1663](https://github.com/vercel/workflow/pull/1663)) +- Clean up temporary Nitro Vite servers and Workflow build contexts after builds. ([#2908](https://github.com/vercel/workflow/pull/2908)) +- Dev watcher respects `.gitignore` and a `WORKFLOW_DEV_WATCH_IGNORED_PATHS` env var, avoiding `EMFILE: too many open files` on large monorepos. *(unreleased)* ([#3085](https://github.com/vercel/workflow/pull/3085)) +- `discoverWorkflowsInNodeModules` option and `WORKFLOW_DISCOVER_NODE_MODULES` env var stop discovery descending into `node_modules`, skipping the cost of scanning third-party dependencies. *(unreleased)* ([#3054](https://github.com/vercel/workflow/pull/3054)) +- Hoisted the `shouldFollowImportsFromFile` check out of `processImportSpecifier` so it is computed once per file instead of once per import specifier. *(unreleased)* ([#3052](https://github.com/vercel/workflow/pull/3052)) + +### Trace viewer + +- Fixed event data loading for `step_created` events ([#1685](https://github.com/vercel/workflow/pull/1685)); fixed stale and mixed data in the span detail panel while navigating ([#2325](https://github.com/vercel/workflow/pull/2325), [#2637](https://github.com/vercel/workflow/pull/2637)); fixed pagination getting stuck on the first page for large runs ([#2200](https://github.com/vercel/workflow/pull/2200)). +- Fixed an `EventRow` crash on spans without `attributes.data`, plus dead-file/JSX/cast cleanup ([#2252](https://github.com/vercel/workflow/pull/2252)); Tailwind v3 compatibility for the encrypted-preview blur utility ([#2108](https://github.com/vercel/workflow/pull/2108)); `Button` hover/focus/radius matched to Geist under Tailwind v3 and v4 ([#2143](https://github.com/vercel/workflow/pull/2143)). +- Fixed duplicate sub-second tick labels and trailing zeros in duration labels ([#2775](https://github.com/vercel/workflow/pull/2775)); fixed middle-truncation rendering ([#2827](https://github.com/vercel/workflow/pull/2827)); span timing uses workflow event occurrence timestamps where available ([#2613](https://github.com/vercel/workflow/pull/2613)). +- Disabled Vite minification for the published web build to avoid false-positive obfuscation flags from supply-chain scanners. ([#1768](https://github.com/vercel/workflow/pull/1768)) +- Parse timezone-naive analytics timestamps as UTC so CLI and local web output shows correct times in any timezone. ([#2899](https://github.com/vercel/workflow/pull/2899)) +- Animate the zoom controls consistently with span focus. *(unreleased)* ([#3060](https://github.com/vercel/workflow/pull/3060)) + +--- + +## ⚪ Internal & Chore + +- Refactored `serialization.ts` into modular files, no runtime change. ([#1299](https://github.com/vercel/workflow/pull/1299)) +- Renamed `useworkflow.dev` URLs to `workflow-sdk.dev` throughout docs and code. ([#1759](https://github.com/vercel/workflow/pull/1759)) +- Simplified the deferred Next.js builder's step-route generation. ([#1796](https://github.com/vercel/workflow/pull/1796)) +- Centralized workflow event-type classifiers and event-data payload helpers. ([#2790](https://github.com/vercel/workflow/pull/2790)) +- Internal v5 API format separately encoding event metadata from user payloads. ([#2055](https://github.com/vercel/workflow/pull/2055)) +- Skipped the abandoned `5.0.0-beta.8/9/10` npm slots left over from an earlier v5 attempt. ([#2168](https://github.com/vercel/workflow/pull/2168)) +- Added CI coverage for CLI and web trace-viewer revivers across all serializable types. ([#2250](https://github.com/vercel/workflow/pull/2250)) +- Reduced e2e timing and polling flakes ([#2665](https://github.com/vercel/workflow/pull/2665)); fixed a race in `world-testing`'s flow invocation counter causing intermittent inline-execution test failures ([#2043](https://github.com/vercel/workflow/pull/2043)). +- Updated vulnerable dependencies to patched releases. ([#2301](https://github.com/vercel/workflow/pull/2301)) +- Bumped `@vercel/queue` 0.3.1 → 0.4.0. ([#2876](https://github.com/vercel/workflow/pull/2876)) +- The "Initial v5 beta release" marker changeset, touching every package. ([#1642](https://github.com/vercel/workflow/pull/1642)) +- Isolated each spawned test server's data directory, fixing flaky Local World tests where concurrent servers shared one directory and re-enqueued each other's in-flight runs. *(unreleased)* ([#3055](https://github.com/vercel/workflow/pull/3055)) +- `WORKFLOW_DISABLE_ANALYTICS_READS=1` opts the world's `analytics` read namespace off, forcing `workflow inspect` list paths onto strongly consistent primary storage — for tests and tooling that read entities immediately after writing them. *(unreleased)* ([#3062](https://github.com/vercel/workflow/pull/3062)) +- Added a Stream Overhead (SO) benchmark scenario modelling an LLM token stream, with deterministic variable-length token deltas and an AI-SDK-shaped structured-delta payload variant; collapsed the benchmark PR-comment smallprint into a dropdown. *(unreleased)* ([#3077](https://github.com/vercel/workflow/pull/3077), [#3080](https://github.com/vercel/workflow/pull/3080)) +- CI backports only stability fixes to `stable`. *(unreleased)* ([#3092](https://github.com/vercel/workflow/pull/3092)) + +### Merged without a changeset + +These landed on `main` but carry no changeset, so they will not appear in release notes: + +- Upgrade postcss to ≥ 8.5.12 (CVE-2026-45623). ([#3067](https://github.com/vercel/workflow/pull/3067)) +- Upgrade postcss to ≥ 8.5.18 (GHSA-r28c-9q8g-f849). ([#3102](https://github.com/vercel/workflow/pull/3102)) + +--- + +## Excluded: backported to 4.x + +108 PRs appear in both lines and are omitted. Six of them were listed in an earlier revision of this document and dropped out when 4.6.1 / 4.6.2 shipped: + +- framework base-path routing ([#2732](https://github.com/vercel/workflow/pull/2732)) +- `extractStreamIds` stack overflow on circular references ([#2687](https://github.com/vercel/workflow/pull/2687)) +- loader sourcemaps in `node_modules` ([#2693](https://github.com/vercel/workflow/pull/2693)) +- the custom Next.js `distDir` dev watcher ([#2813](https://github.com/vercel/workflow/pull/2813)) +- the `world-testing` vitest peer range ([#2916](https://github.com/vercel/workflow/pull/2916)) +- skipping Workflow transforms for generated Nitro artifacts ([#2925](https://github.com/vercel/workflow/pull/2925)) + +The `next` 16.2.11 CVE-2026-64641 upgrade ([#3071](https://github.com/vercel/workflow/pull/3071)) is merged on `main` but was also backported via [#3073](https://github.com/vercel/workflow/pull/3073) and released in 4.6.2, so it is excluded too. diff --git a/docs/content/docs/v5/api-reference/workflow-errors/index.mdx b/docs/content/docs/v5/api-reference/workflow-errors/index.mdx index bd4c061ada..540fc5ccef 100644 --- a/docs/content/docs/v5/api-reference/workflow-errors/index.mdx +++ b/docs/content/docs/v5/api-reference/workflow-errors/index.mdx @@ -82,6 +82,9 @@ All errors extend [`WorkflowError`](/docs/api-reference/workflow-errors/workflow Thrown when an event creation is rejected because the client's event-log snapshot is stale. + + Thrown when an event creation is rejected because another writer already took the event's slot. + Thrown when a request is made before the system is ready to process it. diff --git a/docs/content/docs/v5/api-reference/workflow-errors/meta.json b/docs/content/docs/v5/api-reference/workflow-errors/meta.json index a84eddd1ad..b2de769e0e 100644 --- a/docs/content/docs/v5/api-reference/workflow-errors/meta.json +++ b/docs/content/docs/v5/api-reference/workflow-errors/meta.json @@ -15,6 +15,7 @@ "throttle-error", "entity-conflict-error", "precondition-failed-error", + "slot-conflict-error", "run-expired-error", "run-not-supported-error", "too-early-error" diff --git a/docs/content/docs/v5/api-reference/workflow-errors/slot-conflict-error.mdx b/docs/content/docs/v5/api-reference/workflow-errors/slot-conflict-error.mdx new file mode 100644 index 0000000000..96a4f67f17 --- /dev/null +++ b/docs/content/docs/v5/api-reference/workflow-errors/slot-conflict-error.mdx @@ -0,0 +1,76 @@ +--- +title: SlotConflictError +description: Thrown when an event creation is rejected because another writer already took the event's slot. +type: reference +summary: Catch SlotConflictError when a world rejects an event creation whose slot in the run's event log was already taken. +related: + - /docs/api-reference/workflow-errors/workflow-world-error + - /docs/api-reference/workflow-errors/precondition-failed-error +--- + +`SlotConflictError` is thrown by world implementations when an event creation is rejected because the event's slot in the run's event log was already taken by another writer. It corresponds to HTTP 409 Conflict semantics. + +On a run that numbers its events by slot, each event's id encodes its position in the log: the first event is slot 1, the second slot 2, and so on. Whoever writes a slot first owns it, so a rejected write proves the client was replaying against an event log that was missing at least one event. Retrying the same write can never succeed — the client has to merge the events it was missing, replay, and propose whatever slot that replay lands on. + +The rejection carries the missing events inline so that merge usually costs no extra round-trip: + +- `events` — the events recorded after the client's snapshot, in ascending slot order. Empty when the backend could not read them, in which case the client reloads the log itself. +- `cursor` — cursor to continue the delta from. +- `hasMore` — whether events beyond `events` remain to be fetched. + +This is the slot-numbering counterpart to [`PreconditionFailedError`](/docs/api-reference/workflow-errors/precondition-failed-error), which is how the same staleness is reported for runs guarded by an event-log snapshot watermark instead. A run uses one scheme or the other for its whole life, decided when it is created. + + +The Workflow runtime handles this error automatically: it merges the events it was missing, replays, and re-proposes the write at a free slot, ultimately re-enqueueing the run for a fresh replay if it cannot catch up. You will only encounter it when interacting with world storage APIs directly. + + +```typescript lineNumbers +import { SlotConflictError } from "workflow/errors" +declare const world: { events: { create(...args: any[]): Promise } }; // @setup +declare const runId: string; // @setup +declare const event: any; // @setup + +try { + await world.events.create(runId, event); +} catch (error) { + if (SlotConflictError.is(error)) { // [!code highlight] + console.log(`Slot ${error.eventId} taken; ${error.events.length} event(s) missed`); + } +} +``` + +## API Signature + +### Properties + + + +### Static Methods + +#### `SlotConflictError.is(value)` + +Type-safe check for `SlotConflictError` instances. Preferred over `instanceof` because it works across module boundaries and VM contexts. + +```typescript +import { SlotConflictError } from "workflow/errors" +declare const error: unknown; // @setup + +if (SlotConflictError.is(error)) { + // error is typed as SlotConflictError +} +``` diff --git a/docs/content/docs/v5/configuration/runtime-tuning.mdx b/docs/content/docs/v5/configuration/runtime-tuning.mdx index 2f3e68e8c3..d5a5c7e85a 100644 --- a/docs/content/docs/v5/configuration/runtime-tuning.mdx +++ b/docs/content/docs/v5/configuration/runtime-tuning.mdx @@ -48,6 +48,15 @@ For example, a workflow can run a 10-minute inline step even with `WORKFLOW_REPL - Backends that do not support the guard ignore the snapshot; they must not declare the capability, so guard-dependent optimizations stay off against them even when the flag is set. - Set `0` to disable. +### `WORKFLOW_SLOT_IDENTITY` + +- Default: enabled +- Numbers a new run's events by position instead of by ULID: `evnt_…001` is the run's first event, `evnt_…002` its second. Positions are allocated in order, so the log reads in the order it was written regardless of clock skew between writers. +- Contention becomes explicit rather than silent. Two writers proposing the same position cannot both win: the loser gets a 409 ([`SlotConflictError`](/docs/api-reference/workflow-errors/slot-conflict-error)) carrying the events it was missing, and the runtime merges them, replays, and re-proposes above the events it just learned about. +- Applies only to runs created while it is enabled. A run keeps the identity scheme it was created with for life, so turning the flag on or off never affects runs already in flight. +- Requires a World that supports it. A World that does not rejects the run outright rather than mis-numbering its events. +- Set `0` or `false` to disable, which numbers new runs by ULID as before. + ## Inline execution ### `WORKFLOW_V2_TIMEOUT_MS` @@ -189,6 +198,12 @@ These variables are primarily for tests, debugging, or unusual deployments. - Delay before the unconsumed-event check fires. - Minimum: `10`. +### `WORKFLOW_DEFERRED_CHECK_MAX_GRACE_MS` + +- Default: `15000` +- How long the unconsumed-event check keeps waiting while a step result or hook payload is still on its way to the workflow. An event whose consumer has not been registered yet looks exactly like an orphaned one, so the check waits rather than failing the run. +- Once this budget is spent the check reports regardless, so a delivery that never lands cannot keep a genuinely orphaned event from being detected. + ### `WORKFLOW_LOCK_POLL_INTERVAL_MS` - Default: `10` diff --git a/packages/core/e2e/event-log-race-repro.test.ts b/packages/core/e2e/event-log-race-repro.test.ts index e98f6fd10a..b40fed2ec3 100644 --- a/packages/core/e2e/event-log-race-repro.test.ts +++ b/packages/core/e2e/event-log-race-repro.test.ts @@ -373,6 +373,30 @@ function validateStormReturn(value: unknown): { return { stragglers }; } +/** + * Reads a terminal-failed run's error through `returnValue()`, which hydrates + * the stored payload into an Error. Returns undefined when the read itself + * fails — the outcome is already known from `errorCode`, so a missing message + * degrades the report rather than the classification. + */ +async function readFailureMessage( + run: Run +): Promise<{ name?: string; message?: string } | undefined> { + try { + await run.returnValue(); + return undefined; + } catch (err) { + if (WorkflowRunFailedError.is(err)) { + const cause = err.cause; + return { + name: cause instanceof Error ? cause.name : err.name, + message: cause instanceof Error ? cause.message : err.message, + }; + } + return undefined; + } +} + async function pollTerminalRun( run: Run, startedAt: number, @@ -412,13 +436,20 @@ async function pollTerminalRun( errorCode?: string; error?: { name?: string; message?: string }; }; + // `runs.get` hands back the raw serialized error payload, not an Error, so + // reading `.message` off it yields undefined and the report records the + // code with no diagnosis. Read the failure through the public + // return-value path, which hydrates it. For a corruption that message + // carries the divergent event and what the replay was waiting for, which + // is the whole reason to keep the report. + const hydrated = await readFailureMessage(run); return { ...base, outcome: classifyFailure(failure.errorCode), status: runData.status, errorCode: failure.errorCode, - errorMessage: failure.error?.message, - errorName: failure.error?.name, + errorMessage: hydrated?.message ?? failure.error?.message, + errorName: hydrated?.name ?? failure.error?.name, durationMs: Date.now() - startedAt, }; } diff --git a/packages/core/src/abort-consistency.test.ts b/packages/core/src/abort-consistency.test.ts index fcd5ed09f7..5eb1c35905 100644 --- a/packages/core/src/abort-consistency.test.ts +++ b/packages/core/src/abort-consistency.test.ts @@ -11,6 +11,7 @@ import type { Event, WorkflowRun } from '@workflow/world'; import * as nanoid from 'nanoid'; import { monotonicFactory } from 'ulid'; import { describe, expect, it, vi } from 'vitest'; +import { createCorrelationIdFactory } from './correlation-ids.js'; import { EventsConsumer } from './events-consumer.js'; import type { WorkflowSuspension } from './global.js'; import type { WorkflowOrchestratorContext } from './private.js'; @@ -45,6 +46,10 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { }), invocationsQueue: new Map(), generateUlid: () => ulid(workflowStartedAt), + nextCorrelationId: createCorrelationIdFactory({ + specVersion: undefined, + generateUlid: () => ulid(workflowStartedAt), + }), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) ), diff --git a/packages/core/src/abort-controller.test.ts b/packages/core/src/abort-controller.test.ts index ee839b19d6..1096fc9404 100644 --- a/packages/core/src/abort-controller.test.ts +++ b/packages/core/src/abort-controller.test.ts @@ -12,6 +12,7 @@ import type { Event } from '@workflow/world'; import * as nanoid from 'nanoid'; import { monotonicFactory } from 'ulid'; import { describe, expect, it, vi } from 'vitest'; +import { createCorrelationIdFactory } from './correlation-ids.js'; import { DEFERRED_CHECK_DELAY_MS, EventsConsumer } from './events-consumer.js'; import type { WorkflowOrchestratorContext } from './private.js'; import { ReplayPayloadCache } from './replay-payload-cache.js'; @@ -43,6 +44,10 @@ function setupWorkflowContext( }), invocationsQueue: new Map(), generateUlid: () => ulid(workflowStartedAt), + nextCorrelationId: createCorrelationIdFactory({ + specVersion: undefined, + generateUlid: () => ulid(workflowStartedAt), + }), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) ), diff --git a/packages/core/src/abort-replay-ordering.test.ts b/packages/core/src/abort-replay-ordering.test.ts index 8ddf4aec03..2708380899 100644 --- a/packages/core/src/abort-replay-ordering.test.ts +++ b/packages/core/src/abort-replay-ordering.test.ts @@ -27,6 +27,7 @@ import type { Event } from '@workflow/world'; import * as nanoid from 'nanoid'; import { monotonicFactory } from 'ulid'; import { describe, expect, it, vi } from 'vitest'; +import { createCorrelationIdFactory } from './correlation-ids.js'; import { EventsConsumer } from './events-consumer.js'; import { scheduleWhenIdle, @@ -78,6 +79,10 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { }), invocationsQueue: new Map(), generateUlid: () => ulid(workflowStartedAt), + nextCorrelationId: createCorrelationIdFactory({ + specVersion: undefined, + generateUlid: () => ulid(workflowStartedAt), + }), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) ), diff --git a/packages/core/src/async-deserialization-ordering.test.ts b/packages/core/src/async-deserialization-ordering.test.ts index 7a1ff1d346..4dc20fd9a6 100644 --- a/packages/core/src/async-deserialization-ordering.test.ts +++ b/packages/core/src/async-deserialization-ordering.test.ts @@ -4,6 +4,7 @@ import * as nanoid from 'nanoid'; import { monotonicFactory } from 'ulid'; import { afterEach, beforeAll, describe, expect, it, vi } from 'vitest'; import { registerSerializationClass } from './class-serialization.js'; +import { createCorrelationIdFactory } from './correlation-ids.js'; import { EventsConsumer } from './events-consumer.js'; import type { WorkflowOrchestratorContext } from './private.js'; import { ReplayPayloadCache } from './replay-payload-cache.js'; @@ -58,6 +59,10 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { }), invocationsQueue: new Map(), generateUlid: () => ulid(workflowStartedAt), + nextCorrelationId: createCorrelationIdFactory({ + specVersion: undefined, + generateUlid: () => ulid(workflowStartedAt), + }), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) ), diff --git a/packages/core/src/correlation-ids.test.ts b/packages/core/src/correlation-ids.test.ts new file mode 100644 index 0000000000..f4771e6d7a --- /dev/null +++ b/packages/core/src/correlation-ids.test.ts @@ -0,0 +1,92 @@ +import { + FIRST_SLOT, + SLOT_ID_WIDTH, + SPEC_VERSION_CURRENT, + SPEC_VERSION_SLOT_IDENTITY, + slotFromId, +} from '@workflow/world'; +import { describe, expect, it } from 'vitest'; +import { createCorrelationIdFactory } from './correlation-ids.js'; + +/** Stands in for the invocation's seeded, replay-stable ULID generator. */ +function fakeUlids(): () => string { + let issued = 0; + return () => `01ULID${String(++issued).padStart(20, '0')}`; +} + +function slotFactory() { + return createCorrelationIdFactory({ + specVersion: SPEC_VERSION_SLOT_IDENTITY, + generateUlid: fakeUlids(), + }); +} + +describe('createCorrelationIdFactory', () => { + describe('slot identity', () => { + it('numbers each kind densely from the first slot', () => { + const next = slotFactory(); + expect(slotFromId(next('step'))).toBe(FIRST_SLOT); + expect(slotFromId(next('step'))).toBe(FIRST_SLOT + 1); + expect(slotFromId(next('wait'))).toBe(FIRST_SLOT); + }); + + it('keeps the prefix of the kind it was asked for', () => { + const next = slotFactory(); + expect(next('step')).toMatch( + new RegExp(`^step_[0-9]{${SLOT_ID_WIDTH}}$`) + ); + expect(next('wait')).toMatch( + new RegExp(`^wait_[0-9]{${SLOT_ID_WIDTH}}$`) + ); + }); + + it('issues the same sequence to two fresh invocations', () => { + // Replay stability: the VM is rebuilt per replay and the workflow body + // issues its operations in the same order, so nothing needs seeding. + const replay = () => { + const next = slotFactory(); + return [next('step'), next('wait'), next('step'), next('step')]; + }; + expect(replay()).toEqual(replay()); + }); + + it('does not renumber steps or waits when another kind allocates', () => { + // Kinds that stay on ULIDs (hooks, attributes) draw from generateUlid, + // and a per-kind counter means interleaving them cannot shift a step's + // number — which a single shared sequence would. + const ulids = fakeUlids(); + const next = createCorrelationIdFactory({ + specVersion: SPEC_VERSION_SLOT_IDENTITY, + generateUlid: ulids, + }); + const firstStep = next('step'); + ulids(); + ulids(); + const secondStep = next('step'); + expect(slotFromId(firstStep)).toBe(FIRST_SLOT); + expect(slotFromId(secondStep)).toBe(FIRST_SLOT + 1); + expect(slotFromId(next('wait'))).toBe(FIRST_SLOT); + }); + }); + + describe('ULID identity', () => { + it('draws from the invocation generator for every kind', () => { + const next = createCorrelationIdFactory({ + specVersion: SPEC_VERSION_CURRENT, + generateUlid: fakeUlids(), + }); + // One shared sequence, exactly as before slots existed: an id's number + // reflects the order of allocation across all kinds. + expect(next('step')).toBe('step_01ULID00000000000000000001'); + expect(next('wait')).toBe('wait_01ULID00000000000000000002'); + }); + + it('treats a run with no spec version as ULID-numbered', () => { + const next = createCorrelationIdFactory({ + specVersion: undefined, + generateUlid: fakeUlids(), + }); + expect(slotFromId(next('step'))).toBeUndefined(); + }); + }); +}); diff --git a/packages/core/src/correlation-ids.ts b/packages/core/src/correlation-ids.ts new file mode 100644 index 0000000000..dc5d6a69fc --- /dev/null +++ b/packages/core/src/correlation-ids.ts @@ -0,0 +1,65 @@ +/** + * Correlation-id allocation for a single workflow invocation. + * + * A correlation id names an operation the workflow body issued — a step call, a + * sleep — and must come out identical on every replay of that run, because it + * is how a replay recognises the event that already recorded the operation. + * + * Two schemes exist. A run on ULID identity draws from the invocation's seeded + * monotonic ULID generator, which is replay-stable because its seed and initial + * clock are derived from the run. A run on slot identity counts: the first step + * of the run is `step_…001`, the second `step_…002`, zero-padded to ULID width + * (see `slotIdBody`). Which scheme applies is fixed by the run's persisted + * `specVersion` and never by the build, so a run started before slot identity + * keeps proposing the ids its log already holds. + * + * Counters are **per kind**, and there is nothing to seed them with. The VM is + * rebuilt for every replay and the workflow body issues its operations in the + * same order every time, which is the same argument that licenses the seeded + * ULID sequence today. Recovering counters from the loaded log would be actively + * wrong: the n-th step's id would then depend on how much of the log this + * replay happened to load. + * + * Per-kind is a strict improvement over the shared ULID sequence. Today all + * four id kinds draw from one generator, so introducing a hook allocation + * renumbers every step and wait issued after it; separate counters mean a step's + * number depends only on the steps before it. + * + * Hook and attribute ids stay on ULIDs and are not allocated here. Both are + * written from outside the VM in cases where no counter exists (an attribute set + * on a run from the outside), and both already carry their own per-run + * idempotency, so slots would buy them nothing. + */ + +import { FIRST_SLOT, slotIdBody, usesSlotIdentity } from '@workflow/world'; + +/** Operation kinds whose correlation ids are allocated per run and per kind. */ +export type CorrelationKind = 'step' | 'wait'; + +/** + * Allocates the next correlation id for a kind, prefix included. Returning the + * finished id — rather than a number or a bare body — keeps the prefix from + * ever diverging from the counter it was drawn against. + */ +export type CorrelationIdFactory = (kind: CorrelationKind) => string; + +export function createCorrelationIdFactory({ + specVersion, + generateUlid, +}: { + /** The run's *persisted* spec version. */ + specVersion: number | undefined; + /** The invocation's replay-stable ULID generator. */ + generateUlid: () => string; +}): CorrelationIdFactory { + if (!usesSlotIdentity(specVersion)) { + return (kind) => `${kind}_${generateUlid()}`; + } + + const allocated = new Map(); + return (kind) => { + const slot = (allocated.get(kind) ?? FIRST_SLOT - 1) + 1; + allocated.set(kind, slot); + return `${kind}_${slotIdBody(slot)}`; + }; +} diff --git a/packages/core/src/delivery-barrier-coverage.test.ts b/packages/core/src/delivery-barrier-coverage.test.ts index 836d7d3948..6b97f00092 100644 --- a/packages/core/src/delivery-barrier-coverage.test.ts +++ b/packages/core/src/delivery-barrier-coverage.test.ts @@ -45,6 +45,7 @@ import type { Event } from '@workflow/world'; import * as nanoid from 'nanoid'; import { monotonicFactory } from 'ulid'; import { describe, expect, it, vi } from 'vitest'; +import { createCorrelationIdFactory } from './correlation-ids.js'; import { EventsConsumer } from './events-consumer.js'; import { WorkflowSuspension } from './global.js'; import { @@ -86,6 +87,10 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { }), invocationsQueue: new Map(), generateUlid: () => ulid(workflowStartedAt), + nextCorrelationId: createCorrelationIdFactory({ + specVersion: undefined, + generateUlid: () => ulid(workflowStartedAt), + }), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) ), diff --git a/packages/core/src/events-consumer.test.ts b/packages/core/src/events-consumer.test.ts index ecf828f728..6294803363 100644 --- a/packages/core/src/events-consumer.test.ts +++ b/packages/core/src/events-consumer.test.ts @@ -1,7 +1,15 @@ import { withResolvers } from '@workflow/utils'; import type { Event } from '@workflow/world'; -import { describe, expect, it, vi } from 'vitest'; -import { EventConsumerResult, EventsConsumer } from './events-consumer.js'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { + DEFERRED_CHECK_DELAY_MS, + EventConsumerResult, + EventsConsumer, +} from './events-consumer.js'; + +afterEach(() => { + vi.unstubAllEnvs(); +}); // Helper function to create mock events function createMockEvent(overrides: Partial = {}): Event { @@ -468,12 +476,63 @@ describe('EventsConsumer', () => { expect(consumer.eventIndex).toBe(1); }); - // Wait past the internal 100ms unconsumed-event setTimeout window to - // ensure the cancelled check truly does not fire. - await new Promise((resolve) => setTimeout(resolve, 150)); + // Wait past the internal unconsumed-event setTimeout window to ensure the + // cancelled check truly does not fire. + await new Promise((resolve) => + setTimeout(resolve, DEFERRED_CHECK_DELAY_MS * 1.5) + ); // The new callback consumed the event, so onUnconsumedEvent should NOT be called expect(onUnconsumedEvent).not.toHaveBeenCalled(); }); + + it('waits while a delivery is in flight, then reports once it lands', async () => { + vi.stubEnv('WORKFLOW_DEFERRED_CHECK_DELAY_MS', '10'); + const event = createMockEvent(); + const onUnconsumedEvent = vi.fn(); + let inFlight = true; + const consumer = new EventsConsumer([event], { + onUnconsumedEvent, + getPromiseQueue: () => Promise.resolve(), + isDeliveryInFlight: () => inFlight, + }); + + consumer.subscribe( + vi.fn().mockReturnValue(EventConsumerResult.NotConsumed) + ); + + // Many delay windows pass. A delivery still on its way to the workflow + // means the consumer for this event has not been registered YET — which + // is not the same thing as the event being orphaned. + await new Promise((resolve) => setTimeout(resolve, 200)); + expect(onUnconsumedEvent).not.toHaveBeenCalled(); + + inFlight = false; + await vi.waitFor(() => { + expect(onUnconsumedEvent).toHaveBeenCalledWith(event); + }); + }); + + it('reports once the grace budget runs out even if a delivery never lands', async () => { + vi.stubEnv('WORKFLOW_DEFERRED_CHECK_DELAY_MS', '10'); + vi.stubEnv('WORKFLOW_DEFERRED_CHECK_MAX_GRACE_MS', '50'); + const event = createMockEvent(); + const onUnconsumedEvent = vi.fn(); + const consumer = new EventsConsumer([event], { + onUnconsumedEvent, + getPromiseQueue: () => Promise.resolve(), + // Never clears: a delivery that is abandoned must not park the check + // forever, or a genuinely orphaned event would never be reported. + isDeliveryInFlight: () => true, + }); + + consumer.subscribe( + vi.fn().mockReturnValue(EventConsumerResult.NotConsumed) + ); + + await vi.waitFor(() => { + expect(onUnconsumedEvent).toHaveBeenCalledWith(event); + }); + }); }); }); diff --git a/packages/core/src/events-consumer.ts b/packages/core/src/events-consumer.ts index 4b7cf0742e..5db54b4169 100644 --- a/packages/core/src/events-consumer.ts +++ b/packages/core/src/events-consumer.ts @@ -26,6 +26,26 @@ const getDeferredCheckDelayMs = (): number => min: 10, }); +/** + * Upper bound on how long the unconsumed-event check keeps re-arming while a + * data delivery is still in flight (see `isDeliveryInFlight`). The delay above + * is a margin for a microtask chain; this is a margin for real async work — + * decrypting a hook payload, fetching a remote ref — that has to finish before + * the VM can resume the branch that registers the next event's consumer. + * + * Bounded rather than unbounded so a genuinely orphaned event still reports, + * and so a delivery that never lands cannot park the check forever. + */ +export const DEFERRED_CHECK_MAX_GRACE_MS = 15_000; + +/** Override: `WORKFLOW_DEFERRED_CHECK_MAX_GRACE_MS`. */ +const getDeferredCheckMaxGraceMs = (): number => + envNumber( + 'WORKFLOW_DEFERRED_CHECK_MAX_GRACE_MS', + DEFERRED_CHECK_MAX_GRACE_MS, + { integer: true, min: 0 } + ); + export enum EventConsumerResult { /** * Callback consumed the event, but should not be removed from the callbacks list @@ -65,6 +85,16 @@ export interface EventsConsumerOptions { * deserialization delays the resolve() that triggers the next subscribe(). */ getPromiseQueue: () => Promise; + /** + * Whether a data delivery (step result, hook payload) is still on its way to + * the workflow. The unconsumed-event check re-arms while this holds instead + * of reporting: an event whose consumer has not been registered yet is + * indistinguishable from an orphaned one by log inspection alone, and the + * promise-queue drain does not cover the gap between a delivery's `resolve()` + * and the VM body reaching its next `subscribe()`. Defaults to never in + * flight, which is the plain wall-clock behaviour. + */ + isDeliveryInFlight?: () => boolean; } export class EventsConsumer { @@ -74,6 +104,7 @@ export class EventsConsumer { private onConsumedEvent?: (event: Event) => void; private onUnconsumedEvent: (event: Event) => void; private getPromiseQueue: () => Promise; + private isDeliveryInFlight: () => boolean; private pendingUnconsumedCheck: Promise | null = null; private pendingUnconsumedTimeout: ReturnType | null = null; private unconsumedCheckVersion = 0; @@ -84,6 +115,7 @@ export class EventsConsumer { this.onConsumedEvent = options.onConsumedEvent; this.onUnconsumedEvent = options.onUnconsumedEvent; this.getPromiseQueue = options.getPromiseQueue; + this.isDeliveryInFlight = options.isDeliveryInFlight ?? (() => false); } /** @@ -200,32 +232,66 @@ export class EventsConsumer { // is still unconsumed after the queue drains, it's truly orphaned. if (currentEvent !== null) { const checkVersion = ++this.unconsumedCheckVersion; - this.pendingUnconsumedCheck = this.getPromiseQueue() - .then( - // Yield once after the first queue drain so promise chains resumed by - // that drain can run across the VM boundary and append any follow-up - // async work (for example: step_completed resolves -> for-await loop - // resumes -> the next hook payload starts hydrating). - () => new Promise((resolve) => setTimeout(resolve, 0)) - ) - .then(() => this.getPromiseQueue()) - .then(() => { - // Use a delayed setTimeout after the queue drains. The delay must be - // long enough for promise chains to propagate across the VM boundary - // (from resolve() in the host context through to the workflow code - // calling subscribe() in the VM context). Node.js does not guarantee - // that setTimeout(0) fires after all cross-context microtasks settle, - // so we use a small but non-zero delay. Any subscribe() call that - // arrives during this window will cancel the check via version - // invalidation + clearTimeout. - this.pendingUnconsumedTimeout = setTimeout(() => { - this.pendingUnconsumedTimeout = null; - if (this.unconsumedCheckVersion === checkVersion) { - this.pendingUnconsumedCheck = null; - this.onUnconsumedEvent(currentEvent); - } - }, getDeferredCheckDelayMs()); - }); + this.armUnconsumedCheck( + currentEvent, + checkVersion, + getDeferredCheckMaxGraceMs() + ); } } + + /** + * Wait for the promise queue to drain, then a short delay, then report + * `currentEvent` as unconsumed — unless a `subscribe()` invalidated + * `checkVersion` in the meantime, or a delivery is still in flight, in which + * case re-arm with `graceRemainingMs` reduced by the delay just spent. + */ + private armUnconsumedCheck( + currentEvent: Event, + checkVersion: number, + graceRemainingMs: number + ) { + const delay = getDeferredCheckDelayMs(); + this.pendingUnconsumedCheck = this.getPromiseQueue() + .then( + // Yield once after the first queue drain so promise chains resumed by + // that drain can run across the VM boundary and append any follow-up + // async work (for example: step_completed resolves -> for-await loop + // resumes -> the next hook payload starts hydrating). + () => new Promise((resolve) => setTimeout(resolve, 0)) + ) + .then(() => this.getPromiseQueue()) + .then(() => { + // Use a delayed setTimeout after the queue drains. The delay must be + // long enough for promise chains to propagate across the VM boundary + // (from resolve() in the host context through to the workflow code + // calling subscribe() in the VM context). Node.js does not guarantee + // that setTimeout(0) fires after all cross-context microtasks settle, + // so we use a small but non-zero delay. Any subscribe() call that + // arrives during this window will cancel the check via version + // invalidation + clearTimeout. + this.pendingUnconsumedTimeout = setTimeout(() => { + this.pendingUnconsumedTimeout = null; + if (this.unconsumedCheckVersion !== checkVersion) { + return; + } + if (graceRemainingMs > 0 && this.isDeliveryInFlight()) { + // A delivery is hydrating, or has resolved but is parked behind its + // deferral. The workflow body has not had the chance to register + // this event's consumer yet, so reporting now would reject a + // healthy run: the resulting `ReplayDivergenceError` recurs on + // every replay that is unlucky in the same way and escalates to a + // terminal `CorruptedEventLogError`. + this.armUnconsumedCheck( + currentEvent, + checkVersion, + graceRemainingMs - delay + ); + return; + } + this.pendingUnconsumedCheck = null; + this.onUnconsumedEvent(currentEvent); + }, delay); + }); + } } diff --git a/packages/core/src/hook-sleep-interaction.test.ts b/packages/core/src/hook-sleep-interaction.test.ts index 29d64fcc58..c02d36ad19 100644 --- a/packages/core/src/hook-sleep-interaction.test.ts +++ b/packages/core/src/hook-sleep-interaction.test.ts @@ -4,6 +4,7 @@ import type { Event } from '@workflow/world'; import * as nanoid from 'nanoid'; import { monotonicFactory } from 'ulid'; import { afterEach, describe, expect, it, vi } from 'vitest'; +import { createCorrelationIdFactory } from './correlation-ids.js'; import { EventsConsumer } from './events-consumer.js'; import { WorkflowSuspension } from './global.js'; import type { WorkflowOrchestratorContext } from './private.js'; @@ -58,6 +59,10 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { }), invocationsQueue: new Map(), generateUlid: () => ulid(workflowStartedAt), + nextCorrelationId: createCorrelationIdFactory({ + specVersion: undefined, + generateUlid: () => ulid(workflowStartedAt), + }), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) ), diff --git a/packages/core/src/log-format.test.ts b/packages/core/src/log-format.test.ts index c673893f06..3b78208744 100644 --- a/packages/core/src/log-format.test.ts +++ b/packages/core/src/log-format.test.ts @@ -113,6 +113,29 @@ describe('composeLogLine', () => { `); }); + test('renders errorMessage when the message does not already carry it', () => { + // The replay-divergence warn writes its own summary line and passes the + // error only as metadata, so this is the sole place the divergent event's + // identity appears. Dropping it leaves the log naming a symptom with no + // way to tell which event diverged. + const out = composeLogLine( + PREFIX, + 'Workflow replay diverged; queueing a recovery replay before declaring the event log corrupted', + { + errorCode: 'REPLAY_DIVERGENCE', + errorMessage: + 'Replay could not consume event: eventType=step_created, correlationId=step_00000000000000000000000025.', + divergenceCount: 1, + } + ); + expect(out).toMatchInlineSnapshot(` + "[workflow-sdk] Workflow replay diverged; queueing a recovery replay before declaring the event log corrupted + code REPLAY_DIVERGENCE + error Replay could not consume event: eventType=step_created, correlationId=step_00000000000000000000000025. + divergenceCount 1" + `); + }); + test('falls back gracefully on machine names it cannot parse', () => { const out = composeLogLine(PREFIX, 'msg', { workflowRunId: 'wrun_X', @@ -156,7 +179,8 @@ describe('composeLogLine', () => { user error · Error run wrun_01ABC · myWorkflow (./workflows/x) step step_01XYZ · add (./workflows/x) - retry 4 attempts · 3 max retries" + retry 4 attempts · 3 max retries + error Transient failure" `); }); }); diff --git a/packages/core/src/log-format.ts b/packages/core/src/log-format.ts index d63e847e3f..b3daa3263c 100644 --- a/packages/core/src/log-format.ts +++ b/packages/core/src/log-format.ts @@ -36,7 +36,7 @@ export function composeLogLine( ): string { const [framing, ...rest] = message.split('\n'); const body = rest.join('\n'); - const fields = renderStructuredFields(framing ?? '', metadata); + const fields = renderStructuredFields(message, metadata); const trimmedBody = trimStackBody(body); const lines: string[] = [`${prefix} ${framing ?? ''}`]; @@ -46,19 +46,21 @@ export function composeLogLine( } function renderStructuredFields( - framing: string, + message: string, metadata: Record | undefined ): string | null { if (!metadata || Object.keys(metadata).length === 0) return null; // Drop fields that the message already encodes. We render framings and // stacks into the message string itself in step executor / combined runtime, so - // repeating them here would be pure noise. + // repeating them here would be pure noise. The whole message counts, not just + // its first line: callers that pass `${framing}\n${stack}` put the error's + // text in the stack's leading `Name: message` line. const redundant = new Set(); redundant.add('errorStack'); if ( typeof metadata.errorMessage === 'string' && - framing.includes(metadata.errorMessage as string) + message.includes(metadata.errorMessage as string) ) { redundant.add('errorMessage'); } @@ -130,6 +132,16 @@ function renderStructuredFields( lines.push(` ${kvKey('code')} ${Ansi.dim(errorCode)}`); } + // The message only duplicates the framing when the framing was built from + // the error itself (step executor, terminal run failures), and that case is + // already marked redundant above. Everywhere else — a warn that carries an + // error alongside its own summary line — this is the only place the error's + // own text appears, so dropping it loses the diagnosis. + const errorMessage = pickString(metadata, 'errorMessage'); + if (errorMessage && !redundant.has('errorMessage')) { + lines.push(` ${kvKey('error')} ${errorMessage}`); + } + const hint = pickString(metadata, 'hint'); if (hint) { lines.push(` ${Ansi.hint(hint)}`); diff --git a/packages/core/src/logger.test.ts b/packages/core/src/logger.test.ts index 5560c9e213..78066923ca 100644 --- a/packages/core/src/logger.test.ts +++ b/packages/core/src/logger.test.ts @@ -148,6 +148,7 @@ describe('logger', () => { user error · FatalError run wrun_123 step step_456 + error boom hint: Move the call to a step function.", ], ] @@ -178,7 +179,8 @@ describe('logger', () => { user error · Error run wrun_abc step step_xyz - retry 4 attempts · 3 max retries", + retry 4 attempts · 3 max retries + error Transient failure", ], ] `); diff --git a/packages/core/src/private.ts b/packages/core/src/private.ts index 16fb6a3e4e..89b2e86a61 100644 --- a/packages/core/src/private.ts +++ b/packages/core/src/private.ts @@ -4,6 +4,7 @@ import { withResolvers } from '@workflow/utils'; import type { WorldCapabilities } from '@workflow/world'; +import type { CorrelationIdFactory } from './correlation-ids.js'; import type { EventsConsumer } from './events-consumer.js'; import type { QueueItem } from './global.js'; import type { ReplayPayloadCache } from './replay-payload-cache.js'; @@ -142,6 +143,13 @@ export interface WorkflowOrchestratorContext { */ invocationsQueue: Map; onWorkflowError: (error: Error) => void; + /** + * Allocates the correlation id for a step or wait the workflow body just + * issued. Replay-stable, and the only place those ids are minted — see + * `correlation-ids.ts` for why the two kinds count separately and why + * nothing seeds them from the loaded log. + */ + nextCorrelationId: CorrelationIdFactory; generateUlid: () => string; generateNanoid: () => string; /** @@ -547,12 +555,25 @@ function hasParkedCommittedDelivery(ctx: WorkflowOrchestratorContext): boolean { * delivery still in flight. Empirically, replacing it with `queueMicrotask` * breaks hook/sleep `Promise.race` ordering (CorruptedEventLogError). */ +/** + * Whether some data delivery is still on its way to the workflow — the same + * two windows {@link scheduleWhenIdle} polls on, exposed for callers that need + * to test the condition without waiting on it. + * + * While this holds, the VM has not yet run the continuation that registers the + * next event's consumer, so "no consumer for this event" says nothing about + * whether the event log is well-formed. + */ +export function hasInFlightDelivery(ctx: WorkflowOrchestratorContext): boolean { + return ctx.pendingDeliveries > 0 || hasParkedCommittedDelivery(ctx); +} + export function scheduleWhenIdle( ctx: WorkflowOrchestratorContext, fn: () => void ): void { const check = () => { - if (ctx.pendingDeliveries > 0 || hasParkedCommittedDelivery(ctx)) { + if (hasInFlightDelivery(ctx)) { // A delivery is still hydrating, or is committed but parked behind its // deferral (whose resolve runs on a detached timer, not this queue). // Either way: let the queue drain, then re-check a timer tick later. diff --git a/packages/core/src/runtime.test.ts b/packages/core/src/runtime.test.ts index 3acbb04099..ff4c0830c9 100644 --- a/packages/core/src/runtime.test.ts +++ b/packages/core/src/runtime.test.ts @@ -6,7 +6,10 @@ import { } from '@workflow/errors'; import { type Event, + FIRST_SLOT, SPEC_VERSION_CURRENT, + SPEC_VERSION_SLOT_IDENTITY, + slotFromId, type WorkflowRun, } from '@workflow/world'; import { ulid } from 'ulid'; @@ -19,6 +22,7 @@ import { dehydrateStepReturnValue, dehydrateWorkflowArguments, } from './serialization.js'; +import { getWorkflowMetadata } from './step/get-workflow-metadata.js'; // Capture every promise handed to `waitUntil` so tests can assert that // progress-critical sends are never registered on a detached, unconsumed @@ -1523,6 +1527,17 @@ describe('workflowEntrypoint turbo mode', () => { return undefined; }); + // Records the workflow start time the step body observes, which is the one + // the synthesized run row carries under turbo. + let turboObservedStartedAt: Date | undefined; + registerStepFunction('turboMetadataStep', async () => { + turboObservedStartedAt = getWorkflowMetadata().workflowStartedAt; + return undefined; + }); + + const oneMetadataStepWorkflow = `const s = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("turboMetadataStep"); + async function workflow() { return await s(); }${xform('workflow')}`; + const oneStepWorkflow = `const s = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("turboStep"); async function workflow() { return await s(); }${xform('workflow')}`; @@ -1535,12 +1550,15 @@ describe('workflowEntrypoint turbo mode', () => { return r; }${xform('workflow')}`; - async function makeRunInput(runId: string) { + async function makeRunInput( + runId: string, + specVersion = SPEC_VERSION_CURRENT + ) { return { input: await dehydrateWorkflowArguments([], runId, undefined, []), deploymentId: 'test-deployment', workflowName: 'workflow', - specVersion: SPEC_VERSION_CURRENT, + specVersion, executionContext: {}, }; } @@ -1556,8 +1574,10 @@ describe('workflowEntrypoint turbo mode', () => { attempt: number; source: string; runStartedGate?: Promise; + specVersion?: typeof SPEC_VERSION_CURRENT; }) { const { runId, attempt, source } = opts; + const specVersion = opts.specVersion ?? SPEC_VERSION_CURRENT; const order = turboOrder; const durable: Event[] = []; let seq = 0; @@ -1576,6 +1596,7 @@ describe('workflowEntrypoint turbo mode', () => { runId, workflowName: 'workflow', status: 'running', + specVersion, input: await dehydrateWorkflowArguments([], runId, undefined, []), createdAt: new Date('2024-01-01T00:00:00.000Z'), updatedAt: new Date('2024-01-01T00:00:00.000Z'), @@ -1621,7 +1642,7 @@ describe('workflowEntrypoint turbo mode', () => { }); setWorld({ - specVersion: SPEC_VERSION_CURRENT, + specVersion, createQueueHandler: vi.fn( (_p: string, handler: (m: unknown, md: unknown) => Promise) => async () => { @@ -1629,7 +1650,7 @@ describe('workflowEntrypoint turbo mode', () => { { runId, requestedAt: new Date('2024-01-01T00:00:00.000Z'), - runInput: await makeRunInput(runId), + runInput: await makeRunInput(runId, specVersion), }, { requestId: 'req_turbo', @@ -1715,6 +1736,33 @@ describe('workflowEntrypoint turbo mode', () => { ); }); + it('claims slots above the run own positions on a first delivery', async () => { + // Turbo replays against an empty snapshot, so the log the claims are + // numbered from cannot show `run_created` or the in-flight `run_started`. + // Both positions are nonetheless taken, and the mocked `run_started` + // response reports no event — the same shape as a World that skips the + // preload — so nothing but the floor seeded at turbo entry keeps the first + // batch of claims off them. + const { handlerPromise, eventsCreate } = await driveTurbo({ + runId: 'wrun_turbo_slots', + attempt: 1, + source: stepAndSleepWorkflow, + specVersion: SPEC_VERSION_SLOT_IDENTITY, + }); + + const res = await handlerPromise; + expect(res.status).toBe(204); + + const claimed = eventsCreate.mock.calls + .map((c) => (c[2] as { eventId?: unknown } | undefined)?.eventId) + .filter((id): id is string => typeof id === 'string'); + // The sleep's `wait_created` is claimed, so there is something to assert on. + expect(claimed.length).toBeGreaterThan(0); + for (const eventId of claimed) { + expect(slotFromId(eventId)).toBeGreaterThan(FIRST_SLOT + 1); + } + }); + it('does not turbo when WORKFLOW_TURBO=0 (parity with the awaited path)', async () => { process.env.WORKFLOW_TURBO = '0'; const { handlerPromise, order } = await driveTurbo({ @@ -1761,6 +1809,30 @@ describe('workflowEntrypoint turbo mode', () => { expect((redeliverRunStarted?.[2] as any)?.skipPreload).toBeUndefined(); }); + it('sends run_started the same instant it synthesizes the run from', async () => { + // Turbo starts the run against a locally synthesized run row, so the start + // time this invocation reports comes from the client clock. Backends that + // persist `occurredAt` record the run's `startedAt` from it, so sending it + // is what makes a later replay — which reads the persisted run — report the + // same `workflowStartedAt` this pass already captured into its steps. + turboObservedStartedAt = undefined; + const { handlerPromise, eventsCreate } = await driveTurbo({ + runId: 'wrun_turbo_occurred_at', + attempt: 1, + source: oneMetadataStepWorkflow, + specVersion: SPEC_VERSION_SLOT_IDENTITY, + }); + expect((await handlerPromise).status).toBe(204); + + const runStarted = eventsCreate.mock.calls.find( + (c) => (c[1] as any).eventType === 'run_started' + ); + const occurredAt = (runStarted?.[2] as { occurredAt?: Date } | undefined) + ?.occurredAt; + expect(occurredAt).toBeInstanceOf(Date); + expect(turboObservedStartedAt).toEqual(occurredAt); + }); + it('exits turbo (no forced optimistic) when the suspension creates a wait', async () => { const { handlerPromise, order } = await driveTurbo({ runId: 'wrun_turbo_wait', diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 12eb01272a..5e711b9182 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -4,7 +4,6 @@ import { EntityConflictError, FatalError, MaxEventsExceededError, - PreconditionFailedError, ReplayDivergenceError, RUN_ERROR_CODES, type RunErrorCode, @@ -18,12 +17,15 @@ import { } from '@workflow/utils/parse-name'; import { type Event, + FIRST_SLOT, getQueueTopicPrefix, isLegacySpecVersion, ROOT_RUN_ID_ATTRIBUTE, resolveQueueNamespace, SPEC_VERSION_CURRENT, SPEC_VERSION_SUPPORTS_COMPRESSION, + slotFromId, + usesSlotIdentity, WorkflowInvokePayloadSchema, type WorkflowRun, type World, @@ -50,19 +52,22 @@ import { countStepStartedEvents } from './runtime/count-step-started-events.js'; import { appendUniqueEvents, type EventCreator, + eventCreateFenceFor, getQueueOverhead, getWorkflowQueueName, handleHealthCheckMessage, isPreconditionGuardEnabled, loadWorkflowRunEvents, - type MutableEventLog, memoizeEncryptionKey, parseHealthCheckPayload, queueMessage, - stateUpdatedAtForCreate, + requiresFreshReplay, + stepClaimFence, + toMutableEventLog, + withEventCreateFence, withHealthCheck, - withPreconditionRetry, } from './runtime/helpers.js'; +import { runScopedKey } from './runtime/idempotency-key.js'; import { handleReplayBudgetExhausted, ReplayBudget, @@ -556,6 +561,19 @@ export function workflowEntrypoint( let workflowStartedAt = -1; let preloadedEvents: Event[] | undefined; let preloadedEventsCursor: string | null | undefined; + // Highest slot known to be published on a slot-numbered run, + // for the writes whose snapshot cannot show it: turbo + // backgrounds `run_started` and replays against an empty log, + // so a claim numbered from that log alone would propose a slot + // `run_started` already holds. 0 when the run is not + // slot-numbered — its ids carry no position to compare. + let knownSlotFloor = 0; + const observeSlotFloor = (eventId: string | undefined) => { + const slot = eventId ? slotFromId(eventId) : undefined; + if (slot !== undefined && slot > knownSlotFloor) { + knownSlotFloor = slot; + } + }; // Latency telemetry (TTFS) state — see runtime/step-latency.ts. // Whether this invocation's FIRST event snapshot contained @@ -1007,6 +1025,16 @@ export function workflowEntrypoint( // handler, optimistic step_started, terminal run writes) so // nothing is written before the run exists. recordRunStartedCreateStart(true); + // The instant this invocation calls the run started, sent + // with the event and reused for the synthesized run row + // below. Backends that persist `occurredAt` record the + // run's `startedAt` from it, which is what keeps + // `workflowStartedAt` identical between this optimistic + // pass and every later replay that reads the persisted + // run. Without it the two disagree by the round-trip, and + // a step's captured metadata no longer matches the + // workflow's on the next replay. + const now = new Date(); const startedPromise = createEvent( runStartedEvent, // We background this purely as a write barrier and @@ -1016,7 +1044,7 @@ export function workflowEntrypoint( // run_started request the chained first step_started // waits on — shortening time-to-second-step — and the // wasted list+resolve it would otherwise compute. - { requestId, skipPreload: true } + { requestId, skipPreload: true, occurredAt: now } ); runReadyBarrier = startedPromise; // Turbo backgrounds run_started, so the non-turbo assignment @@ -1029,6 +1057,10 @@ export function workflowEntrypoint( (r) => { const limit = clampMaxEvents(r?.maxEvents); if (limit !== undefined) maxEventsLimit = limit; + // Every write of this invocation is ordered after this + // promise by `runReadyBarrier`, so the slot it reports + // is in hand before the first claim is numbered. + observeSlotFloor(r?.event?.eventId); }, () => {} ); @@ -1045,7 +1077,20 @@ export function workflowEntrypoint( // intentionally truthy here — do not change the load // branches' `if (preloadedEvents)` checks to test length. preloadedEvents = []; - const now = new Date(); + // A slot-numbered run's first two positions are the run's + // own: `run_created` from start(), then the `run_started` + // in flight above. Both are certain before any write of + // this invocation, and turbo replays against the empty + // snapshot skipped just above — so seed the floor with + // them here rather than waiting for the backgrounded + // response to report it. Waiting loses the race: a + // suspension reserves its whole batch of positions + // synchronously, so a batch that starts numbering from an + // empty log claims the two the run already holds and the + // ops holding them lose their claims. + if (usesSlotIdentity(runInput.specVersion)) { + knownSlotFloor = FIRST_SLOT + 1; + } workflowRun = { runId, status: 'running', @@ -1094,6 +1139,7 @@ export function workflowEntrypoint( } workflowRun = result.run; maxEventsLimit = clampMaxEvents(result.maxEvents); + observeSlotFloor(result.event?.eventId); // Anchors RSFS — see the declaration above. runStartedReceivedAtMs = Date.now(); @@ -1369,19 +1415,25 @@ export function workflowEntrypoint( }, })); + // One log for the whole loop: `events` is appended to in + // place by the guard's reloads, so a per-iteration + // rescan for the slot high-water mark would be wasted + // work on an array that never changes identity here. + const waitLog = toMutableEventLog( + events, + eventsCursor, + knownSlotFloor + ); for (const waitEvent of waitsToComplete) { - const waitLog: MutableEventLog = { - events, - cursor: eventsCursor, - }; try { - await withPreconditionRetry( + await withEventCreateFence( runId, waitLog, - (stateUpdatedAt) => + workflowRun.specVersion, + (fence) => createEvent(waitEvent, { requestId, - stateUpdatedAt, + ...fence, }) ); } catch (err) { @@ -1495,6 +1547,20 @@ export function workflowEntrypoint( // point and the inline executeStep mutates eventsCursor. preInlineWriteCursor = eventsCursor; + // One log for everything this replay writes on its way to + // a terminal event: the end-of-run drain inside + // `runWorkflow` (fire-and-forget `*_created` events, and + // the implicit disposal of the abort hooks a completing + // run leaves behind) and the `run_completed` / + // `run_failed` write below. Sharing it is what keeps the + // two from claiming the same slot — the terminal write + // numbers from a snapshot that predates the drain. + const replayWriteLog = toMutableEventLog( + events, + eventsCursor, + knownSlotFloor + ); + // Replay workflow runtimeLogger.debug('Starting workflow replay', { workflowRunId: runId, @@ -1520,7 +1586,8 @@ export function workflowEntrypoint( // `awaitRunReady()` below, so gate those writes on the // backgrounded run_started too. Undefined outside turbo. runReadyBarrier, - world.capabilities + world.capabilities, + replayWriteLog ); await payloadPrewarm; runtimeLogger.debug('Workflow replay completed', { @@ -1532,11 +1599,12 @@ export function workflowEntrypoint( // Workflow completed. Send the snapshot but do NOT // reload-and-retry the create in place: `result` was - // computed by this replay, so a stale (412) rejection must - // force a *fresh replay* (which may observe the new event - // and produce a different result), not re-commit the stale - // result. The catch below lets PreconditionFailedError - // propagate to the queue for re-invocation. + // computed by this replay, so a rejection proving the view + // was incomplete must force a *fresh replay* (which may + // observe the new event and produce a different result), + // not re-commit the stale result. The catch below lets + // `requiresFreshReplay` rejections propagate to the queue + // for re-invocation. try { // Turbo: a workflow that finishes with no steps reaches // here before the backgrounded run_started; order the @@ -1550,7 +1618,10 @@ export function workflowEntrypoint( }, { requestId, - stateUpdatedAt: stateUpdatedAtForCreate(events), + ...eventCreateFenceFor( + replayWriteLog, + workflowRun.specVersion + ), } ); } catch (err) { @@ -1619,10 +1690,11 @@ export function workflowEntrypoint( } // V2: handle suspension without queuing steps. - // Each event creation inside handleSuspension carries the - // loaded snapshot's stateUpdatedAt and self-reloads on a - // stale (412) rejection via the shared event log. We - // guard per-create (rather than wrapping the whole call) + // Each event creation inside handleSuspension carries + // the run's concurrency fence — its own event slot, or + // the loaded snapshot's watermark — and self-reloads on + // a rejection (409/412) via the shared event log. We + // fence per-create (rather than wrapping the whole call) // so a retry never re-issues an already-created event. const suspensionStart = Date.now(); // The snapshot refresh above always sets cachedEvents @@ -1636,10 +1708,11 @@ export function workflowEntrypoint( 'Invariant violation: workflow suspended before its event log was loaded' ); } - const suspensionLog: MutableEventLog = { - events: cachedEvents, - cursor: eventsCursor, - }; + const suspensionLog = toMutableEventLog( + cachedEvents, + eventsCursor, + knownSlotFloor + ); let suspensionResult: Awaited< ReturnType >; @@ -1655,13 +1728,14 @@ export function workflowEntrypoint( replayRecoveryReporter, }); } catch (suspensionError) { - // A suspension create whose stale (412) rejection - // survived the in-guard reload retries: schedule an + // A suspension create whose incomplete-view rejection + // (412 stale watermark, or 409 taken slot) survived + // the in-guard reload retries: schedule an // explicit immediate re-invocation (a rethrow relies // on redelivery of a message the turbo path already // acked — the run would stall for the queue's ~300s // default visibility timeout). - if (PreconditionFailedError.is(suspensionError)) { + if (requiresFreshReplay(suspensionError)) { runtimeLogger.warn( 'Suspension event creation rejected as stale after reload retries; re-invoking run for a fresh replay', { workflowRunId: runId, loopIteration } @@ -1942,7 +2016,10 @@ export function workflowEntrypoint( }, { delaySeconds: backstopDelaySeconds, - idempotencyKey: backstopIdempotencyKey(step), + idempotencyKey: backstopIdempotencyKey( + runId, + step + ), } ) ); @@ -1960,7 +2037,10 @@ export function workflowEntrypoint( requestedAt: new Date(), }, { - idempotencyKey: step.correlationId, + idempotencyKey: runScopedKey( + runId, + step.correlationId + ), } ) ); @@ -1976,6 +2056,7 @@ export function workflowEntrypoint( requestedAt: new Date(), }, getWaitContinuationDispatch( + runId, suspensionResult.waitTimeout.seconds, suspensionResult.waitTimeout.correlationId ) @@ -2114,11 +2195,15 @@ export function workflowEntrypoint( // rejected with 412 — its guarded suspension creates // (retried over the reloaded log, or exhausted into // a queue re-invocation), AND the lazy step_started - // claim of its next inline step, which carries the - // snapshot too (threaded below via - // `stateUpdatedAt`; on rejection the batch is - // abandoned and re-invoked for a fresh replay, so a - // stale view can never commit a step). Hooks created + // claim of its next inline step, which is fenced too + // (threaded below via `stepClaimFence`; on rejection + // the batch is abandoned and re-invoked for a fresh + // replay, so a stale view can never commit a step). + // A slot-numbered run gets there differently — the + // claim merges the missed events and retries in + // place, so the same events are observed without + // discarding the batch. See stepClaimFence. + // Hooks created // by THIS suspension are inside the delta (their // `hook_created` lands before the step-terminal // write), so only their `hook_received` responses @@ -2258,18 +2343,23 @@ export function workflowEntrypoint( turbo, }); - // Precondition-guard snapshot for the inline - // step_started claims: the lazy claim is the first - // durable write of a hot-path step (its step_created - // is deferred), so without a snapshot it would bypass - // the guard entirely and a stale replay could claim — - // and commit — a step scheduled off a view that misses - // an out-of-band event. `stateUpdatedAtForCreate` - // returns undefined when the guard env flag is off, so - // this is a no-op outside guarded deployments; Worlds - // that don't enforce the guard ignore it. - const inlineClaimStateUpdatedAt = - stateUpdatedAtForCreate(cachedEvents ?? []); + // Concurrency fence for the inline step_started claims: + // the lazy claim is the first durable write of a + // hot-path step (its step_created is deferred), so + // without one it would be unguarded and a stale replay + // could claim — and commit — a step scheduled off a view + // that misses an out-of-band event. One log for the + // whole batch so each claim draws its own event slot; + // `stepClaimFence` leaves the claims of a run fenced + // neither way exactly as they were. + // + // The suspension's own log, not a second one over the + // same snapshot: its reservations are what the hook and + // wait creates just above took, and those events are not + // in `cachedEvents` yet. A fresh log would number these + // claims from the same base and hand the batch's first + // step a slot the suspension already holds. + const inlineClaimLog = suspensionLog; replayBudget.pause(); let stepResults: Awaited< @@ -2277,6 +2367,27 @@ export function workflowEntrypoint( >[]; const stepExecutionPromises = inlineExecutions.map( (s, stepIndex) => { + // Drawn here — synchronously, in replay order — + // rather than inside `run`: a slot claim is + // positional, so it has to be assigned before these + // executions start racing each other, and the order + // it is assigned in has to be replay-stable. + const claimFence = stepClaimFence( + runId, + inlineClaimLog, + workflowRun.specVersion, + { + // A lazy start publishes two events: the World + // writes the step's deferred `step_created` + // alongside the claim, so the batch has to + // reserve a slot for that one too — otherwise + // it lands on the slot the next start in the + // batch is holding and costs that start its + // claim. + extraEvents: + s.lazyStepInput !== undefined ? 1 : 0, + } + ); const run = () => executeStep({ world, @@ -2348,7 +2459,7 @@ export function workflowEntrypoint( // see suppressOptimisticStart above. suppressOptimisticStart, runReadyBarrier, - stateUpdatedAt: inlineClaimStateUpdatedAt, + claimFence, ...(stepIndex === 0 && s.lazyStepInput !== undefined && latencyTracking @@ -2390,21 +2501,47 @@ export function workflowEntrypoint( stepExecutionPromises ); } catch (stepErr) { - // A stale (412) rejection of an inline step_started - // claim: the loaded view this batch was scheduled - // from is behind an out-of-band event (e.g. a - // received hook), so the claim was fenced by the - // guard and no step events were written. Abandon the - // batch — any optimistic body result is discarded by + // An incomplete-view rejection of an inline + // step_started claim: the loaded view this batch was + // scheduled from is behind an out-of-band event (e.g. + // a received hook), so the claim was fenced and no + // step events were written. Under the watermark that + // is every 412; under slot identity `stepClaimFence` + // first merges the missed events and re-claims in + // place, so a 409 only arrives here once those + // retries are exhausted. Abandon the batch — any + // optimistic body result is discarded by // executeStep's reconciliation — and re-invoke for a // fresh replay that observes the new event. Wait for // the sibling executions to settle first so no owned // body is in flight when the ack path runs. - if (PreconditionFailedError.is(stepErr)) { - await Promise.allSettled(stepExecutionPromises); + if (requiresFreshReplay(stepErr)) { + const settled = await Promise.allSettled( + stepExecutionPromises + ); runtimeLogger.warn( 'Inline step claim rejected as stale; re-invoking run for a fresh replay', - { workflowRunId: runId, loopIteration } + { + workflowRunId: runId, + loopIteration, + // Which members of the batch were fenced and + // which committed. A fence is per-write, so a + // batch can be split: the rejected claim wrote + // nothing, but a sibling holding a different + // slot may have committed its step. That + // asymmetry is the shape to look for when a + // later replay cannot consume a step event. + batchSteps: inlineExecutions + .map( + (s, i) => + `${s.correlationId}:${settled[i]?.status === 'rejected' ? 'rejected' : 'settled'}` + ) + .join(', '), + errorMessage: + stepErr instanceof Error + ? stepErr.message + : String(stepErr), + } ); // The finally below resumes the replay budget // before this return completes. @@ -2517,7 +2654,7 @@ export function workflowEntrypoint( // correlationId so it dedupes against the // keyed re-dispatch the suspension handler // performs on replay (it also uses - // `idempotencyKey: step.correlationId`). + // the same run-scoped correlationId key). // // Without this, a mixed batch where one step // `completed` with unflushed background ops @@ -2536,7 +2673,10 @@ export function workflowEntrypoint( // retry body could run early/concurrently. // Sharing the key lets the earlier delayed // message win, honoring the backoff. - idempotencyKey: step.correlationId, + idempotencyKey: runScopedKey( + runId, + step.correlationId + ), } ) ) @@ -2594,17 +2734,18 @@ export function workflowEntrypoint( } } } else { - // Stale-snapshot rejection of a result-bearing create - // (run_completed sends the snapshot but is intentionally - // NOT retried in place), or one that survived the - // in-guard reload retries. Don't fail the run — schedule - // an explicit immediate re-invocation so a fresh replay - // observes the new event. Rethrowing instead would rely - // on redelivery of the CURRENT message, which the turbo - // path has already acked — empirically the run then - // stalls for the queue's ~300s default visibility - // timeout before completing. - if (PreconditionFailedError.is(err)) { + // Incomplete-view rejection of a result-bearing create — + // a stale watermark (412) or a taken slot (409), either + // on `run_completed` (which sends its fence but is + // intentionally NOT retried in place) or on a create + // that survived the in-guard reload retries. Don't fail + // the run — schedule an explicit immediate re-invocation + // so a fresh replay observes the new event. Rethrowing + // instead would rely on redelivery of the CURRENT + // message, which the turbo path has already acked — + // empirically the run then stalls for the queue's ~300s + // default visibility timeout before completing. + if (requiresFreshReplay(err)) { runtimeLogger.warn( 'Event creation rejected as stale; re-invoking run for a fresh replay', { workflowRunId: runId, loopIteration } @@ -2740,10 +2881,10 @@ export function workflowEntrypoint( // type identity and custom properties round-trip // through the event log. // - // Precondition-guard asymmetry: unlike `run_completed`, - // this terminal `run_failed` sends no `stateUpdatedAt` - // snapshot, so it is never 412-rejected even if a hook - // landed mid-replay and could have changed the path that + // Fencing asymmetry: unlike `run_completed`, this + // terminal `run_failed` carries no concurrency fence, so + // it is never rejected even if a hook landed mid-replay + // and could have changed the path that // threw. This is intentional and fail-open: a spurious // failure is recoverable (the run can be re-run from the // dashboard), whereas a spurious *completion* commits a diff --git a/packages/core/src/runtime/helpers.test.ts b/packages/core/src/runtime/helpers.test.ts index 09071533e4..9cf79dcdfc 100644 --- a/packages/core/src/runtime/helpers.test.ts +++ b/packages/core/src/runtime/helpers.test.ts @@ -1,5 +1,15 @@ -import { PreconditionFailedError, WorkflowWorldError } from '@workflow/errors'; -import type { Event, World } from '@workflow/world'; +import { + EntityConflictError, + PreconditionFailedError, + SlotConflictError, + WorkflowWorldError, +} from '@workflow/errors'; +import { + type Event, + SPEC_VERSION_SLOT_IDENTITY, + slotEventId, + type World, +} from '@workflow/world'; import { ulid } from 'ulid'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { bytesToBase64, deriveRunKeyPair, seal } from '../sealed-box.js'; @@ -11,6 +21,7 @@ import { SerializationFormat, } from '../serialization.js'; import { + eventCreateFenceFor, getWorkflowQueueName, handleHealthCheckMessage, healthCheck, @@ -18,7 +29,16 @@ import { loadWorkflowRunEvents, type MutableEventLog, memoizeEncryptionKey, + mergeLoadedEvents, + PRECONDITION_MAX_RELOAD_RETRIES, + requiresFreshReplay, + reserveSlot, + stateUpdatedAtForCreate, + stepClaimFence, + toMutableEventLog, + withEventCreateFence, withPreconditionRetry, + withSlotRetry, } from './helpers.js'; // Mock the logger to suppress output during tests @@ -503,6 +523,464 @@ describe('latestEventStateUpdatedAt', () => { }); }); +describe('slot bookkeeping', () => { + const slotEvent = (slot: number) => makeEvent(slotEventId(slot)); + + it('reads maxSlot from the highest slot present, not the last element', () => { + // `appendUniqueEvents` appends without sorting, so a merged log's last + // element is not necessarily its newest event. + const log = toMutableEventLog([slotEvent(3), slotEvent(1)], 'c0'); + expect(log.maxSlot).toBe(3); + expect(log.nextSlot).toBe(4); + }); + + it('reports maxSlot 0 for an empty or ULID-numbered log', () => { + expect(toMutableEventLog([], null).maxSlot).toBe(0); + expect( + toMutableEventLog([makeUlidEvent(1_700_000_000_000)], null).maxSlot + ).toBe(0); + }); + + it('starts at a floor the snapshot cannot show', () => { + // Turbo replays against an empty log while its `run_started` write is still + // in flight, so the snapshot alone would number the first claim onto a slot + // that write already holds. + const log = toMutableEventLog([], null, 2); + expect(log.maxSlot).toBe(2); + expect(reserveSlot(log)).toBe(3); + }); + + it('ignores a floor the snapshot has already passed', () => { + const log = toMutableEventLog([slotEvent(5)], 'c0', 2); + expect(log.maxSlot).toBe(5); + }); + + it('keeps the floor across a merge', () => { + const log = toMutableEventLog([], null, 2); + mergeLoadedEvents(log, [slotEvent(1)]); + expect(log.maxSlot).toBe(2); + }); + + it('never lowers maxSlot when an older delta is merged in', () => { + const log = toMutableEventLog([slotEvent(1), slotEvent(3)], 'c0'); + mergeLoadedEvents(log, [slotEvent(2)]); + expect(log.maxSlot).toBe(3); + expect(log.events).toHaveLength(3); + }); + + it('raises the reservation pointer past a newer delta', () => { + const log = toMutableEventLog([slotEvent(1)], 'c0'); + reserveSlot(log); + reserveSlot(log); + expect(log.nextSlot).toBe(4); + + mergeLoadedEvents(log, [slotEvent(2), slotEvent(5)]); + + expect(log.maxSlot).toBe(5); + expect(reserveSlot(log)).toBe(6); + }); + + it('never rewinds the reservation pointer onto an outstanding slot', () => { + // A writer that loses its slot merges the delta and reserves again while + // its siblings are still in flight on theirs. Rewinding to `maxSlot + 1` + // would hand it slot 4, which a sibling already holds. + const log = toMutableEventLog([slotEvent(1)], 'c0'); + expect(reserveSlot(log)).toBe(2); + expect(reserveSlot(log)).toBe(3); + expect(reserveSlot(log)).toBe(4); + + mergeLoadedEvents(log, [slotEvent(2)]); + + expect(log.maxSlot).toBe(2); + expect(reserveSlot(log)).toBe(5); + }); + + it('deduplicates merged events by id', () => { + const log = toMutableEventLog([slotEvent(1)], 'c0'); + mergeLoadedEvents(log, [slotEvent(1), slotEvent(2)]); + expect(log.events.map((e) => e.eventId)).toEqual([ + slotEventId(1), + slotEventId(2), + ]); + }); + + it('hands out contiguous distinct slots for a synchronous burst', () => { + // The suspension flush issues every operation synchronously and awaits + // them together; without contiguous reservation they would all propose the + // same slot and all but one would conflict. + const log = toMutableEventLog([slotEvent(4)], 'c0'); + const burst = Array.from({ length: 20 }, () => reserveSlot(log)); + expect(burst).toEqual(Array.from({ length: 20 }, (_, i) => 5 + i)); + expect(new Set(burst).size).toBe(burst.length); + // Reservations sit past maxSlot rather than moving it: only merged events + // prove a slot is taken. + expect(log.maxSlot).toBe(4); + }); + + it('proposes a padded event id only for a slot-identity run', () => { + const log = toMutableEventLog([], null); + expect(eventCreateFenceFor(log, SPEC_VERSION_SLOT_IDENTITY)).toEqual({ + eventId: slotEventId(1), + maxSlot: 0, + }); + expect(eventCreateFenceFor(log, SPEC_VERSION_SLOT_IDENTITY)).toEqual({ + eventId: slotEventId(2), + maxSlot: 0, + }); + }); + + it('reserves a slot per extra event and names the top one', () => { + // A lazy inline `step_started` publishes two events: the World also writes + // the `step_created` it deferred, which takes the slot below the claim. + // Reserving it here is what keeps it off the slot the next write of the + // same batch will claim. + const log = toMutableEventLog([slotEvent(1)], null); + expect( + eventCreateFenceFor(log, SPEC_VERSION_SLOT_IDENTITY, { extraEvents: 1 }) + ).toEqual({ eventId: slotEventId(3), maxSlot: 1 }); + expect( + eventCreateFenceFor(log, SPEC_VERSION_SLOT_IDENTITY, { extraEvents: 1 }) + ).toEqual({ eventId: slotEventId(5), maxSlot: 1 }); + // A single-event write in the same batch still gets the next free slot. + expect(eventCreateFenceFor(log, SPEC_VERSION_SLOT_IDENTITY)).toEqual({ + eventId: slotEventId(6), + maxSlot: 1, + }); + }); + + it('burns no slot on an extra event of a ULID-numbered run', () => { + const log = toMutableEventLog([], null); + eventCreateFenceFor(log, SPEC_VERSION_SLOT_IDENTITY - 1, { + extraEvents: 1, + }); + expect(log.nextSlot).toBe(1); + }); + + it('proposes no event id for a ULID-numbered run', () => { + // A run whose ids the backend mints must not burn slots either. + const log = toMutableEventLog([], null); + const fence = eventCreateFenceFor(log, SPEC_VERSION_SLOT_IDENTITY - 1); + expect(fence?.eventId).toBeUndefined(); + expect(log.nextSlot).toBe(1); + }); +}); + +describe('stateUpdatedAtForCreate', () => { + it('sends no watermark for a slot-numbered run even with the guard on', () => { + // The event id is the fence for these runs. Inference would be worse than + // useless here: a padded slot body is valid Crockford base32, so decoding + // it yields epoch 0 rather than failing, and the client would claim a + // snapshot older than every event in the log. + const events = [makeEvent(slotEventId(1))]; + expect( + stateUpdatedAtForCreate(events, SPEC_VERSION_SLOT_IDENTITY) + ).toBeUndefined(); + }); + + it('sends the snapshot watermark for a ULID-numbered run', () => { + const time = 1_700_000_000_000; + expect( + stateUpdatedAtForCreate( + [makeUlidEvent(time)], + SPEC_VERSION_SLOT_IDENTITY - 1 + ) + ).toBe(time); + }); +}); + +describe('withSlotRetry', () => { + const slotEvent = (slot: number) => makeEvent(slotEventId(slot)); + + beforeEach(() => { + eventsListMock.mockReset(); + }); + + it('claims the next free slot and passes the observed maxSlot alongside it', async () => { + const log = toMutableEventLog([slotEvent(1), slotEvent(2)], 'c0'); + const op = vi.fn(async () => 'ok'); + + await expect(withSlotRetry('wrun_test', log, op)).resolves.toBe('ok'); + expect(op).toHaveBeenCalledWith({ eventId: slotEventId(3), maxSlot: 2 }); + expect(eventsListMock).not.toHaveBeenCalled(); + }); + + it('merges the conflict delta and reclaims past it, without reloading', async () => { + // The inline delta is the whole point of the 409 body: the client learns + // which events it was missing without a follow-up round-trip. + const log = toMutableEventLog([slotEvent(1)], 'c0'); + const claimed: string[] = []; + const op = vi.fn(async ({ eventId }: { eventId?: string }) => { + claimed.push(eventId as string); + if (claimed.length === 1) { + throw new SlotConflictError('taken', { + eventId: eventId as string, + events: [slotEvent(2), slotEvent(3)], + cursor: 'c1', + }); + } + return 'done'; + }); + + await expect(withSlotRetry('wrun_test', log, op)).resolves.toBe('done'); + expect(claimed).toEqual([slotEventId(2), slotEventId(4)]); + expect(log.events).toHaveLength(3); + expect(log.cursor).toBe('c1'); + expect(eventsListMock).not.toHaveBeenCalled(); + }); + + it('tops the delta up from the backend when it was truncated', async () => { + const log = toMutableEventLog([slotEvent(1)], 'c0'); + eventsListMock.mockResolvedValueOnce({ + data: [slotEvent(3)], + cursor: 'c2', + hasMore: false, + }); + let attempts = 0; + const op = vi.fn(async () => { + attempts++; + if (attempts === 1) { + throw new SlotConflictError('taken', { + eventId: slotEventId(2), + events: [slotEvent(2)], + cursor: 'c1', + hasMore: true, + }); + } + return 'done'; + }); + + await expect(withSlotRetry('wrun_test', log, op)).resolves.toBe('done'); + expect(eventsListMock).toHaveBeenCalledTimes(1); + expect(log.maxSlot).toBe(3); + expect(op).toHaveBeenLastCalledWith({ + eventId: slotEventId(4), + maxSlot: 3, + }); + }); + + it('falls back to a full incremental load when the rejection carried no delta', async () => { + const log = toMutableEventLog([slotEvent(1)], 'c0'); + eventsListMock.mockResolvedValueOnce({ + data: [slotEvent(2)], + cursor: 'c1', + hasMore: false, + }); + let attempts = 0; + const op = vi.fn(async () => { + attempts++; + if (attempts === 1) { + throw new SlotConflictError('taken', { eventId: slotEventId(2) }); + } + return 'done'; + }); + + await expect(withSlotRetry('wrun_test', log, op)).resolves.toBe('done'); + expect(eventsListMock).toHaveBeenCalledTimes(1); + expect(op).toHaveBeenLastCalledWith({ + eventId: slotEventId(3), + maxSlot: 2, + }); + }); + + it('rethrows the conflict once the reclaim budget is spent', async () => { + // Escaping to a fresh replay is the correct fallback, not a failure mode: + // the merged events can change what the workflow body decides, and only a + // replay from the top can act on that. + const log = toMutableEventLog([slotEvent(1)], 'c0'); + eventsListMock.mockResolvedValue({ + data: [], + cursor: 'c1', + hasMore: false, + }); + const op = vi.fn(async ({ eventId }: { eventId?: string }) => { + throw new SlotConflictError('taken', { eventId: eventId as string }); + }); + + await expect(withSlotRetry('wrun_test', log, op)).rejects.toBeInstanceOf( + SlotConflictError + ); + expect(op).toHaveBeenCalledTimes(PRECONDITION_MAX_RELOAD_RETRIES + 1); + expect(eventsListMock).toHaveBeenCalledTimes( + PRECONDITION_MAX_RELOAD_RETRIES + ); + }); + + it('rethrows a non-conflict error immediately, without merging', async () => { + const log = toMutableEventLog([slotEvent(1)], 'c0'); + const op = vi.fn(async () => { + throw new PreconditionFailedError('stale'); + }); + + await expect(withSlotRetry('wrun_test', log, op)).rejects.toBeInstanceOf( + PreconditionFailedError + ); + expect(op).toHaveBeenCalledTimes(1); + expect(eventsListMock).not.toHaveBeenCalled(); + }); +}); + +describe('withEventCreateFence', () => { + beforeEach(() => { + eventsListMock.mockReset(); + }); + + it('fences a slot-numbered run by event id and retries its 409s', async () => { + const log = toMutableEventLog([makeEvent(slotEventId(1))], 'c0'); + let attempts = 0; + const op = vi.fn(async ({ eventId }: { eventId?: string }) => { + attempts++; + if (attempts === 1) { + throw new SlotConflictError('taken', { + eventId: eventId as string, + events: [makeEvent(slotEventId(2))], + cursor: 'c1', + }); + } + return 'done'; + }); + + await expect( + withEventCreateFence('wrun_test', log, SPEC_VERSION_SLOT_IDENTITY, op) + ).resolves.toBe('done'); + expect(op).toHaveBeenLastCalledWith({ + eventId: slotEventId(3), + maxSlot: 2, + }); + }); + + it('fences a ULID-numbered run by watermark and retries its 412s', async () => { + const time = 1_700_000_000_000; + const log = toMutableEventLog([makeUlidEvent(time)], 'c0'); + eventsListMock.mockResolvedValueOnce({ + data: [makeUlidEvent(time + 1000)], + cursor: 'c1', + hasMore: false, + }); + let attempts = 0; + const op = vi.fn(async () => { + attempts++; + if (attempts === 1) { + throw new PreconditionFailedError('stale'); + } + return 'done'; + }); + + await expect( + withEventCreateFence('wrun_test', log, SPEC_VERSION_SLOT_IDENTITY - 1, op) + ).resolves.toBe('done'); + expect(op).toHaveBeenLastCalledWith({ stateUpdatedAt: time + 1000 }); + expect(eventsListMock).toHaveBeenCalledTimes(1); + }); +}); + +describe('stepClaimFence', () => { + const slotEvent = (slot: number) => makeEvent(slotEventId(slot)); + + beforeEach(() => { + eventsListMock.mockReset(); + }); + + it('numbers a batch in the order its claims were built, not the order they fire', async () => { + // The batch is built during replay and only starts racing afterwards, so + // the slot of each member has to be fixed at build time for the numbering + // to be replay-stable. + const log = toMutableEventLog([slotEvent(1)], 'c0'); + const first = stepClaimFence('wrun_test', log, SPEC_VERSION_SLOT_IDENTITY, { + extraEvents: 1, + }); + const second = stepClaimFence('wrun_test', log, SPEC_VERSION_SLOT_IDENTITY); + + const claimed: (string | undefined)[] = []; + const record = (fence?: { eventId?: string }) => { + claimed.push(fence?.eventId); + return Promise.resolve('ok'); + }; + // Fired in reverse: the numbering must not depend on it. + await second(record); + await first(record); + + expect(claimed).toEqual([slotEventId(4), slotEventId(3)]); + }); + + it('reclaims a lost slot in place, keeping the batch adjacent', async () => { + // The server allocates an outside event from the same next-free pointer + // the client reserves from, so losing a claim is routine. Abandoning the + // batch on it would leave this step's events far later in the log than its + // siblings' — an order no single replay can consume — and leave the lost + // slot permanently empty. + const log = toMutableEventLog([slotEvent(1)], 'c0'); + const claim = stepClaimFence('wrun_test', log, SPEC_VERSION_SLOT_IDENTITY); + const sibling = stepClaimFence( + 'wrun_test', + log, + SPEC_VERSION_SLOT_IDENTITY + ); + + const claimed: string[] = []; + const op = vi.fn(async (fence?: { eventId?: string }) => { + claimed.push(fence?.eventId as string); + if (claimed.length === 1) { + // An out-of-band hook took slot 2 while the batch was being built. + throw new SlotConflictError('taken', { + eventId: fence?.eventId as string, + events: [slotEvent(2)], + cursor: 'c1', + }); + } + return 'done'; + }); + + await expect(claim(op)).resolves.toBe('done'); + await expect(sibling(async (f) => f?.eventId)).resolves.toBe( + slotEventId(3) + ); + // Reclaimed above the merged event rather than propagating the conflict. + expect(claimed).toEqual([slotEventId(2), slotEventId(4)]); + }); + + it('leaves a ULID-numbered batch on one shared watermark, unretried', async () => { + // A 412 compares time, so every member of the batch carries the same fence + // value and the batch fails as a unit — which is what the caller's + // fresh-replay path expects. + const time = 1_700_000_000_000; + const log = toMutableEventLog([makeUlidEvent(time)], 'c0'); + const claim = stepClaimFence( + 'wrun_test', + log, + SPEC_VERSION_SLOT_IDENTITY - 1 + ); + const op = vi.fn(async () => { + throw new PreconditionFailedError('stale'); + }); + + await expect(claim(op)).rejects.toBeInstanceOf(PreconditionFailedError); + expect(op).toHaveBeenCalledTimes(1); + expect(op).toHaveBeenCalledWith({ stateUpdatedAt: time }); + expect(eventsListMock).not.toHaveBeenCalled(); + }); +}); + +describe('requiresFreshReplay', () => { + it('covers both fences, so neither numbering fails the run', () => { + // Each fence reports an incomplete view in its own dialect. A caller that + // recognises only one of them fails runs on the other. + expect(requiresFreshReplay(new PreconditionFailedError('stale'))).toBe( + true + ); + expect( + requiresFreshReplay( + new SlotConflictError('taken', { eventId: slotEventId(3) }) + ) + ).toBe(true); + }); + + it('leaves every other rejection to its own handler', () => { + expect(requiresFreshReplay(new EntityConflictError('exists'))).toBe(false); + expect(requiresFreshReplay(new Error('boom'))).toBe(false); + expect(requiresFreshReplay(undefined)).toBe(false); + }); +}); + describe('withPreconditionRetry', () => { let originalGuard: string | undefined; @@ -522,10 +1000,7 @@ describe('withPreconditionRetry', () => { it('passes no snapshot to op when the guard is explicitly disabled', async () => { process.env.WORKFLOW_PRECONDITION_GUARD = '0'; - const log: MutableEventLog = { - events: [makeUlidEvent(1_700_000_000_000)], - cursor: 'c0', - }; + const log = toMutableEventLog([makeUlidEvent(1_700_000_000_000)], 'c0'); const op = vi.fn(async (stateUpdatedAt?: number) => { expect(stateUpdatedAt).toBeUndefined(); return 'ok'; @@ -541,10 +1016,7 @@ describe('withPreconditionRetry', () => { it('sends a snapshot by default when the guard variable is unset (on by default)', async () => { delete process.env.WORKFLOW_PRECONDITION_GUARD; const time = 1_700_000_000_000; - const log: MutableEventLog = { - events: [makeUlidEvent(time)], - cursor: 'c0', - }; + const log = toMutableEventLog([makeUlidEvent(time)], 'c0'); const op = vi.fn(async (stateUpdatedAt?: number) => { expect(stateUpdatedAt).toBe(time); return 'ok'; @@ -558,10 +1030,7 @@ describe('withPreconditionRetry', () => { it('passes the latest snapshot time to op and returns its result without reloading', async () => { const time = 1_700_000_000_000; - const log: MutableEventLog = { - events: [makeUlidEvent(time)], - cursor: 'c0', - }; + const log = toMutableEventLog([makeUlidEvent(time)], 'c0'); const op = vi.fn(async (stateUpdatedAt?: number) => { expect(stateUpdatedAt).toBe(time); return 'ok'; @@ -575,10 +1044,7 @@ describe('withPreconditionRetry', () => { }); it('reloads the event log and retries on a stale (412) rejection, then succeeds', async () => { - const log: MutableEventLog = { - events: [makeUlidEvent(1_700_000_000_000)], - cursor: 'c0', - }; + const log = toMutableEventLog([makeUlidEvent(1_700_000_000_000)], 'c0'); // Each reload returns one newer event and advances the cursor. eventsListMock.mockResolvedValueOnce({ data: [makeUlidEvent(1_700_000_001_000)], @@ -610,10 +1076,7 @@ describe('withPreconditionRetry', () => { }); it('rethrows the precondition error after exhausting reload retries', async () => { - const log: MutableEventLog = { - events: [makeUlidEvent(1_700_000_000_000)], - cursor: 'c0', - }; + const log = toMutableEventLog([makeUlidEvent(1_700_000_000_000)], 'c0'); eventsListMock.mockResolvedValue({ data: [], cursor: 'c1', @@ -633,10 +1096,7 @@ describe('withPreconditionRetry', () => { }); it('rethrows non-precondition errors immediately without reloading', async () => { - const log: MutableEventLog = { - events: [makeUlidEvent(1_700_000_000_000)], - cursor: 'c0', - }; + const log = toMutableEventLog([makeUlidEvent(1_700_000_000_000)], 'c0'); const op = vi.fn(async () => { throw new Error('boom'); }); diff --git a/packages/core/src/runtime/helpers.ts b/packages/core/src/runtime/helpers.ts index 5ce8dc38e1..586c0f1a68 100644 --- a/packages/core/src/runtime/helpers.ts +++ b/packages/core/src/runtime/helpers.ts @@ -1,6 +1,7 @@ import { PreconditionFailedError, RUN_ERROR_CODES, + SlotConflictError, WorkflowWorldError, } from '@workflow/errors'; import type { @@ -16,10 +17,13 @@ import type { import { getQueueTopicPrefix, HealthCheckPayloadSchema, + maxSlotOf, resolveQueueNamespace, SPEC_VERSION_CURRENT, SPEC_VERSION_LEGACY, + slotEventId, ulidToDate, + usesSlotIdentity, } from '@workflow/world'; import { monotonicFactory } from 'ulid'; import { runtimeLogger } from '../logger.js'; @@ -653,6 +657,76 @@ export const PRECONDITION_MAX_RELOAD_RETRIES = 2; export interface MutableEventLog { events: Event[]; cursor: string | null; + /** + * Highest slot present in `events`, or 0 for a log that is empty or + * ULID-numbered. Maintained by `mergeLoadedEvents` from the events merged in, + * never from the array's last element: events are appended without sorting, + * so after a merge the last element need not be the newest. + */ + maxSlot: number; + /** + * Next slot `reserveSlot` will hand out. Absolute, and it only ever moves + * forward: a merge can raise it past the events it brought in, but must never + * lower it onto a slot already handed to a writer that is still in flight. + */ + nextSlot: number; +} + +/** + * A `MutableEventLog` over a freshly loaded snapshot. + * + * `slotFloor` is a slot known to be published that the snapshot may not contain + * — the run's own `run_started`, whose write turbo backgrounds while replaying + * against an empty log. Numbering a claim from the snapshot alone would then + * propose a slot that is already taken, so every first write of a turbo + * invocation would conflict and cost the run an extra replay. + */ +export function toMutableEventLog( + events: Event[], + cursor: string | null, + slotFloor = 0 +): MutableEventLog { + const maxSlot = Math.max(maxSlotOf(events), slotFloor); + return { + events, + cursor, + maxSlot, + nextSlot: maxSlot + 1, + }; +} + +/** + * Merges loaded events into `log` in place, keeping `maxSlot` current and + * advancing the reservation pointer past the events merged in. + */ +export function mergeLoadedEvents( + log: MutableEventLog, + events: readonly Event[] +): void { + appendUniqueEvents(log.events, events); + log.maxSlot = Math.max(log.maxSlot, maxSlotOf(events)); + log.nextSlot = Math.max(log.nextSlot, log.maxSlot + 1); +} + +/** + * Claims the next free slot in `log`, synchronously at the moment an event is + * issued. + * + * Reservations are contiguous rather than all-`maxSlot + 1` because a + * suspension flushes its operations concurrently: without them every operation + * in the flush would propose the same slot and all but one would conflict, on + * every single flush. Operations are built in deterministic replay order, so + * the slot each one draws is replay-stable too. + * + * The pointer is never rewound by a merge, only pushed forward. A writer that + * loses its slot merges the delta and reserves again while its siblings still + * hold theirs; rewinding to `maxSlot + 1` would hand it a sibling's slot and + * turn one conflict into a chain of them. + */ +export function reserveSlot(log: MutableEventLog): number { + const slot = log.nextSlot; + log.nextSlot = slot + 1; + return slot; } /** @@ -706,8 +780,23 @@ export function latestEventStateUpdatedAt(events: Event[]): number | undefined { * The `stateUpdatedAt` to attach to a replay-context event creation: * the loaded snapshot's ULID time when the precondition guard is enabled, * `undefined` (no guard, backend behaves as before) otherwise. + * + * Always `undefined` for a run that numbers its events by slot, where the event + * id is itself the concurrency fence and the watermark is dead weight. The mode + * is passed in explicitly rather than inferred from the event ids, because + * inference would silently produce a *wrong* value instead of none: a padded + * slot body is valid Crockford base32, so `latestEventStateUpdatedAt` decodes it + * to epoch 0 rather than failing its fail-open check, and the client would send + * `stateUpdatedAt: 0` — which a backend still running the watermark guard reads + * as a snapshot older than every event. */ -export function stateUpdatedAtForCreate(events: Event[]): number | undefined { +export function stateUpdatedAtForCreate( + events: Event[], + specVersion?: number +): number | undefined { + if (usesSlotIdentity(specVersion)) { + return undefined; + } return isPreconditionGuardEnabled() ? latestEventStateUpdatedAt(events) : undefined; @@ -752,7 +841,7 @@ export async function withPreconditionRetry( runId, log.cursor ?? undefined ); - appendUniqueEvents(log.events, loaded.events); + mergeLoadedEvents(log, loaded.events); // When several creates share one `log` (e.g. hook creations under // `Promise.all` in `handleSuspension`), concurrent 412s can reload // concurrently. The event merge above is safe — `appendUniqueEvents` @@ -771,6 +860,238 @@ export type EventCreator = ( params?: CreateEventParams ) => Promise; +/** + * The concurrency fence a replay-context event creation carries. Exactly one of + * the two schemes is ever populated: `stateUpdatedAt` for a run guarded by the + * event-log watermark, `eventId`/`maxSlot` for a run that numbers its events by + * slot. + */ +export interface EventCreateFence { + stateUpdatedAt?: number; + eventId?: string; + maxSlot?: number; +} + +/** + * The fence for a create that is deliberately **not** retried in place, because + * a rejection means the committed decision itself is stale and only a fresh + * replay can revise it (`run_completed`). + * + * Claims a slot off `log` for a slot-numbered run — which counts as a + * reservation, so a caller that fences several creates from one log gets a + * distinct slot per create. `undefined` when the run is fenced neither way, + * leaving the create exactly as unfenced as it was before either mechanism + * existed. + * + * `extraEvents` is how many events *besides* the one being created this write + * publishes: a lazy inline `step_started` also materializes the `step_created` + * it deferred. Those events take the slots immediately below the claim, so this + * reserves them too and names the top one — a World that writes a pair + * derives the lower id from the one it was given. + * + * The reservation has to happen here rather than at the World or its backend. + * Slots are handed out for a whole concurrent batch synchronously, before any of + * it lands, so a second event numbered off the log as the backend sees it would + * take the slot already promised to the next write in the batch — and every + * write after the first in a fan-out would lose its claim. + */ +export function eventCreateFenceFor( + log: MutableEventLog, + specVersion: number | undefined, + options?: { extraEvents?: number } +): EventCreateFence | undefined { + if (usesSlotIdentity(specVersion)) { + return reserveSlotFence(log, options?.extraEvents ?? 0); + } + const stateUpdatedAt = stateUpdatedAtForCreate(log.events, specVersion); + return stateUpdatedAt !== undefined ? { stateUpdatedAt } : undefined; +} + +/** + * Reserves this write's slots off `log` and names the one the event itself + * takes. + * + * `extraEvents` sit below the one being created, matching the order a reader + * expects (a step is created before it starts), so their slots are reserved + * first and the claim names the last of the run — a World that writes a pair + * derives the lower id from the one it was given. + */ +function reserveSlotFence( + log: MutableEventLog, + extraEvents: number +): EventCreateFence { + const maxSlot = log.maxSlot; + for (let i = 0; i < extraEvents; i++) { + reserveSlot(log); + } + return { eventId: slotEventId(reserveSlot(log)), maxSlot }; +} + +/** + * Runs one event create under whichever fence its run uses, handling a lost + * claim however that run's scheme requires. + */ +export type FencedCreate = ( + op: (fence: EventCreateFence | undefined) => Promise +) => Promise; + +/** + * The fence for an inline step's `step_started` claim. + * + * A slot-numbered run retries a lost claim in place; a watermark-guarded run + * does not, and lets the rejection abandon the batch for a fresh replay. The + * asymmetry is in what a rejection proves, and it is the difference between a + * batch that stays contiguous and one that splits: + * + * - A 412 compares the *time* of the newest outside event, so every claim in a + * batch carries the same fence value and a stale view fails all of them. The + * batch is abandoned as a unit, nothing is written, and the fresh replay + * reschedules from a complete view. + * - A 409 only proves another writer took this write's *number*. That happens + * routinely without any staleness: the server allocates outside events from + * the same next-free pointer the client reserves from, so any outside event + * landing mid-batch takes the slot the batch's next claim is holding. Fencing + * the batch on it splits it — the loser writes nothing while its siblings + * commit, and the loser's events land far later in the log (or never), leaving + * an order no single replay can consume. Taking another number instead keeps + * the batch's events adjacent and its slots dense. + */ +export function stepClaimFence( + runId: string, + log: MutableEventLog, + specVersion: number | undefined, + options?: { extraEvents?: number } +): FencedCreate { + if (usesSlotIdentity(specVersion)) { + // Reserved here, synchronously, rather than when the claim fires: a batch's + // claims have to be numbered in replay order, and they only start racing + // each other afterwards. Retries re-reserve at that point by necessity — + // the merged delta has moved the log — but by then this write is the only + // one of the batch still choosing a slot. + const initialFence = reserveSlotFence(log, options?.extraEvents ?? 0); + return (op) => withSlotRetry(runId, log, op, { ...options, initialFence }); + } + const fence = eventCreateFenceFor(log, specVersion, options); + return (op) => op(fence); +} + +/** + * Runs a replay-context event creation that claims its own event slot. + * + * The claim is the fence: the backend inserts the proposed `eventId` + * conditionally, so a `SlotConflictError` (409) proves another writer got there + * first and that this replay therefore ran against an event log missing at least + * one event. Re-sending the same write is pointless — it would lose the same + * slot again — so each attempt merges the events it was missing (inline off the + * rejection, topped up from the backend when the delta was truncated) and claims + * a fresh slot past them. + * + * Bounded by `PRECONDITION_MAX_RELOAD_RETRIES`, after which the error + * propagates and the run is re-invoked from the queue for a fresh replay. That + * fallback is not merely a giving-up path: merged events can change what the + * workflow body decides, and only a replay from the top can act on them. + */ +export async function withSlotRetry( + runId: string, + log: MutableEventLog, + op: (fence: EventCreateFence) => Promise, + options?: { extraEvents?: number; initialFence?: EventCreateFence } +): Promise { + for (let attempt = 0; ; attempt++) { + // Claimed per attempt, not once up front: a merged delta moves the log's + // high-water mark, so the previous claim is stale by definition. The first + // attempt can carry a slot the caller reserved earlier, for a caller whose + // numbering has to be assigned in a particular order (see stepClaimFence). + const fence = + (attempt === 0 ? options?.initialFence : undefined) ?? + reserveSlotFence(log, options?.extraEvents ?? 0); + const eventId = fence.eventId; + try { + return await op(fence); + } catch (error) { + if ( + !SlotConflictError.is(error) || + attempt >= PRECONDITION_MAX_RELOAD_RETRIES + ) { + throw error; + } + runtimeLogger.info( + 'Event creation lost its slot; merging missed events and reclaiming', + { + workflowRunId: runId, + eventId, + attempt: attempt + 1, + maxRetries: PRECONDITION_MAX_RELOAD_RETRIES, + } + ); + await mergeSlotConflictDelta(runId, log, error); + } + } +} + +/** + * Merges the event-log delta a slot conflict carries into `log`. + * + * The inline delta is an optimization, not a contract: a backend that could not + * read it sends none, and one that paginated it sets `hasMore`. Either way the + * fallback is the same full incremental load the runtime would otherwise have + * done, so the merged log is authoritative in every case. + */ +async function mergeSlotConflictDelta( + runId: string, + log: MutableEventLog, + conflict: SlotConflictError +): Promise { + const inline = conflict.events as Event[]; + if (inline.length > 0) { + mergeLoadedEvents(log, inline); + log.cursor = conflict.cursor ?? log.cursor; + } + if (inline.length === 0 || conflict.hasMore) { + const loaded = await loadWorkflowRunEvents(runId, log.cursor ?? undefined); + mergeLoadedEvents(log, loaded.events); + log.cursor = loaded.cursor ?? log.cursor; + } +} + +/** + * Runs a replay-context event creation under whichever concurrency fence the + * run uses: its event slot when it numbers events by slot, the event-log + * watermark otherwise. + * + * The two retry loops stay separate rather than being folded together. They + * differ in what a rejection proves and in what the client does about it, and + * both are live at once while runs on the older numbering drain — keeping them + * apart is what makes a rollout's 409s and 412s separately countable. + */ +export function withEventCreateFence( + runId: string, + log: MutableEventLog, + specVersion: number | undefined, + op: (fence: EventCreateFence) => Promise +): Promise { + if (usesSlotIdentity(specVersion)) { + return withSlotRetry(runId, log, op); + } + return withPreconditionRetry(runId, log, (stateUpdatedAt) => + op({ stateUpdatedAt }) + ); +} + +/** + * Whether a rejected event create means "this replay's view of the log was + * incomplete", the one condition whose only remedy is replaying from the top. + * + * Both fences report it, one per numbering: a 412 says the snapshot's watermark + * is behind, a 409 says the slot this replay counted to is already occupied. + * Neither is a failure of the run — the run's own decisions may simply need + * revising against the events it did not see — so a caller that gets one + * re-invokes for a fresh replay rather than failing. + */ +export function requiresFreshReplay(error: unknown): boolean { + return PreconditionFailedError.is(error) || SlotConflictError.is(error); +} + /** * CORS headers for health check responses. * Allows the observability UI to check endpoint health from a different origin. diff --git a/packages/core/src/runtime/idempotency-key.test.ts b/packages/core/src/runtime/idempotency-key.test.ts new file mode 100644 index 0000000000..c4b4ca5cbb --- /dev/null +++ b/packages/core/src/runtime/idempotency-key.test.ts @@ -0,0 +1,85 @@ +import { readdirSync, readFileSync } from 'node:fs'; +import { join, relative } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; +import { runScopedKey } from './idempotency-key.js'; + +const CORE_SRC = fileURLToPath(new URL('..', import.meta.url)); + +/** + * Expressions allowed as the value of an `idempotencyKey:` property in core: + * the two run-scoping builders, or a type declaration. + */ +const RUN_SCOPED_VALUE = + /^\s*(string;|runScopedKey\(|backstopIdempotencyKey\()/; + +function sourceFiles(dir: string): string[] { + return readdirSync(dir, { withFileTypes: true }).flatMap((entry) => { + const path = join(dir, entry.name); + if (entry.isDirectory()) return sourceFiles(path); + if (!entry.name.endsWith('.ts')) return []; + if (entry.name.endsWith('.test.ts')) return []; + return [path]; + }); +} + +/** Every place the file assigns an `idempotencyKey`, with the value assigned. */ +function keySites( + file: string, + lines: string[] +): Array<{ at: string; value: string }> { + const PROPERTY = 'idempotencyKey:'; + return lines.flatMap((line, index) => { + const at = line.indexOf(PROPERTY); + const trimmed = line.trim(); + if (at === -1 || trimmed.startsWith('//') || trimmed.startsWith('*')) { + return []; + } + // The value may sit on the following line when the formatter wraps it. + const sameLine = line.slice(at + PROPERTY.length); + return [ + { + at: `${relative(CORE_SRC, file)}:${index + 1}: ${trimmed}`, + value: sameLine.trim() ? sameLine : (lines[index + 1] ?? ''), + }, + ]; + }); +} + +describe('runScopedKey', () => { + it('prefixes the run and joins parts with colons', () => { + expect(runScopedKey('wrun_1', 'step_2', 'backstop')).toBe( + 'wrun_1:step_2:backstop' + ); + }); + + it('separates runs sharing a correlation id', () => { + // The whole point: under slot identity the first step of every run of a + // workflow is `step_…001`, and the queue those messages are sent to is + // shared by every run of that workflow. + expect(runScopedKey('wrun_a', 'step_001')).not.toBe( + runScopedKey('wrun_b', 'step_001') + ); + }); +}); + +describe('queue idempotency keys', () => { + /** + * A key that is not run-scoped is dropped silently by the world's dedupe for + * the length of its retention window (24h on Vercel Queues): the send is + * answered normally, no callback is dispatched, and the step is never + * executed. There is no error to find afterwards, so the only defence is that + * no site produces a key any other way. + */ + it('are all produced by the run-scoping builders', () => { + const sites = sourceFiles(CORE_SRC).flatMap((file) => + keySites(file, readFileSync(file, 'utf8').split('\n')) + ); + + expect(sites.filter((site) => !RUN_SCOPED_VALUE.test(site.value))).toEqual( + [] + ); + // Guard the scan itself: a pattern that matches nothing would pass above. + expect(sites.length).toBeGreaterThanOrEqual(4); + }); +}); diff --git a/packages/core/src/runtime/idempotency-key.ts b/packages/core/src/runtime/idempotency-key.ts new file mode 100644 index 0000000000..1a3ed9b1d9 --- /dev/null +++ b/packages/core/src/runtime/idempotency-key.ts @@ -0,0 +1,33 @@ +/** + * Run scoping for queue idempotency keys. + * + * Every keyed enqueue the runtime performs is deduped by the world on + * `(queue, idempotencyKey)`, and the dedupe record outlives the first delivery + * by a long way — Vercel Queues keeps it until message-retention TTL (24h), + * world-postgres keeps a completed-keys cache. Queue names are per workflow, not + * per run, so any key built only from a correlation id is shared by every + * concurrent run of that workflow the moment correlation ids stop being unique + * per run — which is exactly what slot numbering does: the first step of every + * run of `myWorkflow` is `step_…001`. + * + * A collision there is silent and total. The send is answered normally (Vercel + * Queues v3 returns a fresh message id; only the legacy provider reported + * duplicates), so no error surfaces in the SDK; the dispatcher records the + * message as a duplicate and excludes it from notifications, so no callback is + * dispatched; the orchestrator returns without a timeout and acks. The step is + * never executed, nothing anywhere reports a failure, and the run stalls for the + * length of the dedupe window. The inline-ownership backstop cannot recover it + * either — that path needs a `step_started` which never happened. + * + * So every key carries the run it belongs to, built here rather than at each + * call site. Keys are opaque to the worlds, so the prefix costs only length: a + * run id plus a correlation id plus a suffix is well under the 256-char cap. + */ + +/** + * Builds a queue idempotency key scoped to `runId`. Parts are joined with `:`, + * so a key keeps the readable shape it had before scoping was introduced. + */ +export function runScopedKey(runId: string, ...parts: string[]): string { + return [runId, ...parts].join(':'); +} diff --git a/packages/core/src/runtime/start.test.ts b/packages/core/src/runtime/start.test.ts index 29a7c80419..d04a415bb7 100644 --- a/packages/core/src/runtime/start.test.ts +++ b/packages/core/src/runtime/start.test.ts @@ -2,6 +2,7 @@ import { WorkflowRuntimeError, WorkflowWorldError } from '@workflow/errors'; import { SPEC_VERSION_CURRENT, SPEC_VERSION_LEGACY, + SPEC_VERSION_MAX_SUPPORTED, SPEC_VERSION_SUPPORTS_ATTRIBUTES, SPEC_VERSION_SUPPORTS_CBOR_QUEUE_TRANSPORT, } from '@workflow/world'; @@ -136,7 +137,7 @@ describe('start', () => { } as any); await expect(start(validWorkflow, [])).rejects.toThrow( - 'requires a World with matching spec version' + 'requires a World with spec version' ); expect(mockEventsCreate).not.toHaveBeenCalled(); expect(mockQueue).not.toHaveBeenCalled(); @@ -174,7 +175,7 @@ describe('start', () => { } as any); await expect(start(validWorkflow, [])).rejects.toThrow( - 'requires a World with matching spec version' + 'requires a World with spec version' ); expect(mockEventsCreate).not.toHaveBeenCalled(); expect(mockQueue).not.toHaveBeenCalled(); @@ -186,19 +187,43 @@ describe('start', () => { }); setWorld({ - specVersion: SPEC_VERSION_CURRENT + 1, + specVersion: SPEC_VERSION_MAX_SUPPORTED + 1, getDeploymentId: vi.fn().mockResolvedValue('deploy_123'), events: { create: mockEventsCreate }, queue: mockQueue, } as any); await expect(start(validWorkflow, [])).rejects.toThrow( - 'requires a World with matching spec version' + 'requires a World with spec version' ); expect(mockEventsCreate).not.toHaveBeenCalled(); expect(mockQueue).not.toHaveBeenCalled(); }); + it('accepts a world that mints a newer version this runtime supports', async () => { + // A world opted into slot identity stamps a version above the runtime's + // current one. The runtime can read and write those runs, so the + // handshake has to pass and the run has to keep the world's version. + const validWorkflow = Object.assign(() => Promise.resolve('result'), { + workflowId: 'test-workflow', + }); + + setWorld({ + specVersion: SPEC_VERSION_MAX_SUPPORTED, + getDeploymentId: vi.fn().mockResolvedValue('deploy_123'), + events: { create: mockEventsCreate }, + queue: mockQueue, + } as any); + + await start(validWorkflow, []); + + expect(mockEventsCreate).toHaveBeenCalledWith( + expect.stringMatching(/^wrun_/), + expect.objectContaining({ specVersion: SPEC_VERSION_MAX_SUPPORTED }), + expect.anything() + ); + }); + it('should use provided specVersion when passed in options', async () => { const validWorkflow = Object.assign(() => Promise.resolve('result'), { workflowId: 'test-workflow', diff --git a/packages/core/src/runtime/step-executor.test.ts b/packages/core/src/runtime/step-executor.test.ts index b669cce223..e2d0869285 100644 --- a/packages/core/src/runtime/step-executor.test.ts +++ b/packages/core/src/runtime/step-executor.test.ts @@ -171,7 +171,7 @@ describe('executeStep — compute instance stamping', () => { counter += 1; }); - it('stamps computeInstanceId on step_started without displacing the stateUpdatedAt guard', async () => { + it('stamps computeInstanceId on step_started without displacing the claim fence', async () => { const world = makeWorld(); const stepName = uniqueStepName(); const { runId, stepId } = await setupRunningStep({ @@ -191,7 +191,7 @@ describe('executeStep — compute instance stamping', () => { workflowStartedAt: Date.now(), stepId, stepName, - stateUpdatedAt: 1_700_000_000_000, + claimFence: (op) => op({ stateUpdatedAt: 1_700_000_000_000 }), }); const started = createSpy.mock.calls.filter( diff --git a/packages/core/src/runtime/step-executor.ts b/packages/core/src/runtime/step-executor.ts index 9b4daa8ce7..bb68eab33b 100644 --- a/packages/core/src/runtime/step-executor.ts +++ b/packages/core/src/runtime/step-executor.ts @@ -50,7 +50,12 @@ import { isOptimisticInlineStartExplicitlyDisabled, } from './constants.js'; import { getPortLazy } from './get-port-lazy.js'; -import { type EventCreator, memoizeEncryptionKey } from './helpers.js'; +import { + type EventCreateFence, + type EventCreator, + type FencedCreate, + memoizeEncryptionKey, +} from './helpers.js'; import { ReplayRecoveryReporter } from './replay-recovery-reporter.js'; import { computeStepLatencyEventData, @@ -137,20 +142,27 @@ export interface StepExecutorParams { */ inlineDeltaSinceCursor?: string; /** - * Precondition-guard snapshot (epoch ms of the latest event the caller's - * replay loaded) to attach to this step's `step_started` claim. On the lazy - * inline path the claim is the step's FIRST durable write (its - * `step_created` is deferred), so without this the claim would bypass the - * optimistic-concurrency guard entirely: a replay working from a stale view - * could claim — and then commit — a step scheduled without observing an - * out-of-band event. A guard-enforcing World rejects a stale claim with - * `PreconditionFailedError` (412); executeStep does NOT translate that - * rejection (re-claiming in place would still commit the stale schedule), - * so it propagates for the caller to abandon the batch and force a fresh - * replay. Undefined when the guard is disabled or the caller has no - * snapshot; Worlds that don't enforce the guard ignore it. + * Runs this step's `step_started` claim under its run's concurrency fence: + * the event slot the claim occupies, or the caller's replay snapshot + * (`stateUpdatedAt`, epoch ms of the latest event it loaded) for a run on the + * older numbering. + * + * On the lazy inline path the claim is the step's FIRST durable write (its + * `step_created` is deferred), so without a fence it would be unguarded + * entirely: a replay working from a stale view could claim — and then commit — + * a step scheduled without observing an out-of-band event. A fencing World + * rejects such a claim with `SlotConflictError` (409) or + * `PreconditionFailedError` (412). + * + * Whether a rejection is retried in place is the caller's decision, made per + * scheme — see `stepClaimFence`. Either way executeStep does NOT translate a + * rejection that reaches it, so an unretried one propagates for the caller to + * abandon the batch and force a fresh replay. + * + * Undefined when the caller has no snapshot, or when the watermark guard is + * disabled on a run that uses it; Worlds that fence neither way ignore it. */ - stateUpdatedAt?: number; + claimFence?: FencedCreate; /** * Suppress optimistic inline start for this step regardless of * `WORKFLOW_OPTIMISTIC_INLINE_START` / `forceOptimisticStart`: take the @@ -260,6 +272,10 @@ export async function executeStep( stepName, } = params; const isVercel = process.env.VERCEL_URL !== undefined; + // Unfenced when the caller passes no fence — every World that fences + // ignores the field it does not understand, so this is the same create it + // was before either mechanism existed. + const runClaim: FencedCreate = params.claimFence ?? ((op) => op(undefined)); // Gate payload compression on the run's specVersion. const compression = (params.runSpecVersion ?? 0) >= SPEC_VERSION_SUPPORTS_COMPRESSION; @@ -521,13 +537,13 @@ export async function executeStep( let step: Step; // Params for the `step_started` create on either path below: the ambient - // compute-instance stamp plus the optimistic-concurrency claim guard. - const startEventParams: CreateEventParams = { + // compute-instance stamp plus whichever fence the claim is running under. + const startEventParams = ( + fence: EventCreateFence | undefined + ): CreateEventParams => ({ computeInstanceId: COMPUTE_INSTANCE_ID, - ...(params.stateUpdatedAt !== undefined - ? { stateUpdatedAt: params.stateUpdatedAt } - : {}), - }; + ...fence, + }); // `Date.now()` taken immediately before the `step_started` create is // issued (either path below) — anchors RSFS's end point. See // StepLatencyEventData.rsfs and the call sites below. @@ -567,26 +583,28 @@ export async function executeStep( // RSFS measures the run_started-to-POST stretch, and the barrier // wait IS part of that stretch under turbo. stepStartPostSentAtMs = Date.now(); - return createEvent( - { - eventType: 'step_started', - specVersion: SPEC_VERSION_CURRENT, - correlationId: stepId, - eventData: { - stepName, - workflowName, - input: params.lazyStepInput, - // Inline-ownership stamp — see StepExecutorParams.ownerMessageId. - ...(params.ownerMessageId !== undefined - ? { ownerMessageId: params.ownerMessageId } - : {}), + // Fence the claim — see StepExecutorParams.claimFence. A rejection + // the fence does not retry surfaces via reconcileOptimisticStart as a + // non-translatable error: the body result is discarded and the + // rejection propagates to the caller. + return runClaim((fence) => + createEvent( + { + eventType: 'step_started', + specVersion: SPEC_VERSION_CURRENT, + correlationId: stepId, + eventData: { + stepName, + workflowName, + input: params.lazyStepInput, + // Inline-ownership stamp — see StepExecutorParams.ownerMessageId. + ...(params.ownerMessageId !== undefined + ? { ownerMessageId: params.ownerMessageId } + : {}), + }, }, - }, - // Guard the claim — see StepExecutorParams.stateUpdatedAt. A stale - // (412) rejection surfaces via reconcileOptimisticStart as a - // non-translatable error: the body result is discarded and the - // rejection propagates to the caller. - startEventParams + startEventParams(fence) + ) ); } ); @@ -627,26 +645,28 @@ export async function executeStep( ? { ownerMessageId: params.ownerMessageId } : {}; stepStartPostSentAtMs = Date.now(); - const startResult = await createEvent( - { - eventType: 'step_started', - specVersion: SPEC_VERSION_CURRENT, - correlationId: stepId, - eventData: - params.lazyStepInput !== undefined - ? { - stepName, - workflowName, - input: params.lazyStepInput, - ...ownershipStamp, - } - : { stepName, ...ownershipStamp }, - }, - // Guard the claim — see StepExecutorParams.stateUpdatedAt. A stale - // (412) rejection is intentionally NOT translated by - // startErrorToResult below, so it propagates to the caller for a - // fresh replay. - startEventParams + // Fence the claim — see StepExecutorParams.claimFence. A rejection the + // fence does not retry is intentionally NOT translated by + // startErrorToResult below, so it propagates to the caller for a fresh + // replay. + const startResult = await runClaim((fence) => + createEvent( + { + eventType: 'step_started', + specVersion: SPEC_VERSION_CURRENT, + correlationId: stepId, + eventData: + params.lazyStepInput !== undefined + ? { + stepName, + workflowName, + input: params.lazyStepInput, + ...ownershipStamp, + } + : { stepName, ...ownershipStamp }, + }, + startEventParams(fence) + ) ); if (!startResult.step) { diff --git a/packages/core/src/runtime/step-ownership.test.ts b/packages/core/src/runtime/step-ownership.test.ts index 5fda3bb4f2..c0de6cece9 100644 --- a/packages/core/src/runtime/step-ownership.test.ts +++ b/packages/core/src/runtime/step-ownership.test.ts @@ -1,6 +1,7 @@ import type { Event } from '@workflow/world'; import { afterEach, describe, expect, it } from 'vitest'; import type { StepInvocationQueueItem } from '../global.js'; +import { runScopedKey } from './idempotency-key.js'; import { backstopIdempotencyKey, hasPendingStepOwnedByMessage, @@ -9,6 +10,7 @@ import { } from './step-ownership.js'; const LEASE_ENV = 'WORKFLOW_INLINE_OWNERSHIP_LEASE_SECONDS'; +const RUN_ID = 'wrun_01ABC'; function makeStep( overrides: Partial = {} @@ -86,34 +88,47 @@ describe('stepLeaseRemainingSeconds', () => { }); describe('backstopIdempotencyKey', () => { - it('never collides with the step message dedupe key (bare correlationId)', () => { - // The owner's retry handoff enqueues the step keyed by correlationId - // with a ~1s backoff; a backstop occupying that key would absorb the - // retry and stall the run for the full lease. + it('never collides with the step message dedupe key', () => { + // The owner's retry handoff enqueues the step keyed by its run-scoped + // correlationId with a ~1s backoff; a backstop occupying that key would + // absorb the retry and stall the run for the full lease. const step = makeStep(); - expect(backstopIdempotencyKey(step)).not.toBe(step.correlationId); + expect(backstopIdempotencyKey(RUN_ID, step)).not.toBe( + runScopedKey(RUN_ID, step.correlationId) + ); + }); + + it('is isolated per run', () => { + // Slot-numbered correlation IDs repeat across runs of a workflow, and + // queues dedupe per queue — which is per workflow, not per run. + const step = makeStep(); + expect(backstopIdempotencyKey('wrun_other', step)).not.toBe( + backstopIdempotencyKey(RUN_ID, step) + ); }); it('is stable across wake replays within one ownership epoch', () => { // Every wake that observes the same latest step_started derives the // same key, capping fan-out at one pending backstop per epoch. - expect(backstopIdempotencyKey(makeStep())).toBe( - backstopIdempotencyKey(makeStep()) + expect(backstopIdempotencyKey(RUN_ID, makeStep())).toBe( + backstopIdempotencyKey(RUN_ID, makeStep()) ); }); it('changes when owner recovery re-stamps the step', () => { const initial = makeStep({ lastStartedAt: 1_000_000 }); const reStamped = makeStep({ lastStartedAt: 1_030_000 }); - expect(backstopIdempotencyKey(reStamped)).not.toBe( - backstopIdempotencyKey(initial) + expect(backstopIdempotencyKey(RUN_ID, reStamped)).not.toBe( + backstopIdempotencyKey(RUN_ID, initial) ); }); it('is isolated per correlation ID', () => { expect( - backstopIdempotencyKey(makeStep({ correlationId: 'step_A' })) - ).not.toBe(backstopIdempotencyKey(makeStep({ correlationId: 'step_B' }))); + backstopIdempotencyKey(RUN_ID, makeStep({ correlationId: 'step_A' })) + ).not.toBe( + backstopIdempotencyKey(RUN_ID, makeStep({ correlationId: 'step_B' })) + ); }); it('re-arms through a full owner-recovery cycle despite in-flight key retention', () => { @@ -142,9 +157,9 @@ describe('backstopIdempotencyKey', () => { // Epoch 1: owner stamps at T0; a wake replay arms the backstop. const epoch1 = makeStep({ lastStartedAt: 1_000_000 }); - expect(enqueue(backstopIdempotencyKey(epoch1))).toBe('accepted'); + expect(enqueue(backstopIdempotencyKey(RUN_ID, epoch1))).toBe('accepted'); // A second wake in the same epoch is deduped (fan-out stays capped). - expect(enqueue(backstopIdempotencyKey(epoch1))).toBe('deduped'); + expect(enqueue(backstopIdempotencyKey(RUN_ID, epoch1))).toBe('deduped'); // Owner crashes; queue redelivery re-stamps step_started at T1 // (owner recovery) → new ownership epoch, lease refreshed. @@ -153,7 +168,7 @@ describe('backstopIdempotencyKey', () => { // The epoch-1 backstop fires during the refreshed lease. Its handler // replays, sees ownership active with time remaining, and re-arms — // while its own message is still in flight (key not yet released). - const rearm = enqueue(backstopIdempotencyKey(epoch2)); + const rearm = enqueue(backstopIdempotencyKey(RUN_ID, epoch2)); expect(rearm).toBe('accepted'); // Owner dies for good: the epoch-2 backstop is the recovery path, and diff --git a/packages/core/src/runtime/step-ownership.ts b/packages/core/src/runtime/step-ownership.ts index 021511a175..b05d49cd19 100644 --- a/packages/core/src/runtime/step-ownership.ts +++ b/packages/core/src/runtime/step-ownership.ts @@ -1,6 +1,7 @@ import type { Event } from '@workflow/world'; import type { StepInvocationQueueItem } from '../global.js'; import { getInlineOwnershipLeaseSeconds } from './constants.js'; +import { runScopedKey } from './idempotency-key.js'; /** * Inline step ownership helpers for the pending-step dispatch decision table @@ -73,12 +74,23 @@ export function stepLeaseRemainingSeconds( * lease has time remaining, which requires `lastStartedAt` to be set. * * The key must also never be the step message's own `idempotencyKey` - * (the bare correlation ID): the owner's retry handoff enqueues the step - * under that key with a short backoff, and a pending backstop sharing it - * would absorb the retry — turning a 1s backoff into a full-lease stall. + * (the run-scoped correlation ID): the owner's retry handoff enqueues the + * step under that key with a short backoff, and a pending backstop sharing + * it would absorb the retry — turning a 1s backoff into a full-lease stall. + * + * Scoped to the run because correlation IDs are only unique within one (see + * `idempotency-key.ts`). */ -export function backstopIdempotencyKey(step: StepInvocationQueueItem): string { - return `${step.correlationId}:backstop:${step.lastStartedAt}`; +export function backstopIdempotencyKey( + runId: string, + step: StepInvocationQueueItem +): string { + return runScopedKey( + runId, + step.correlationId, + 'backstop', + String(step.lastStartedAt) + ); } /** diff --git a/packages/core/src/runtime/suspension-handler.ts b/packages/core/src/runtime/suspension-handler.ts index b3ef25b848..546d263aeb 100644 --- a/packages/core/src/runtime/suspension-handler.ts +++ b/packages/core/src/runtime/suspension-handler.ts @@ -31,7 +31,7 @@ import { getMaxInlineSteps } from './constants.js'; import { type EventCreator, type MutableEventLog, - withPreconditionRetry, + withEventCreateFence, } from './helpers.js'; import { ReplayRecoveryReporter } from './replay-recovery-reporter.js'; @@ -42,10 +42,11 @@ export interface SuspensionHandlerParams { span?: Span; requestId?: string; /** - * The runtime's loaded event log. Each event creation is sent with this - * snapshot's `stateUpdatedAt` and, if the backend rejects it as stale (412), - * the log is reloaded in place and the create is retried — see - * `withPreconditionRetry`. Guarding per-create (rather than the whole + * The runtime's loaded event log. Each event creation carries a fence derived + * from this snapshot — its own event slot, or the snapshot's `stateUpdatedAt` + * for a run on the older numbering — and if the backend rejects the write, the + * log is reloaded in place and the create is retried; see + * `withEventCreateFence`. Fencing per-create (rather than the whole * suspension) ensures a retry never re-issues an already-created event. */ eventLog?: MutableEventLog; @@ -237,16 +238,18 @@ export async function handleSuspension({ reporter.withEventCreate(params, (p) => world.events.create(runId, data, p) ); - // Adds the optimistic-concurrency guard when the caller supplied a loaded - // event log; without one it creates directly (callers with no replay - // snapshot, e.g. tests). The guard reloads + retries on a stale (412) - // rejection, keeping `eventLog` current in place. It wraps `createEvent` - // rather than the reverse so a retried attempt re-takes the telemetry claim, - // matching how the wait-completion writes in `runtime.ts` compose the two. + // Creates under the run's concurrency fence when the caller supplied a + // loaded event log; without one it creates directly (callers with no replay + // snapshot, e.g. tests). The fence reloads + retries on a rejection, keeping + // `eventLog` current in place. Fencing per-create rather than per-suspension + // is what makes a retry safe: it never re-issues an already-created event. + // It wraps `createEvent` rather than the reverse so a retried attempt + // re-takes the telemetry claim, matching how the wait-completion writes in + // `runtime.ts` compose the two. const createGuarded: EventCreator = (data, params) => eventLog - ? withPreconditionRetry(runId, eventLog, (stateUpdatedAt) => - createEvent(data, { ...params, stateUpdatedAt }) + ? withEventCreateFence(runId, eventLog, run.specVersion, (fence) => + createEvent(data, { ...params, ...fence }) ) : createEvent(data, params); // Separate queue items by type diff --git a/packages/core/src/runtime/wait-continuation.test.ts b/packages/core/src/runtime/wait-continuation.test.ts index 3d79cd5b91..dc611a9524 100644 --- a/packages/core/src/runtime/wait-continuation.test.ts +++ b/packages/core/src/runtime/wait-continuation.test.ts @@ -5,39 +5,57 @@ import { WAIT_CONTINUATION_MAX_DELAY_SECONDS, } from './wait-continuation.js'; +const RUN_ID = 'wrun_01ABC'; const CORR_ID = 'wait_01ABC'; +/** The run-scoped key a mid-range wait dedupes on. */ +const KEY = `${RUN_ID}:${CORR_ID}`; const NOW = new Date('2026-05-19T12:00:20.500Z').getTime(); describe('getWaitContinuationDispatch', () => { - describe('mid-range waits (bare correlationId key)', () => { - it('uses the bare correlationId so re-observations dedupe', () => { - expect(getWaitContinuationDispatch(60, CORR_ID, NOW)).toEqual({ + describe('mid-range waits (unsuffixed key)', () => { + it('uses the run-scoped correlationId so re-observations dedupe', () => { + expect(getWaitContinuationDispatch(RUN_ID, 60, CORR_ID, NOW)).toEqual({ delaySeconds: 60, - idempotencyKey: CORR_ID, + idempotencyKey: KEY, }); }); + it('is isolated per run', () => { + // Slot-numbered correlation IDs repeat across runs of a workflow, and a + // queue's dedupe scope is the workflow, not the run — so two runs + // sleeping at the same point must not share a continuation key. + const other = getWaitContinuationDispatch('wrun_other', 60, CORR_ID, NOW); + expect(other.idempotencyKey).not.toBe(KEY); + }); + it('is stable across suspension passes targeting the same deadline', () => { - const pass1 = getWaitContinuationDispatch(60, CORR_ID, NOW); - const pass2 = getWaitContinuationDispatch(45, CORR_ID, NOW + 15_000); + const pass1 = getWaitContinuationDispatch(RUN_ID, 60, CORR_ID, NOW); + const pass2 = getWaitContinuationDispatch( + RUN_ID, + 45, + CORR_ID, + NOW + 15_000 + ); expect(pass2.idempotencyKey).toBe(pass1.idempotencyKey); }); it('covers the full band up to the max delay', () => { const low = getWaitContinuationDispatch( + RUN_ID, NEAR_ELAPSED_WAIT_THRESHOLD_SECONDS + 1, CORR_ID, NOW ); const high = getWaitContinuationDispatch( + RUN_ID, WAIT_CONTINUATION_MAX_DELAY_SECONDS, CORR_ID, NOW ); - expect(low.idempotencyKey).toBe(CORR_ID); + expect(low.idempotencyKey).toBe(KEY); expect(high).toEqual({ delaySeconds: WAIT_CONTINUATION_MAX_DELAY_SECONDS, - idempotencyKey: CORR_ID, + idempotencyKey: KEY, }); }); }); @@ -46,22 +64,28 @@ describe('getWaitContinuationDispatch', () => { it('suffixes the key with the current epoch second', () => { expect( getWaitContinuationDispatch( + RUN_ID, NEAR_ELAPSED_WAIT_THRESHOLD_SECONDS, CORR_ID, NOW ) ).toEqual({ delaySeconds: NEAR_ELAPSED_WAIT_THRESHOLD_SECONDS, - idempotencyKey: `${CORR_ID}:${Math.floor(NOW / 1000)}`, + idempotencyKey: `${KEY}:${Math.floor(NOW / 1000)}`, }); }); it('collapses same-second duplicates but frees the key for a later retry', () => { - const first = getWaitContinuationDispatch(1, CORR_ID, NOW); - const sameSecond = getWaitContinuationDispatch(1, CORR_ID, NOW + 400); + const first = getWaitContinuationDispatch(RUN_ID, 1, CORR_ID, NOW); + const sameSecond = getWaitContinuationDispatch( + RUN_ID, + 1, + CORR_ID, + NOW + 400 + ); // A retry can only be enqueued after the >= 1s delay of the first // message, which guarantees a later epoch-second bucket. - const retry = getWaitContinuationDispatch(1, CORR_ID, NOW + 1000); + const retry = getWaitContinuationDispatch(RUN_ID, 1, CORR_ID, NOW + 1000); expect(sameSecond.idempotencyKey).toBe(first.idempotencyKey); expect(retry.idempotencyKey).not.toBe(first.idempotencyKey); }); @@ -71,15 +95,23 @@ describe('getWaitContinuationDispatch', () => { const SEVEN_DAYS = 7 * 24 * 3600; // 604800s > 7 * MAX_DELAY (579600s) it('clamps the delay to the max and suffixes the key with the hop index', () => { - expect(getWaitContinuationDispatch(SEVEN_DAYS, CORR_ID, NOW)).toEqual({ + expect( + getWaitContinuationDispatch(RUN_ID, SEVEN_DAYS, CORR_ID, NOW) + ).toEqual({ delaySeconds: WAIT_CONTINUATION_MAX_DELAY_SECONDS, - idempotencyKey: `${CORR_ID}:hop-8`, + idempotencyKey: `${KEY}:hop-8`, }); }); it('keeps the key stable for re-observations within the same hop window', () => { - const pass1 = getWaitContinuationDispatch(SEVEN_DAYS, CORR_ID, NOW); + const pass1 = getWaitContinuationDispatch( + RUN_ID, + SEVEN_DAYS, + CORR_ID, + NOW + ); const pass2 = getWaitContinuationDispatch( + RUN_ID, SEVEN_DAYS - 3600, CORR_ID, NOW + 3600_000 @@ -92,6 +124,7 @@ describe('getWaitContinuationDispatch', () => { const keys: string[] = []; while (remaining > NEAR_ELAPSED_WAIT_THRESHOLD_SECONDS) { const { delaySeconds, idempotencyKey } = getWaitContinuationDispatch( + RUN_ID, remaining, CORR_ID, NOW + (SEVEN_DAYS - remaining) * 1000 @@ -107,18 +140,19 @@ describe('getWaitContinuationDispatch', () => { expect(new Set(keys).size).toBe(keys.length); // 604800s chains as 7 max-delay hops + 1 remainder hop. expect(keys).toHaveLength(8); - expect(keys[keys.length - 1]).toBe(CORR_ID); + expect(keys[keys.length - 1]).toBe(KEY); }); it('uses a fresh key when the final partial hop lands in the near-elapsed band', () => { // Remaining drops below the near-elapsed threshold only at the very // end; the second-bucketed key never collides with hop keys. const nearEnd = getWaitContinuationDispatch( + RUN_ID, NEAR_ELAPSED_WAIT_THRESHOLD_SECONDS, CORR_ID, NOW + SEVEN_DAYS * 1000 ); - expect(nearEnd.idempotencyKey).toMatch(new RegExp(`^${CORR_ID}:\\d+$`)); + expect(nearEnd.idempotencyKey).toMatch(new RegExp(`^${KEY}:\\d+$`)); }); }); @@ -142,6 +176,7 @@ describe('getWaitContinuationDispatch', () => { // A wait exactly at the default near-elapsed threshold would previously // return its full (> max) remaining time as the delay. const { delaySeconds } = getWaitContinuationDispatch( + RUN_ID, NEAR_ELAPSED_WAIT_THRESHOLD_SECONDS, CORR_ID, NOW diff --git a/packages/core/src/runtime/wait-continuation.ts b/packages/core/src/runtime/wait-continuation.ts index 02eb1cffc8..97790c50c5 100644 --- a/packages/core/src/runtime/wait-continuation.ts +++ b/packages/core/src/runtime/wait-continuation.ts @@ -7,6 +7,9 @@ * elapsed waits" pass). This module decides the message's `delaySeconds` * and `idempotencyKey`. * + * Every key is scoped to the run (see `idempotency-key.ts`); the variations + * below are suffixes on top of that. + * * The continuation is keyed on the wait's correlationId: while a wait is * pending, every replay pass over the run re-observes it (e.g., once per * step completion in `Promise.all([steps..., sleep()])`), and without @@ -58,6 +61,7 @@ */ import { envNumber } from '@workflow/world'; +import { runScopedKey } from './idempotency-key.js'; /** * Maximum `delaySeconds` for a single wait-continuation message. Waits @@ -99,9 +103,11 @@ export interface WaitContinuationDispatch { * Computes the queue delay and idempotency key for a wait-continuation * message. `timeoutSeconds` is the time until the wait's `resumeAt` * (floored at 1s by the suspension handler); `waitCorrelationId` - * identifies the wait so repeated suspension passes dedupe. + * identifies the wait so repeated suspension passes dedupe, and `runId` + * keeps it from deduping against another run of the same workflow. */ export function getWaitContinuationDispatch( + runId: string, timeoutSeconds: number, waitCorrelationId: string, now: number = Date.now() @@ -120,14 +126,20 @@ export function getWaitContinuationDispatch( if (timeoutSeconds <= nearElapsedThreshold) { return { delaySeconds: timeoutSeconds, - idempotencyKey: `${waitCorrelationId}:${Math.floor(now / 1000)}`, + idempotencyKey: runScopedKey( + runId, + waitCorrelationId, + String(Math.floor(now / 1000)) + ), }; } const hop = Math.ceil(timeoutSeconds / maxDelaySeconds); + // First hop carries no suffix, so a single-hop wait keeps exactly one key for + // its lifetime. + const hopSuffix = hop === 1 ? [] : [`hop-${hop}`]; return { delaySeconds: Math.min(timeoutSeconds, maxDelaySeconds), - idempotencyKey: - hop === 1 ? waitCorrelationId : `${waitCorrelationId}:hop-${hop}`, + idempotencyKey: runScopedKey(runId, waitCorrelationId, ...hopSuffix), }; } diff --git a/packages/core/src/runtime/world-compatibility.ts b/packages/core/src/runtime/world-compatibility.ts index c06c26eefb..f3de904359 100644 --- a/packages/core/src/runtime/world-compatibility.ts +++ b/packages/core/src/runtime/world-compatibility.ts @@ -1,19 +1,40 @@ import { WorkflowRuntimeError } from '@workflow/errors'; import type { World } from '@workflow/world'; -import { SPEC_VERSION_CURRENT } from '@workflow/world'; +import { + SPEC_VERSION_CURRENT, + SPEC_VERSION_MAX_SUPPORTED, +} from '@workflow/world'; type WorldSpecVersionMetadata = Pick; +/** + * Rejects a World whose protocol this runtime does not speak. + * + * A World declares the spec version it stamps on the runs it creates. Anything + * from {@link SPEC_VERSION_CURRENT} up to {@link SPEC_VERSION_MAX_SUPPORTED} is + * fine: the upper end covers a World opted into a newer identity scheme that + * this runtime already understands, and only versions this runtime has no code + * for are refused. Below the current version means the World package predates + * this runtime and cannot record what it emits. + */ export function assertWorldSupportsRuntimeProtocol( world: WorldSpecVersionMetadata ): void { - if (world.specVersion === SPEC_VERSION_CURRENT) { + if ( + world.specVersion !== undefined && + world.specVersion >= SPEC_VERSION_CURRENT && + world.specVersion <= SPEC_VERSION_MAX_SUPPORTED + ) { return; } const supportedVersion = world.specVersion ?? 'none'; + const supported = + SPEC_VERSION_CURRENT === SPEC_VERSION_MAX_SUPPORTED + ? `${SPEC_VERSION_CURRENT}` + : `${SPEC_VERSION_CURRENT} to ${SPEC_VERSION_MAX_SUPPORTED}`; throw new WorkflowRuntimeError( - `This Workflow runtime requires a World with matching spec version ${SPEC_VERSION_CURRENT}, ` + + `This Workflow runtime requires a World with spec version ${supported}, ` + `but the configured World declares spec version ${supportedVersion}. ` + 'Install a World package version compatible with the current Workflow runtime.' ); diff --git a/packages/core/src/step-delivery-hop-count.test.ts b/packages/core/src/step-delivery-hop-count.test.ts index f7ea3e09b1..1ca6d9ab90 100644 --- a/packages/core/src/step-delivery-hop-count.test.ts +++ b/packages/core/src/step-delivery-hop-count.test.ts @@ -29,6 +29,7 @@ import type { Event } from '@workflow/world'; import * as nanoid from 'nanoid'; import { monotonicFactory } from 'ulid'; import { describe, expect, it, vi } from 'vitest'; +import { createCorrelationIdFactory } from './correlation-ids.js'; import { EventsConsumer } from './events-consumer.js'; import { WorkflowSuspension } from './global.js'; import type { WorkflowOrchestratorContext } from './private.js'; @@ -69,6 +70,10 @@ function setupWorkflowContext( }), invocationsQueue: new Map(), generateUlid: () => ulid(workflowStartedAt), + nextCorrelationId: createCorrelationIdFactory({ + specVersion: undefined, + generateUlid: () => ulid(workflowStartedAt), + }), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) ), diff --git a/packages/core/src/step-delivery-ordering.test.ts b/packages/core/src/step-delivery-ordering.test.ts index 85933b7e7c..1f52e092f7 100644 --- a/packages/core/src/step-delivery-ordering.test.ts +++ b/packages/core/src/step-delivery-ordering.test.ts @@ -4,6 +4,7 @@ import type { Event } from '@workflow/world'; import * as nanoid from 'nanoid'; import { monotonicFactory } from 'ulid'; import { afterEach, describe, expect, it, vi } from 'vitest'; +import { createCorrelationIdFactory } from './correlation-ids.js'; import { EventsConsumer } from './events-consumer.js'; import { WorkflowSuspension } from './global.js'; import type { WorkflowOrchestratorContext } from './private.js'; @@ -118,6 +119,10 @@ function setupWorkflowContext( }), invocationsQueue: new Map(), generateUlid: () => ulid(workflowStartedAt), + nextCorrelationId: createCorrelationIdFactory({ + specVersion: undefined, + generateUlid: () => ulid(workflowStartedAt), + }), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) ), diff --git a/packages/core/src/step-hydration-memoization.test.ts b/packages/core/src/step-hydration-memoization.test.ts index 7df9abdf38..97bc5687e4 100644 --- a/packages/core/src/step-hydration-memoization.test.ts +++ b/packages/core/src/step-hydration-memoization.test.ts @@ -2,6 +2,7 @@ import type { Event } from '@workflow/world'; import * as nanoid from 'nanoid'; import { monotonicFactory } from 'ulid'; import { afterEach, describe, expect, it, vi } from 'vitest'; +import { createCorrelationIdFactory } from './correlation-ids.js'; import { EventsConsumer } from './events-consumer.js'; import type { WorkflowOrchestratorContext } from './private.js'; import { ReplayPayloadCache } from './replay-payload-cache.js'; @@ -43,6 +44,10 @@ function setupWorkflowContext( }), invocationsQueue: new Map(), generateUlid: () => ulid(workflowStartedAt), + nextCorrelationId: createCorrelationIdFactory({ + specVersion: undefined, + generateUlid: () => ulid(workflowStartedAt), + }), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) ), diff --git a/packages/core/src/step.test.ts b/packages/core/src/step.test.ts index 80c5e5a30a..9ea1419d6a 100644 --- a/packages/core/src/step.test.ts +++ b/packages/core/src/step.test.ts @@ -9,6 +9,7 @@ import * as nanoid from 'nanoid'; import { monotonicFactory } from 'ulid'; import { beforeAll, describe, expect, it, vi } from 'vitest'; import { registerSerializationClass } from './class-serialization.js'; +import { createCorrelationIdFactory } from './correlation-ids.js'; import { EventsConsumer } from './events-consumer.js'; import { WorkflowSuspension } from './global.js'; import type { WorkflowOrchestratorContext } from './private.js'; @@ -62,6 +63,10 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { }), invocationsQueue: new Map(), generateUlid: () => ulid(workflowStartedAt), // All generated ulids use the workflow's started at time + nextCorrelationId: createCorrelationIdFactory({ + specVersion: undefined, + generateUlid: () => ulid(workflowStartedAt), + }), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) ), diff --git a/packages/core/src/step.ts b/packages/core/src/step.ts index 7801b25d6d..0deb4e5c64 100644 --- a/packages/core/src/step.ts +++ b/packages/core/src/step.ts @@ -24,7 +24,7 @@ export function createUseStep(ctx: WorkflowOrchestratorContext) { ): Promise { const { promise, resolve, reject } = withResolvers(); - const correlationId = `step_${ctx.generateUlid()}`; + const correlationId = ctx.nextCorrelationId('step'); const queueItem: StepInvocationQueueItem = { type: 'step', diff --git a/packages/core/src/workflow.ts b/packages/core/src/workflow.ts index ec76b2b986..76fd5e00dd 100644 --- a/packages/core/src/workflow.ts +++ b/packages/core/src/workflow.ts @@ -10,13 +10,18 @@ import type { Event, WorkflowRun, WorldCapabilities } from '@workflow/world'; import { SPEC_VERSION_SUPPORTS_COMPRESSION } from '@workflow/world'; import * as nanoid from 'nanoid'; import { monotonicFactory } from 'ulid'; +import { createCorrelationIdFactory } from './correlation-ids.js'; import { EventConsumerResult, EventsConsumer } from './events-consumer.js'; import type { QueueItem } from './global.js'; import { ENOTSUP, WorkflowSuspension } from './global.js'; import { runtimeLogger } from './logger.js'; -import type { WorkflowOrchestratorContext } from './private.js'; +import { + hasInFlightDelivery, + type WorkflowOrchestratorContext, +} from './private.js'; import { ReplayPayloadCache } from './replay-payload-cache.js'; import { getPortLazy } from './runtime/get-port-lazy.js'; +import type { MutableEventLog } from './runtime/helpers.js'; import { runIdCreatedAt } from './runtime/run-id-time.js'; import { handleSuspension } from './runtime/suspension-handler.js'; import { getWorld } from './runtime/world.js'; @@ -73,7 +78,15 @@ async function drainPendingQueueItems( * In turbo mode, gates final `*_created` writes on backgrounded * `run_started`. Undefined when `run_started` is awaited. */ - runReadyBarrier?: Promise + runReadyBarrier?: Promise, + /** + * The replay's event log, so the drain's writes claim their slots from the + * same source the terminal `run_completed` / `run_failed` write draws from. + * Without it the drain writes unfenced — the World picks the next free slot — + * and the terminal write, numbering from a snapshot taken before the drain, + * proposes the slot the drain just took and loses it. + */ + eventLog?: MutableEventLog ): Promise { if (pendingQueue.size === 0) return; // Implicitly dispose any abort hooks (system hooks) that are still alive at @@ -100,6 +113,7 @@ async function drainPendingQueueItems( world, run: workflowRun, runReadyBarrier, + eventLog, }); } catch (err) { runtimeLogger.warn( @@ -136,7 +150,13 @@ export async function runWorkflow( * Features supported by the World executing this workflow. Missing * capabilities are treated as unsupported. */ - worldCapabilities?: WorldCapabilities + worldCapabilities?: WorldCapabilities, + /** + * The caller's event log for this replay. Its only use here is the end-of-run + * drain, whose writes have to be ordered with the caller's terminal write — + * see {@link drainPendingQueueItems}. + */ + eventLog?: MutableEventLog ): Promise { return trace(`workflow.run ${workflowRun.workflowName}`, async (span) => { span?.setAttributes({ @@ -191,22 +211,34 @@ export async function runWorkflow( // by step/hook/sleep callbacks as events are processed. const promiseQueueHolder = { current: Promise.resolve() }; + // Assigned immediately below. The consumer needs to test the context's + // delivery state, and the context needs the consumer. + let workflowContext: WorkflowOrchestratorContext; + const eventsConsumer = new EventsConsumer(events, { onConsumedEvent: (event) => { updateTimestamp(+event.createdAt); }, onUnconsumedEvent: (event) => { + // Name what the replay was waiting for instead. An unconsumable event + // is almost always one whose entity this replay never issued, or + // issued under a different correlation ID; the pending invocation + // queue is the only place that distinction is visible, and without it + // the log names a symptom with no way to reach the cause. + const pending = [...workflowContext.invocationsQueue.keys()]; workflowDiscontinuation.reject( new ReplayDivergenceError( - `Replay could not consume event: eventType=${event.eventType}, correlationId=${event.correlationId}, eventId=${event.eventId}.`, + `Replay could not consume event: eventType=${event.eventType}, correlationId=${event.correlationId}, eventId=${event.eventId}. Pending invocations: ${pending.length > 0 ? pending.join(', ') : '(none)'}.`, { eventId: event.eventId } ) ); }, getPromiseQueue: () => promiseQueueHolder.current, + isDeliveryInFlight: () => + workflowContext !== undefined && hasInFlightDelivery(workflowContext), }); - const workflowContext: WorkflowOrchestratorContext = { + workflowContext = { runId: workflowRun.runId, encryptionKey, worldCapabilities, @@ -216,6 +248,13 @@ export async function runWorkflow( // Correlation IDs must be replay-stable. `startedAt` differs between a // turbo delivery and a later server-backed replay, so use fixedTimestamp. generateUlid: () => ulid(fixedTimestamp), + // Mode comes from the run's persisted spec version, never from this + // build: a run whose log holds ULID correlation ids must keep proposing + // them however new the code replaying it is. + nextCorrelationId: createCorrelationIdFactory({ + specVersion: workflowRun.specVersion, + generateUlid: () => ulid(fixedTimestamp), + }), generateNanoid, invocationsQueue: new Map(), // Use getter/setter so the EventsConsumer's getPromiseQueue() always @@ -851,7 +890,8 @@ export async function runWorkflow( vmGlobalThis, workflowRun, 'completed', - runReadyBarrier + runReadyBarrier, + eventLog ); return dehydrated; @@ -868,7 +908,8 @@ export async function runWorkflow( vmGlobalThis, workflowRun, 'failed', - runReadyBarrier + runReadyBarrier, + eventLog ); throw err; diff --git a/packages/core/src/workflow/hook.test.ts b/packages/core/src/workflow/hook.test.ts index 0cc4a950e0..307293f571 100644 --- a/packages/core/src/workflow/hook.test.ts +++ b/packages/core/src/workflow/hook.test.ts @@ -12,6 +12,7 @@ import { aliasSerializationClass, RUN_CLASS_ID, } from '../class-serialization.js'; +import { createCorrelationIdFactory } from '../correlation-ids.js'; import { EventsConsumer } from '../events-consumer.js'; import { WorkflowSuspension } from '../global.js'; import type { WorkflowOrchestratorContext } from '../private.js'; @@ -23,7 +24,10 @@ import { createWebhook } from './create-hook.js'; import { createCreateHook } from './hook.js'; // Helper to setup context to simulate a workflow run -function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { +function setupWorkflowContext( + events: Event[], + onUnconsumedEvent: (event: Event) => void = () => {} +): WorkflowOrchestratorContext { const context = createContext({ seed: 'test', fixedTimestamp: 1753481739458, @@ -42,11 +46,15 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { replayPayloadCache: new ReplayPayloadCache(undefined), globalThis: context.globalThis, eventsConsumer: new EventsConsumer(events, { - onUnconsumedEvent: () => {}, + onUnconsumedEvent, getPromiseQueue: () => Promise.resolve(), }), invocationsQueue: new Map(), generateUlid: () => ulid(workflowStartedAt), + nextCorrelationId: createCorrelationIdFactory({ + specVersion: undefined, + generateUlid: () => ulid(workflowStartedAt), + }), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) ), @@ -466,6 +474,65 @@ describe('createCreateHook', () => { expect(runtimeErrors).toHaveLength(0); }); + it('should discard a hook_received ordered after the hook_disposed', async () => { + // The world orders a delivery by when its event row commits, not by when + // the payload arrived, so a delivery that raced the disposal can land after + // it in the log. Nothing else in the run can consume that event, so the + // hook's own consumer has to swallow it — otherwise the events consumer + // reports an orphan and the replay diverges on a well-formed log. + const ops: Promise[] = []; + const onUnconsumedEvent = vi.fn(); + const ctx = setupWorkflowContext( + [ + { + eventId: 'evnt_0', + runId: 'wrun_123', + eventType: 'hook_created', + correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + eventData: { token: 'test-token' }, + createdAt: new Date(), + }, + { + eventId: 'evnt_1', + runId: 'wrun_123', + eventType: 'hook_disposed', + correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + eventData: { token: 'test-token' }, + createdAt: new Date(), + }, + { + eventId: 'evnt_2', + runId: 'wrun_123', + eventType: 'hook_received', + correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + eventData: { + token: 'test-token', + payload: await dehydrateStepReturnValue( + { message: 'lost the race' }, + 'wrun_test', + undefined, + ops + ), + }, + createdAt: new Date(), + }, + ], + onUnconsumedEvent + ); + + const createHook = createCreateHook(ctx); + createHook({ token: 'test-token' }); + + // The whole log is consumed: the disposal retires the hook and the late + // delivery is dropped on the floor. + await vi.waitFor(() => { + expect(ctx.eventsConsumer.eventIndex).toBe(3); + }); + expect(onUnconsumedEvent).not.toHaveBeenCalled(); + expect(ctx.onWorkflowError).not.toHaveBeenCalled(); + expect(ctx.invocationsQueue.size).toBe(0); + }); + it('should handle multiple hook_received events with iterator', async () => { const ops: Promise[] = []; const ctx = setupWorkflowContext([ diff --git a/packages/core/src/workflow/hook.ts b/packages/core/src/workflow/hook.ts index e079827051..f7d1f209ac 100644 --- a/packages/core/src/workflow/hook.ts +++ b/packages/core/src/workflow/hook.ts @@ -154,8 +154,9 @@ export function createCreateHook(ctx: WorkflowOrchestratorContext) { eventLogEmpty = true; if ( - (promises.length > 0 && payloadsQueue.length === 0) || - (getConflictPromises.length > 0 && !hasCreated && !hasConflict) + !hasDisposedEvent && + ((promises.length > 0 && payloadsQueue.length === 0) || + (getConflictPromises.length > 0 && !hasCreated && !hasConflict)) ) { scheduleWhenIdle(ctx, () => { ctx.onWorkflowError( @@ -171,6 +172,31 @@ export function createCreateHook(ctx: WorkflowOrchestratorContext) { return EventConsumerResult.NotConsumed; } + if (hasDisposedEvent) { + // A delivery ordered AFTER this hook's own `hook_disposed`. The world + // orders a delivery by when its event row is written, not by when the + // payload arrived, so a delivery that raced the disposal — arriving + // first, committing second — lands here. Swallow it: the hook is gone, + // there is no consumer to hand the payload to, and every awaiter was + // already settled by `disposeHook`. + // + // This consumer must stay registered to do that. Retiring it on + // `hook_disposed` leaves the late delivery with no consumer at all, + // which the events consumer reports as an orphaned event — + // a `ReplayDivergenceError` that recurs on every replay of a log that + // is otherwise perfectly well-formed, escalating to a terminal + // `CorruptedEventLogError`. + webhookLogger.warn( + 'Discarding a hook delivery ordered after disposal', + { + correlationId, + eventId: event.eventId, + eventType: event.eventType, + } + ); + return EventConsumerResult.Consumed; + } + const eventToken = 'eventData' in event && event.eventData && 'token' in event.eventData ? event.eventData.token @@ -418,8 +444,10 @@ export function createCreateHook(ctx: WorkflowOrchestratorContext) { ctx.invocationsQueue.delete(correlationId); // Mark that the event log confirms disposal happened hasDisposedEvent = true; - // We're done processing any more events for this hook - return EventConsumerResult.Finished; + // Stay registered as a tombstone rather than retiring: a delivery that + // raced this disposal can still be ordered after it, and nothing else + // in the run can consume it. See the `hasDisposedEvent` branch above. + return EventConsumerResult.Consumed; } // This replay installed a different consumer than the stored event needs. @@ -539,8 +567,9 @@ export function createCreateHook(ctx: WorkflowOrchestratorContext) { // Drain any pending promises that are waiting for payloads. // Without this, promises created by `await hook` or the async iterator's - // `yield await this` would hang forever since the event consumer will - // never deliver another hook_received after disposal. + // `yield await this` would hang forever: a hook_received ordered after + // the disposal is discarded rather than handed to an awaiter, so nothing + // will ever settle them. if (promises.length > 0) { promises.length = 0; scheduleWhenIdle(ctx, () => { diff --git a/packages/core/src/workflow/sleep.test.ts b/packages/core/src/workflow/sleep.test.ts index a7c4723dd7..2a459d435e 100644 --- a/packages/core/src/workflow/sleep.test.ts +++ b/packages/core/src/workflow/sleep.test.ts @@ -4,6 +4,7 @@ import type { Event } from '@workflow/world'; import * as nanoid from 'nanoid'; import { monotonicFactory } from 'ulid'; import { describe, expect, it, vi } from 'vitest'; +import { createCorrelationIdFactory } from '../correlation-ids.js'; import { EventsConsumer } from '../events-consumer.js'; import { WorkflowSuspension } from '../global.js'; import type { WorkflowOrchestratorContext } from '../private.js'; @@ -38,6 +39,10 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { }), invocationsQueue: new Map(), generateUlid: () => ulid(workflowStartedAt), + nextCorrelationId: createCorrelationIdFactory({ + specVersion: undefined, + generateUlid: () => ulid(workflowStartedAt), + }), generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) ), diff --git a/packages/core/src/workflow/sleep.ts b/packages/core/src/workflow/sleep.ts index c8848d0c3a..aedf1d2392 100644 --- a/packages/core/src/workflow/sleep.ts +++ b/packages/core/src/workflow/sleep.ts @@ -15,7 +15,7 @@ export function createSleep(ctx: WorkflowOrchestratorContext) { param: StringValue | Date | number ): Promise { const { promise, resolve } = withResolvers(); - const correlationId = `wait_${ctx.generateUlid()}`; + const correlationId = ctx.nextCorrelationId('wait'); // Calculate the resume time const resumeAt = parseDurationToDate(param); diff --git a/packages/errors/src/index.ts b/packages/errors/src/index.ts index 3d80cd3772..dcc6599beb 100644 --- a/packages/errors/src/index.ts +++ b/packages/errors/src/index.ts @@ -809,6 +809,59 @@ export class PreconditionFailedError extends WorkflowWorldError { } } +/** + * Thrown when the backend rejects an event creation because the event slot the + * client named was already taken by another writer (HTTP 409). + * + * On a run that numbers its events by slot, whoever writes a slot first owns + * it, and the loser has by definition been replaying against an event log + * missing at least one event. Retrying the same write can therefore never + * succeed: the client has to merge the events it was missing, replay, and + * propose whatever slot that replay lands on. The rejection carries those + * events inline so the common case costs no extra round-trip. + * + * Distinct from `PreconditionFailedError` (412), which is the equivalent + * rejection for a run guarded by the `stateUpdatedAt` watermark instead. Both + * mechanisms are live at once while runs on the older numbering drain. + * + * The workflow runtime handles this automatically. Users interacting with world + * storage backends directly may encounter it. + * + * @property eventId - The slot-numbered event id that was already taken. + * @property events - The events recorded after the client's snapshot, in + * ascending slot order. Empty when the backend could not read them, in which + * case the client reloads the log itself. + * @property cursor - Cursor to continue the delta from, or `null`. + * @property hasMore - Whether events beyond `events` remain to be fetched. + */ +export class SlotConflictError extends WorkflowWorldError { + readonly eventId: string; + readonly events: unknown[]; + readonly cursor: string | null; + readonly hasMore: boolean; + + constructor( + message: string, + options: { + eventId: string; + events?: unknown[]; + cursor?: string | null; + hasMore?: boolean; + } + ) { + super(message, { status: 409 }); + this.name = 'SlotConflictError'; + this.eventId = options.eventId; + this.events = options.events ?? []; + this.cursor = options.cursor ?? null; + this.hasMore = options.hasMore ?? false; + } + + static is(value: unknown): value is SlotConflictError { + return isError(value) && value.name === 'SlotConflictError'; + } +} + /** * Thrown when awaiting `run.returnValue` on a workflow run that was cancelled. * diff --git a/packages/workflow/src/internal/errors.ts b/packages/workflow/src/internal/errors.ts index 4490e7ed4f..9e0cdb3774 100644 --- a/packages/workflow/src/internal/errors.ts +++ b/packages/workflow/src/internal/errors.ts @@ -5,6 +5,7 @@ export { PreconditionFailedError, RunExpiredError, RunNotSupportedError, + SlotConflictError, StepNotRegisteredError, ThrottleError, TooEarlyError, diff --git a/packages/world-local/src/fs.ts b/packages/world-local/src/fs.ts index 70178b344c..8d2ec5d203 100644 --- a/packages/world-local/src/fs.ts +++ b/packages/world-local/src/fs.ts @@ -466,6 +466,11 @@ export async function deleteJSON(filePath: string): Promise { await withWindowsRetry(() => fs.unlink(filePath)); } catch (error) { if ((error as any).code !== 'ENOENT') throw error; + } finally { + // The cache stands in for an `fs.access` on the write path, so a path that + // no longer exists may not stay in it: a later create-if-absent write of the + // same path would be rejected as a duplicate of a file that is gone. + createdFilesCache.delete(filePath); } } @@ -579,6 +584,14 @@ interface PaginatedFileSystemQueryConfig { cursor?: string; getCreatedAt(filename: string): Date | null; getId?(item: T): string; + /** + * The time an item sorts and paginates by, when that is not its `createdAt`. + * A slot-numbered event log orders by slot — the position is the order — and + * a writer that loses a slot re-proposes above the winner while keeping the + * stamp it started with, so `createdAt` there disagrees with the log. Such an + * item reports one shared time and lets the `getId` tie-break order it. + */ + getOrderTime?: (item: NoInfer) => number; } // Cursor format: "timestamp|id" for tie-breaking interface ParsedCursor { @@ -615,6 +628,7 @@ export async function paginatedFileSystemQuery( cursor, getCreatedAt, getId, + getOrderTime = (item: T) => item.createdAt.getTime(), } = config; // Validate filePrefix (typically `${runId}-`) so request-derived prefixes @@ -718,7 +732,7 @@ export async function paginatedFileSystemQuery( // Double-check cursor filtering with actual createdAt from JSON // (in case ULID timestamp differs from stored createdAt) if (parsedCursor) { - const itemTime = item.createdAt.getTime(); + const itemTime = getOrderTime(item); const cursorTime = parsedCursor.timestamp.getTime(); if (sortOrder === 'desc') { @@ -746,8 +760,8 @@ export async function paginatedFileSystemQuery( // 5. Sort by createdAt (and by ID for tie-breaking if getId is provided) validItems.sort((a, b) => { - const aTime = a.createdAt.getTime(); - const bTime = b.createdAt.getTime(); + const aTime = getOrderTime(a); + const bTime = getOrderTime(b); const timeComparison = sortOrder === 'asc' ? aTime - bTime : bTime - aTime; // If timestamps are equal and we have getId, use ID for stable sorting @@ -768,7 +782,7 @@ export async function paginatedFileSystemQuery( const nextCursor = items.length > 0 ? createCursor( - items[items.length - 1].createdAt, + new Date(getOrderTime(items[items.length - 1])), getId?.(items[items.length - 1]) ) : null; diff --git a/packages/world-local/src/index.ts b/packages/world-local/src/index.ts index f8e2166fb7..aa77546732 100644 --- a/packages/world-local/src/index.ts +++ b/packages/world-local/src/index.ts @@ -2,7 +2,7 @@ import { promises as fs } from 'node:fs'; import { rm } from 'node:fs/promises'; import path from 'node:path'; import type { QueuePrefix, World } from '@workflow/world'; -import { reenqueueActiveRuns, SPEC_VERSION_CURRENT } from '@workflow/world'; +import { mintedSpecVersion, reenqueueActiveRuns } from '@workflow/world'; import { warnIfRunningInVercelDeployment } from './build-target-mismatch.js'; import type { Config } from './config.js'; import { config, resolveRecoverActiveRuns } from './config.js'; @@ -72,7 +72,10 @@ export function createWorld(args?: Partial): LocalWorld { ); const recoverActiveRuns = resolveRecoverActiveRuns(mergedConfig); return { - specVersion: SPEC_VERSION_CURRENT, + // What this world stamps on new runs: slot identity, unless + // WORKFLOW_SLOT_IDENTITY switches it off. Every world reads both schemes + // whatever this says. + specVersion: mintedSpecVersion(), ...queue, ...storage, ...instrumentObject('world.streams', { diff --git a/packages/world-local/src/storage/events-storage.ts b/packages/world-local/src/storage/events-storage.ts index 7efd49cb1e..c84ffbf7c7 100644 --- a/packages/world-local/src/storage/events-storage.ts +++ b/packages/world-local/src/storage/events-storage.ts @@ -5,11 +5,13 @@ import { HookNotFoundError, RunExpiredError, RunNotSupportedError, + SlotConflictError, TooEarlyError, WorkflowRunNotFoundError, WorkflowWorldError, } from '@workflow/errors'; import type { + CreateEventParams, Event, EventResult, Hook, @@ -33,9 +35,15 @@ import { isTerminalStepStatus, isTerminalWorkflowRunStatus, requiresNewerWorld, + SLOT_RETRY_BUDGET_MS, SPEC_VERSION_CURRENT, + SPEC_VERSION_MAX_SUPPORTED, StepSchema, + slotEventId, + slotFromId, + slotRetryDelay, ulidToDate, + usesSlotIdentity, validateAttributeChanges, validateUlidTimestamp, WaitSchema, @@ -85,6 +93,7 @@ import { } from './hooks-storage.js'; import { handleLegacyEvent } from './legacy.js'; import { withRunFileLock } from './runs-storage.js'; +import { createSlotBook, RUN_CREATED_SLOT } from './slots.js'; /** * Per-run event ceiling the Local World reports on run responses (mirrors the @@ -122,6 +131,20 @@ function getMaxEventsPerRun(): number { // but a shared filesystem), exactly matching the cross-process // semantics without spawning subprocesses. +/** + * The time an event orders and paginates by. A slot-numbered run's order is its + * slot order — the position *is* the order, the way the sort key is for the + * other backends — so every such event reports the same time and lets the + * event-id tie-break do the ordering. Ordering those by `createdAt` reads the + * log in an order no replay produced: a writer that loses a slot re-proposes + * above the winner while keeping the stamp it started with, and a caller that + * reserves slots for a whole flush commits them in whatever order the network + * returns. A ULID-numbered run keeps its wall-clock order, which its ids agree + * with anyway. + */ +const eventOrderTime = (event: { eventId: string; createdAt: Date }): number => + slotFromId(event.eventId) === undefined ? event.createdAt.getTime() : 0; + const HookTokenClaimSchema = z.object({ // The token-claim writer below has always persisted `hookId`, but // this read schema previously omitted it, which is the bug fixed @@ -259,6 +282,7 @@ async function findExistingHookCreatedEventId( event.correlationId === correlationId, limit: 1, getCreatedAt: getObjectCreatedAt('evnt'), + getOrderTime: eventOrderTime, getId: (event) => event.eventId, }); return result.data[0]?.eventId ?? null; @@ -506,6 +530,8 @@ export function createEventsStorage( const cachedPathsByRunId = new Map>(); let totalCachedEventBytes = 0; + const slots = createSlotBook(basedir, tag); + function deleteCachedEvent(eventPath: string): void { const event = eventCache.get(eventPath); if (!event) { @@ -525,6 +551,7 @@ export function createEventsStorage( for (const cachedPath of cachedPathsByRunId.get(runId) ?? []) { deleteCachedEvent(cachedPath); } + slots.forget(runId); } function clearCache(): void { @@ -532,6 +559,7 @@ export function createEventsStorage( cachedEventBytes.clear(); cachedPathsByRunId.clear(); totalCachedEventBytes = 0; + slots.clear(); } function cacheEvent( @@ -592,6 +620,65 @@ export function createEventsStorage( } } + /** + * The events a caller that just lost a slot is missing: one ascending page of + * the run's log after the cursor it wrote from, minus anything at or below the + * highest slot it already held. Because slots are dense, that second filter is + * exact — a caller cannot be missing an event whose position it can name. + * + * Returned inline with the conflict so the common case (a handful of events + * arrived out of band) costs the caller no extra round-trip. `hasMore` is + * forwarded verbatim: an overflowing delta is the caller's signal to page from + * `cursor` instead of treating this as the whole story. + */ + async function eventsAfterClaim( + runId: string, + params: CreateEventParams | undefined + ): Promise<{ events: Event[]; cursor: string | null; hasMore: boolean }> { + const page = await paginatedFileSystemQuery({ + directory: path.join(basedir, 'events'), + schema: EventSchema, + cachedItems: eventCache, + filePrefix: `${runId}-`, + sortOrder: 'asc', + ...(typeof params?.sinceCursor === 'string' + ? { cursor: params.sinceCursor } + : {}), + getCreatedAt: getObjectCreatedAt('evnt'), + getOrderTime: eventOrderTime, + getId: (event) => event.eventId, + }); + const maxSlot = params?.maxSlot ?? 0; + const resolveData = params?.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION; + const missing = page.data.filter( + (event) => (slotFromId(event.eventId) ?? 0) > maxSlot + ); + return { + events: + resolveData === 'none' + ? missing.map((event) => stripEventDataRefs(event, resolveData)) + : missing, + cursor: page.cursor, + hasMore: page.hasMore, + }; + } + + /** + * The 409 a caller gets when the slot it named turns out to belong to someone + * else, carrying the events it is missing so it can merge, replay and + * re-propose at a free position. + */ + async function slotConflict( + runId: string, + eventId: string, + params: CreateEventParams | undefined + ): Promise { + return new SlotConflictError( + `Slot ${slotFromId(eventId)} of run "${runId}" is already taken`, + { eventId, ...(await eventsAfterClaim(runId, params)) } + ); + } + async function storeEvent(event: Event): Promise { const eventPath = taggedPath( basedir, @@ -643,6 +730,58 @@ export function createEventsStorage( if ('correlationId' in data && typeof data.correlationId === 'string') { assertSafeEntityId('correlationId', data.correlationId); } + if (params?.eventId !== undefined) { + assertSafeEntityId('eventId', params.eventId); + } + + // A slot-numbered create reserves its position before running the + // validation and materialization that may still reject it. Handing the + // reservation back on the way out is what keeps the log dense: an + // abandoned slot below a sibling's published one is a hole that can never + // be filled, and a log with a hole can no longer prove it is complete. + const reserved = new Set(); + let reservedRunId: string | undefined; + /** + * Undo actions for the duplicate-suppression claims a create takes before + * its event exists, newest last. Run only when the create ends without + * publishing anything: the answer to a lost position is to propose the + * same operation one position higher, and a claim left behind is what + * would reject that retry as a duplicate of a write that never landed. + */ + const abandonedClaims: Array<() => Promise> = []; + /** + * Whether any event of this create became reader-visible. Once one has, + * nothing the create claimed may be undone: the entity an event describes + * has to keep existing even if a later step of the same call fails. + */ + let eventCommitted = false; + /** + * Hands the slots of a create that never published back to the allocator, + * so an abandoned reservation below a sibling's published slot does not + * become a hole the run can never fill. + */ + async function releasingSlots( + result: Promise + ): Promise { + try { + return await result; + } catch (error) { + if (reservedRunId !== undefined) { + for (const slot of reserved) { + slots.release(reservedRunId, slot); + } + } + if (!eventCommitted) { + for (const undo of abandonedClaims.reverse()) { + // Best effort: the throw the caller sees is the one that matters, + // and a claim that outlives its create is a duplicate suppressed + // for a write that is not coming back. + await undo().catch(() => {}); + } + } + throw error; + } + } // Step lifecycle events are serialized per-step via an in-process mutex // so that the "check state, then write" sequence in step_started / @@ -653,7 +792,9 @@ export function createEventsStorage( const lockKey = tag ? `${runId}-${data.correlationId}.${tag}` : `${runId}-${data.correlationId}`; - return withInProcessLock(stepLocks, lockKey, () => createImpl()); + return releasingSlots( + withInProcessLock(stepLocks, lockKey, () => createImpl()) + ); } // `hook_created` is serialized per-(runId, hookId) so the // "claim token, write hook entity, write event" sequence runs to @@ -682,9 +823,11 @@ export function createEventsStorage( const lockKey = tag ? `${runId}-${data.correlationId}.hook.${tag}` : `${runId}-${data.correlationId}.hook`; - return withInProcessLock(hookLocks, lockKey, () => createImpl()); + return releasingSlots( + withInProcessLock(hookLocks, lockKey, () => createImpl()) + ); } - return createImpl(); + return releasingSlots(createImpl()); async function createImpl(): Promise { // Most paths use the freshly-generated candidate eventId. The @@ -719,6 +862,18 @@ export function createEventsStorage( // specVersion is always sent by the runtime, but we provide a fallback for safety const effectiveSpecVersion = data.specVersion ?? SPEC_VERSION_CURRENT; + // Whether this run numbers its events by slot. Read from what was + // persisted, never from this request or this build, so a run stays in + // the mode it was created in for life — a run whose log holds ULID ids + // must never be handed a slot id, and vice versa. `run_created` is the + // one event that decides the mode instead of reading it; the + // resilient-start path below decides it too, on the request that + // creates the run. + let slotMode = + data.eventType === 'run_created' + ? usesSlotIdentity(effectiveSpecVersion) + : await slots.usesSlots(effectiveRunId); + // Get current run state for validation (if not creating a new run) // Skip run validation for step_completed and step_retrying - they only operate // on running steps, and running steps are always allowed to modify regardless @@ -800,8 +955,14 @@ export function createEventsStorage( ); if (created) { - // We created the run — also write the run_created event. - const runCreatedEventId = `evnt_${monotonicUlid()}`; + // We created the run, so this request also decided its mode. + slotMode = usesSlotIdentity(effectiveSpecVersion); + // We created the run — also write the run_created event. Its + // slot needs no allocation: a run's own `run_created` provably + // has nothing before it. + const runCreatedEventId = slotMode + ? slotEventId(RUN_CREATED_SLOT) + : `evnt_${monotonicUlid()}`; const runCreatedEvent: Event = { eventType: 'run_created', runId: effectiveRunId, @@ -820,6 +981,7 @@ export function createEventsStorage( }, }; await storeEvent(runCreatedEvent); + slots.observe(effectiveRunId, runCreatedEventId); currentRun = createdRun; } else { // Run already exists (concurrent run_created won the @@ -836,6 +998,17 @@ export function createEventsStorage( } } + // The run entity we just read is the authority on the mode, and it can + // appear between the probe above and this read: start() issues + // `run_created` and the queue send concurrently, so the delivery's + // `run_started` can arrive while the run is still being published. A + // stale "no" there would number that one event with a ULID on an + // otherwise slot-numbered run, and the hole it leaves in the numbering + // costs the log its completeness proof for life. + if (currentRun && data.eventType !== 'run_created') { + slotMode = usesSlotIdentity(currentRun.specVersion); + } + // run_failed on a non-existent run is rejected to match the // postgres and vercel worlds, which both surface this as a // WorkflowRunNotFoundError rather than silently persisting an @@ -857,7 +1030,7 @@ export function createEventsStorage( if (requiresNewerWorld(currentRun.specVersion)) { throw new RunNotSupportedError( currentRun.specVersion!, - SPEC_VERSION_CURRENT + SPEC_VERSION_MAX_SUPPORTED ); } @@ -873,20 +1046,106 @@ export function createEventsStorage( } } - // ============================================================ - // VALIDATION: Terminal state and event ordering checks - // ============================================================ - // Lazy step start: a step_started carrying step-creation data // (stepName + input) is allowed to arrive with no prior step_created // — it creates the step on the fly (see the materialization block // below). This mirrors the resilient run_started path. Detect it here - // so the entity-creation terminal-run guard treats it like a creation - // and the "step must exist" ordering guard doesn't reject it. + // so the second event it publishes can be numbered alongside the + // first, the entity-creation terminal-run guard treats it like a + // creation, and the "step must exist" ordering guard doesn't reject it. const createsChildEntity = isChildEntityCreationEvent(data); const lazyStepStart = createsChildEntity && data.eventType === 'step_started'; + // ============================================================ + // EVENT ID: the caller's slot claim, an allocated slot, or a ULID + // ============================================================ + // A run's own `run_created` owns the first slot — provably, since + // nothing precedes it — so it takes that position outright instead of + // allocating one, and every other event allocates above it even when it + // is the first to arrive here. Allocation is append-only (see SlotBook), + // so `run_created` cannot get the first position by asking for the lowest + // free one: a `run_started` racing it (start() issues the creation and + // the queue send in parallel) may already have moved the book past it. + const ownsFirstSlot = data.eventType === 'run_created'; + // A slot-numbered run's ids name positions in its log, so an id is + // either claimed by a caller that holds the log (and is therefore + // asserting the log is complete up to that position) or allocated here + // for a caller that has no log — a step completion reporting in, a + // cancellation from an API call. + // + // The position of the second event a lazy start publishes, when this + // one publishes two. Consumed by the materialization below; released + // again if that block turns out not to need it. + let companionSlot: number | undefined; + if (params?.eventId !== undefined) { + const claimedSlot = slotFromId(params.eventId); + if (!slotMode) { + throw new WorkflowWorldError( + `Event id "${params.eventId}" was supplied for run "${effectiveRunId}", whose events are not numbered by slot`, + { status: 400 } + ); + } + if (claimedSlot === undefined) { + throw new WorkflowWorldError( + `Event id "${params.eventId}" is not a slot id, and run "${effectiveRunId}" numbers its events by slot`, + { status: 400 } + ); + } + eventId = params.eventId; + reservedRunId = effectiveRunId; + reserved.add(claimedSlot); + slots.claim(effectiveRunId, claimedSlot); + // One request, two events: a lazy start also publishes the + // `step_created` it deferred. A claim names the *top* of the pair, + // so the second event takes the slot immediately below it — the + // caller reserved both positions and named only one, which is what + // keeps the pair from landing on a position another write in the + // same batch is already holding. + if (lazyStepStart) { + companionSlot = claimedSlot - 1; + if (companionSlot < RUN_CREATED_SLOT + 1) { + throw new WorkflowWorldError( + `Event id "${params.eventId}" leaves no slot below it in run "${effectiveRunId}" for the "step_created" published alongside it`, + { status: 400 } + ); + } + reserved.add(companionSlot); + slots.claim(effectiveRunId, companionSlot); + } + // Reject a doomed claim before the materialization below creates + // the step, hook or wait this event will now never accompany. A + // caller that re-proposes at the next slot would otherwise + // collide with its own orphan and read that as "my write already + // landed". See SlotBook.isWritten. + for (const slot of companionSlot === undefined + ? [claimedSlot] + : [companionSlot, claimedSlot]) { + if (await slots.isWritten(effectiveRunId, slot)) { + throw await slotConflict( + effectiveRunId, + slotEventId(slot), + params + ); + } + } + } else if (slotMode) { + reservedRunId = effectiveRunId; + let slot: number; + if (ownsFirstSlot) { + slot = RUN_CREATED_SLOT; + slots.claim(effectiveRunId, slot); + } else { + slot = await slots.reserve(effectiveRunId); + } + reserved.add(slot); + eventId = slotEventId(slot); + } + + // ============================================================ + // VALIDATION: Terminal state and event ordering checks + // ============================================================ + // Run terminal state validation if (currentRun && isTerminalWorkflowRunStatus(currentRun.status)) { // Idempotent operation: run_cancelled on already cancelled run is allowed @@ -1127,13 +1386,40 @@ export function createEventsStorage( // strictly dominates all visible events of the run guarantees the // terminal event replays last. See mintRunDominantEventKey for // the dominance argument. - const dominantKey = await mintRunDominantEventKey( - basedir, - effectiveRunId, - tag - ); - eventId = dominantKey.eventId; - event = { ...event, eventId, createdAt: dominantKey.createdAt }; + // + // A key the *caller* chose is left alone. Its slot was picked from + // the caller's own log, so a concurrent event either sits below it + // (and already replays first) or takes the slot itself — in which + // case the publish below conflicts and the caller merges and + // re-proposes, which is the stronger answer. Re-numbering it here + // would also be actively wrong: the caller reserves slots for a + // whole flush of concurrent ops at once, so moving this one to + // "highest visible + 1" would steal the slot a sibling op is still + // in flight with. + if (params?.eventId === undefined) { + const dominantKey = await mintRunDominantEventKey( + basedir, + effectiveRunId, + tag, + slotMode + ); + const staleSlot = slotFromId(eventId); + const dominantSlot = slotFromId(dominantKey.eventId); + if (staleSlot !== undefined && staleSlot !== dominantSlot) { + // Only reachable when the log moved under us, which means the + // slot we held is now someone else's written event — handing it + // back leaves no hole. + reserved.delete(staleSlot); + slots.release(effectiveRunId, staleSlot); + } + if (dominantSlot !== undefined) { + reservedRunId = effectiveRunId; + reserved.add(dominantSlot); + slots.claim(effectiveRunId, dominantSlot); + } + eventId = dominantKey.eventId; + event = { ...event, eventId, createdAt: dominantKey.createdAt }; + } } // Create/update entity based on event type (event-sourced architecture) @@ -1416,6 +1702,7 @@ export function createEventsStorage( `Step "${data.correlationId}" already created` ); } + abandonedClaims.push(() => fs.unlink(stepCreatedLockPath)); const stepData = data.eventData as { stepName: string; input: any; @@ -1437,10 +1724,14 @@ export function createEventsStorage( specVersion: effectiveSpecVersion, }; const stepCompositeKey = `${effectiveRunId}-${data.correlationId}`; - await writeJSON( - taggedPath(basedir, 'steps', stepCompositeKey, tag), - step + const stepEntityPath = taggedPath( + basedir, + 'steps', + stepCompositeKey, + tag ); + await writeJSON(stepEntityPath, step); + abandonedClaims.push(() => deleteJSON(stepEntityPath)); } else if (data.eventType === 'step_started') { // step_started: Increments attempt, sets status to 'running' // Sets startedAt only on the first start (not updated on retries) @@ -1479,6 +1770,7 @@ export function createEventsStorage( `Step "${data.correlationId}" already created` ); } else { + abandonedClaims.push(() => fs.unlink(stepCreatedLockPath)); const createdStep: Step = { runId: effectiveRunId, stepId: data.correlationId, @@ -1494,24 +1786,35 @@ export function createEventsStorage( updatedAt: now, specVersion: effectiveSpecVersion, }; - await writeJSON( - taggedPath( - basedir, - 'steps', - `${effectiveRunId}-${data.correlationId}`, - tag - ), - createdStep + const lazyStepEntityPath = taggedPath( + basedir, + 'steps', + `${effectiveRunId}-${data.correlationId}`, + tag ); + await writeJSON(lazyStepEntityPath, createdStep); + abandonedClaims.push(() => deleteJSON(lazyStepEntityPath)); // Write the synthetic step_created event so replay observes it // (the client step consumer sets hasCreatedEvent only on a - // step_created event). Its eventId is a fresh monotonic ULID. + // step_created event). Its eventId is a second slot, or a fresh + // monotonic ULID — one request, two events. // Ordering vs. the step_started event row does not affect // correctness: the step_started consumer is a no-op and only // step_created flips hasCreatedEvent, so the end state is the // same whichever sorts first — this matches the resilient // run_started → run_created precedent in this file. - const stepCreatedEventId = `evnt_${monotonicUlid()}`; + let stepCreatedEventId = `evnt_${monotonicUlid()}`; + if (slotMode) { + // A claimed start numbers this event one below its own + // position, which the caller reserved for exactly this. A start + // that allocated takes the next free slot instead: nothing + // outside this world named either position. + const slot = + companionSlot ?? (await slots.reserve(effectiveRunId)); + companionSlot = undefined; + reserved.add(slot); + stepCreatedEventId = slotEventId(slot); + } const stepCreatedEvent: Event = { eventType: 'step_created', runId: effectiveRunId, @@ -1524,15 +1827,39 @@ export function createEventsStorage( input: lazyData.input, }, }; - await writeJSON( - taggedPath( - basedir, - 'events', - `${effectiveRunId}-${stepCreatedEventId}`, - tag - ), - stepCreatedEvent + const stepCreatedEventPath = taggedPath( + basedir, + 'events', + `${effectiveRunId}-${stepCreatedEventId}`, + tag ); + if (slotMode) { + // The position decides this event as much as it decides the + // start it rides with, so it is published the same way: whoever + // links the file first owns the slot. A loss here is the + // caller's to resolve — it named this position — and the undo + // list above takes the step entity and its claim back out, so + // the re-proposal one position higher starts the step lazily + // again instead of tripping its own leftovers. + const published = await writeExclusive( + stepCreatedEventPath, + JSON.stringify(stepCreatedEvent, jsonReplacer, 2) + ); + if (!published) { + throw await slotConflict( + effectiveRunId, + stepCreatedEventId, + params + ); + } + } else { + await writeJSON(stepCreatedEventPath, stepCreatedEvent); + } + // Readers can see this event from here on, so the step entity it + // describes has to keep existing even if the start it rides with + // goes on to lose its own position. + eventCommitted = true; + slots.observe(effectiveRunId, stepCreatedEventId); validatedStep = createdStep; stepCreatedLazily = true; } @@ -2137,6 +2464,7 @@ export function createEventsStorage( `Wait "${data.correlationId}" already exists` ); } + abandonedClaims.push(() => fs.unlink(waitCreatedLockPath)); const waitData = data.eventData as { resumeAt?: Date; }; @@ -2150,10 +2478,14 @@ export function createEventsStorage( updatedAt: now, specVersion: effectiveSpecVersion, }; - await writeJSON( - taggedPath(basedir, 'waits', waitCompositeKey, tag), - wait + const waitEntityPath = taggedPath( + basedir, + 'waits', + waitCompositeKey, + tag ); + await writeJSON(waitEntityPath, wait); + abandonedClaims.push(() => deleteJSON(waitEntityPath)); } else if (data.eventType === 'wait_completed') { // wait_completed: Transitions wait to 'completed', rejects duplicates. // Uses writeExclusive on a lock file to atomically prevent concurrent @@ -2215,11 +2547,17 @@ export function createEventsStorage( // race here; whoever links the file first wins, the loser // throws EntityConflictError, and the runtime's existing // concurrent-replay catch path at suspension-handler.ts:142 - // swallows it. For all other event types, eventIds are - // monotonic ULIDs (globally unique by construction) so a - // collision indicates a real bug and EntityConflictError is + // swallows it. For all other event types of a ULID-numbered run, + // eventIds are monotonic ULIDs (globally unique by construction) so + // a collision indicates a real bug and EntityConflictError is // also the right surface — same shape as step_created's // claim-file behavior. + // + // A slot-numbered run collides by design: the id names a position in + // the log, so a loser is not a bug but a writer whose log was missing + // an event. It gets a SlotConflictError carrying that event instead + // (see below), and this write is the authority that decides it — the + // allocator's book is only ever a hint. // Last-instant re-validation for `hook_received` (see the acceptance // check above). The per-hook in-process lock already serializes // resume vs. dispose within one storage instance; this second check @@ -2235,115 +2573,178 @@ export function createEventsStorage( throw new HookNotFoundError(data.correlationId); } - const compositeKey = `${effectiveRunId}-${eventId}`; - const eventPath = taggedPath(basedir, 'events', compositeKey, tag); - // Capture the serialized payload before the write's `await` so the - // cached snapshot can't observe a later mutation (see - // rememberStoredEvent). - const serializedEvent = JSON.stringify(event, jsonReplacer, 2); - - // Cross-process terminal-run guard for `hook_received`. A terminal - // transition (run_completed / run_failed / run_cancelled) in ANY - // process (1) publishes a durable `runTerminalMarkerPath` marker and - // (2) reaps the run's staged hook_received events, both BEFORE it - // writes the terminal run state or appends its terminal event (see - // the terminal-transition block earlier in this function). In-memory - // locks cannot close the shared-filesystem race this backend - // explicitly supports, and a published event file is immediately - // visible to `events.list()` in other processes — so it can never be - // "rolled back" after the fact. Instead, the event stays INVISIBLE - // to readers until a single atomic filesystem operation decides its - // fate: - // - // 1. (fast path) reject if the run is already terminal — by - // marker, or by run state for runs that predate the marker — - // so the common case never creates a file. - // 2. STAGE the event at a non-reader-visible path under `.locks`. - // 3. re-CHECK the terminal marker; reject if present. - // 4. PROMOTE the staged file into `events/` with an atomic hard - // link; reject if the staged file was reaped (`'missing'`). - // - // Correctness: the reap's `unlink` and step 4's `link` target the - // same staged file, so the filesystem serializes them — exactly one - // wins. If the link wins, the event was reader-visible before the - // reap completed, and therefore before the terminal state and - // terminal event were written: acceptance happened-before the - // termination and legitimately precedes it. If the unlink wins, - // promotion fails and the event is never visible to any reader — - // there is nothing to roll back. A resume that stages after the - // reap has passed necessarily stages after the marker was - // committed, so step 3 rejects it. Rejections before step 4 unlink - // a file no reader can see. - let eventPublished: boolean; - if (data.eventType === 'hook_received') { - // Step 1: fast path. The marker is the authoritative durable - // signal; the run-state read additionally rejects runs whose - // terminal state was written without a marker (e.g. runs that - // terminated on an older storage version). - const terminalByMarker = await isRunTerminalCommitted( - basedir, - effectiveRunId, - tag - ); - const runNow = terminalByMarker - ? null - : await readJSONWithFallback( - basedir, - 'runs', - effectiveRunId, - WorkflowRunSchema, - tag - ); - if ( - terminalByMarker || - (runNow && isTerminalWorkflowRunStatus(runNow.status)) - ) { - throw new RunExpiredError( - `Workflow run "${effectiveRunId}" is already in a terminal state` - ); - } - - const stagedPath = pendingHookEventPath( - basedir, - effectiveRunId, - eventId, - tag - ); - const staged = await writeExclusive(stagedPath, serializedEvent); - if (!staged) { - // eventId is a freshly generated ULID; its staging path can - // only be occupied by a previous crashed attempt of this very - // event, which never promoted. Surface the same conflict shape - // as a visible-path collision. - throw new EntityConflictError( - `Event "${eventId}" already exists for run "${effectiveRunId}"` + /** + * One attempt at publishing the event at the position `eventId` + * currently names: `true` when this call made it reader-visible, + * `false` when the position was already taken. What a loss means is the + * loop's decision — a position this world allocated is simply retried + * one higher, a position the caller claimed is a conflict it has to + * resolve. + */ + async function publishOnce(): Promise { + // Cross-process terminal-run guard for `hook_received`. A terminal + // transition (run_completed / run_failed / run_cancelled) in ANY + // process (1) publishes a durable `runTerminalMarkerPath` marker and + // (2) reaps the run's staged hook_received events, both BEFORE it + // writes the terminal run state or appends its terminal event (see + // the terminal-transition block earlier in this function). In-memory + // locks cannot close the shared-filesystem race this backend + // explicitly supports, and a published event file is immediately + // visible to `events.list()` in other processes — so it can never be + // "rolled back" after the fact. Instead, the event stays INVISIBLE + // to readers until a single atomic filesystem operation decides its + // fate: + // + // 1. (fast path) reject if the run is already terminal — by + // marker, or by run state for runs that predate the marker — + // so the common case never creates a file. + // 2. STAGE the event at a non-reader-visible path under `.locks`. + // 3. re-CHECK the terminal marker; reject if present. + // 4. PROMOTE the staged file into `events/` with an atomic hard + // link; reject if the staged file was reaped (`'missing'`). + // + // Correctness: the reap's `unlink` and step 4's `link` target the + // same staged file, so the filesystem serializes them — exactly one + // wins. If the link wins, the event was reader-visible before the + // reap completed, and therefore before the terminal state and + // terminal event were written: acceptance happened-before the + // termination and legitimately precedes it. If the unlink wins, + // promotion fails and the event is never visible to any reader — + // there is nothing to roll back. A resume that stages after the + // reap has passed necessarily stages after the marker was + // committed, so step 3 rejects it. Rejections before step 4 unlink + // a file no reader can see. + if (data.eventType === 'hook_received') { + // Step 1: fast path. The marker is the authoritative durable + // signal; the run-state read additionally rejects runs whose + // terminal state was written without a marker (e.g. runs that + // terminated on an older storage version). + const terminalByMarker = await isRunTerminalCommitted( + basedir, + effectiveRunId, + tag ); - } - try { - if (await isRunTerminalCommitted(basedir, effectiveRunId, tag)) { + const runNow = terminalByMarker + ? null + : await readJSONWithFallback( + basedir, + 'runs', + effectiveRunId, + WorkflowRunSchema, + tag + ); + if ( + terminalByMarker || + (runNow && isTerminalWorkflowRunStatus(runNow.status)) + ) { throw new RunExpiredError( `Workflow run "${effectiveRunId}" is already in a terminal state` ); } - const promoted = await promoteExclusive(stagedPath, eventPath); - if (promoted === 'missing') { - // A terminal transition reaped the staged file between the - // check and the link — the atomic loss of the arbitration. - throw new RunExpiredError( - `Workflow run "${effectiveRunId}" is already in a terminal state` + + const stagedPath = pendingHookEventPath( + basedir, + effectiveRunId, + eventId, + tag + ); + const staged = await writeExclusive(stagedPath, serializedEvent); + if (!staged) { + // For a ULID-numbered run the eventId is freshly generated, so + // its staging path can only be occupied by a previous crashed + // attempt of this very event, which never promoted. A + // slot-numbered run can also collide here with another instance + // that allocated the same slot from its own book. Either way the + // event is not reader-visible, so there is no delta to hand back + // and nothing for the caller to merge: surface the same conflict + // shape as a visible-path collision — or, when this world + // allocated the position itself, let the loop below re-probe and + // take the next one. + if (reallocatesSlot) { + return false; + } + throw new EntityConflictError( + `Event "${eventId}" already exists for run "${effectiveRunId}"` ); } - eventPublished = promoted === 'linked'; - } finally { - // The staged path is not reader-visible; removing it is pure - // cleanup on every outcome (already gone when reaped). - await deleteJSON(stagedPath).catch(() => {}); + try { + if (await isRunTerminalCommitted(basedir, effectiveRunId, tag)) { + throw new RunExpiredError( + `Workflow run "${effectiveRunId}" is already in a terminal state` + ); + } + const promoted = await promoteExclusive(stagedPath, eventPath); + if (promoted === 'missing') { + // A terminal transition reaped the staged file between the + // check and the link — the atomic loss of the arbitration. + throw new RunExpiredError( + `Workflow run "${effectiveRunId}" is already in a terminal state` + ); + } + return promoted === 'linked'; + } finally { + // The staged path is not reader-visible; removing it is pure + // cleanup on every outcome (already gone when reaped). + await deleteJSON(stagedPath).catch(() => {}); + } } - } else { - eventPublished = await writeExclusive(eventPath, serializedEvent); + return await writeExclusive(eventPath, serializedEvent); } - if (!eventPublished) { + // A write that allocated its own position may take the next free one + // when it loses: nothing outside this world named the slot, so which + // position the event lands on is this world's business, and the caller + // — a step reporting its completion, a hook being received — has no log + // to reconcile. A write whose position the *caller* claimed may not: + // the claim asserts a log complete up to that position, so losing it + // means that log is stale and only the caller can resolve it. + // `run_created` is excluded even though it named its own position: the + // first slot is the only position it can ever occupy, so losing it means + // the run already has a creation event, and appending a second one above + // it would be worse than the duplicate the publish is reporting. + const reallocatesSlot = + slotMode && params?.eventId === undefined && !ownsFirstSlot; + const slotDeadline = Date.now() + SLOT_RETRY_BUDGET_MS; + let compositeKey = ''; + let eventPath = ''; + let serializedEvent = ''; + let eventPublished = false; + + for (let round = 0; ; round++) { + compositeKey = `${effectiveRunId}-${eventId}`; + eventPath = taggedPath(basedir, 'events', compositeKey, tag); + // Capture the serialized payload before the write's `await` so the + // cached snapshot can't observe a later mutation (see + // rememberStoredEvent). + serializedEvent = JSON.stringify(event, jsonReplacer, 2); + eventPublished = await publishOnce(); + if (eventPublished) { + break; + } + if (reallocatesSlot && Date.now() < slotDeadline) { + // The position is someone else's — either published there or + // staged for it. Record that, top the book up from disk, and try + // again above whatever the log has reached rather than surfacing a + // conflict the caller cannot act on. Re-reading rather than + // incrementing bounds the search: every round at least one writer + // wins, so the top of the log is never further than the number of + // writers still contending for it. + const lost = slotFromId(eventId); + if (lost !== undefined) { + reserved.delete(lost); + } + slots.observe(effectiveRunId, eventId); + await slots.refresh(effectiveRunId); + await new Promise((resolve) => + setTimeout(resolve, slotRetryDelay(round)) + ); + const slot = await slots.reserve(effectiveRunId); + reservedRunId = effectiveRunId; + reserved.add(slot); + eventId = slotEventId(slot); + event = { ...event, eventId }; + continue; + } // For `hook_created`, losing the event publish means the // event was already committed at this exact (canonical) // path. The original publisher may have crashed between @@ -2364,14 +2765,54 @@ export function createEventsStorage( tag ); } + if (reallocatesSlot) { + // Out of budget: every position this writer tried was taken by + // someone else. Surfacing it as a 503 puts the whole operation + // back on the queue rather than stalling the run here. + throw new WorkflowWorldError( + `Could not place an event in run "${effectiveRunId}" within ${SLOT_RETRY_BUDGET_MS}ms of contention`, + { status: 503 } + ); + } + if (slotMode) { + // Losing a claimed slot means someone else's event occupies this + // position, so the log this event was derived from is missing at + // least that event — the whole proposed event is stale, not just + // its id. Hand back what the caller is missing so it can merge, + // replay and re-propose, and forget the run's book so the next + // allocation re-reads the log this instance evidently does not + // have. + // + // Reaching here means the slot was taken *after* the pre-check at + // the claim site, so the entity this event was going to describe + // has already been materialized. Only two storage instances + // sharing a directory can do that, since one instance's book + // hands the same slot to nobody else. + slots.forget(effectiveRunId); + throw await slotConflict(effectiveRunId, eventId, params); + } throw new EntityConflictError( `Event "${eventId}" already exists for run "${effectiveRunId}"` ); } // The event is now committed; cache it so an immediate sequential - // replay can serve it without rereading from disk. + // replay can serve it without rereading from disk. Nothing this create + // claimed may be undone from here on: readers can see the event, so the + // entity it describes has to keep existing even if a later step of this + // call fails. + eventCommitted = true; rememberStoredEvent(event, eventPath, serializedEvent); + slots.observe(effectiveRunId, eventId); + if (companionSlot !== undefined) { + // A start that carried creation data for a step that already existed + // synthesized no `step_created`, so the position below it went + // unused. Hand it back instead of leaving it outstanding for the life + // of the process, where it would block the allocator from ever + // filling that position. + reserved.delete(companionSlot); + slots.release(effectiveRunId, companionSlot); + } // Write the hook entity ONLY now that the event publish has // committed. Doing this earlier (in the `hook_created` @@ -2409,6 +2850,7 @@ export function createEventsStorage( sortOrder: 'asc', limit: 1000, getCreatedAt: getObjectCreatedAt('evnt'), + getOrderTime: eventOrderTime, getId: (e) => e.eventId, }); events = allEvents.data; @@ -2457,6 +2899,7 @@ export function createEventsStorage( sortOrder: 'asc', cursor: params.sinceCursor, getCreatedAt: getObjectCreatedAt('evnt'), + getOrderTime: eventOrderTime, getId: (e) => e.eventId, }); events = @@ -2517,6 +2960,7 @@ export function createEventsStorage( limit: params.pagination?.limit, cursor: params.pagination?.cursor, getCreatedAt: getObjectCreatedAt('evnt'), + getOrderTime: eventOrderTime, getId: (event) => event.eventId, }); @@ -2549,6 +2993,7 @@ export function createEventsStorage( limit: params.pagination?.limit, cursor: params.pagination?.cursor, getCreatedAt: getObjectCreatedAt('evnt'), + getOrderTime: eventOrderTime, getId: (event) => event.eventId, }); diff --git a/packages/world-local/src/storage/helpers.ts b/packages/world-local/src/storage/helpers.ts index d7c2cc78bb..2817fb0574 100644 --- a/packages/world-local/src/storage/helpers.ts +++ b/packages/world-local/src/storage/helpers.ts @@ -1,6 +1,7 @@ import { createHash } from 'node:crypto'; import fs from 'node:fs/promises'; import path from 'node:path'; +import { FIRST_SLOT, maxSlotOf, slotEventId } from '@workflow/world'; import { decodeTime, monotonicFactory } from 'ulid'; import { hasTag, @@ -208,6 +209,46 @@ export async function reapPendingHookEvents( } } +/** + * The event ids of `runId` that are visible in the given tag's view, read from + * the event filenames alone — no file contents, so the cost is one `readdir` + * however large the log is. + * + * A missing `events` directory means the run provably has no events yet. Any + * other failure is thrown: callers derive an event key from this scan, and a + * silently short answer would mint a key that collides with, or fails to + * dominate, an event that is actually there. + */ +export async function listRunEventIds( + basedir: string, + runId: string, + tag?: string +): Promise { + let files: string[] = []; + try { + files = await fs.readdir(path.join(basedir, 'events')); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') { + throw error; + } + } + const prefix = `${runId}-`; + const eventIds: string[] = []; + for (const file of files) { + if (!file.startsWith(prefix) || !file.endsWith('.json')) { + continue; + } + const fileId = file.slice(0, -'.json'.length); + // Mirror read visibility: untagged files are visible to every tag, + // tagged files only to their own tag. + if (!isUntagged(fileId) && !(tag && hasTag(fileId, tag))) { + continue; + } + eventIds.push(stripTag(fileId).slice(prefix.length)); + } + return eventIds; +} + /** * Mint an event key (eventId + createdAt) that sorts strictly AFTER every * reader-visible event of the run in the given tag's view. @@ -229,38 +270,35 @@ export async function reapPendingHookEvents( * >= every visible event's `createdAt`, which was stamped at that event's * `createImpl()` entry — before its publish, and thus before this call. * Equal-`createdAt` ties fall to the strictly-dominant eventId. + * + * A slot-numbered run takes the slot above the highest visible one, which + * dominates by construction, paired with the wall clock — `createdAt` needs + * only to be >= every visible one, by the same argument as above. This is + * the one allocation that deliberately does *not* fill a hole below the max: + * a lower slot would sort before the events it has to follow, and density + * matters less here than replay order, since a hole below a terminal event + * means the run already lost an event it can never write. */ export async function mintRunDominantEventKey( basedir: string, runId: string, - tag?: string + tag: string | undefined, + slotMode: boolean ): Promise<{ eventId: string; createdAt: Date }> { - let files: string[] = []; - try { - files = await fs.readdir(path.join(basedir, 'events')); - } catch (error) { - // Only ENOENT ("no events directory yet") means there is provably - // nothing visible to dominate. Any other failure would silently mint a - // wall-clock key with no dominance guarantee over an already-accepted - // hook — abort the terminal transition instead; its retry re-runs this - // scan. - if ((error as NodeJS.ErrnoException).code !== 'ENOENT') { - throw error; - } + const eventIds = await listRunEventIds(basedir, runId, tag); + if (slotMode) { + // Above every event on disk, and above the run's own first slot even when + // that event has not landed yet: only `run_created` may occupy it, and a + // terminal event is never the run's first. + return { + eventId: slotEventId( + Math.max(maxSlotOf(eventIds.map(toEventRef)), FIRST_SLOT) + 1 + ), + createdAt: new Date(), + }; } - const prefix = `${runId}-`; let maxUlid: string | null = null; - for (const file of files) { - if (!file.startsWith(prefix) || !file.endsWith('.json')) { - continue; - } - const fileId = file.slice(0, -'.json'.length); - // Mirror read visibility: untagged files are visible to every tag, - // tagged files only to their own tag. - if (!isUntagged(fileId) && !(tag && hasTag(fileId, tag))) { - continue; - } - const candidate = stripTag(fileId).slice(prefix.length); + for (const candidate of eventIds) { if (!maxUlid || candidate > maxUlid) { maxUlid = candidate; } @@ -279,6 +317,10 @@ export async function mintRunDominantEventKey( return { eventId: `evnt_${monotonicUlid(ts)}`, createdAt: new Date(ts) }; } +function toEventRef(eventId: string): { eventId: string } { + return { eventId }; +} + /** * Path of the exclusive-create claim file that reserves a hook token. */ diff --git a/packages/world-local/src/storage/slot-identity.test.ts b/packages/world-local/src/storage/slot-identity.test.ts new file mode 100644 index 0000000000..b7c579e557 --- /dev/null +++ b/packages/world-local/src/storage/slot-identity.test.ts @@ -0,0 +1,464 @@ +import { promises as fs } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { SlotConflictError } from '@workflow/errors'; +import type { Storage } from '@workflow/world'; +import { + FIRST_SLOT, + maxSlotOf, + SPEC_VERSION_CURRENT, + SPEC_VERSION_SLOT_IDENTITY, + slotEventId, + slotFromId, +} from '@workflow/world'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { createStorage } from './index.js'; + +let testDir: string; +let storage: Storage; + +beforeEach(async () => { + testDir = await fs.mkdtemp(path.join(os.tmpdir(), 'slot-identity-')); + storage = createStorage(testDir); +}); + +afterEach(async () => { + await fs.rm(testDir, { recursive: true, force: true }); +}); + +/** Start a run whose events are numbered by slot, and return its id. */ +async function newSlotRun(): Promise { + const result = await storage.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + eventData: { + deploymentId: 'dpl_test', + workflowName: 'test-workflow', + input: new Uint8Array(), + }, + }); + if (!result.run) { + throw new Error('Expected run to be created'); + } + return result.run.runId; +} + +/** + * The slots of the run's log, in list order. The page size is explicit: the + * default would silently truncate a fan-out and make a dense log look sparse. + */ +async function slotsOf(runId: string): Promise { + const { data } = await eventsOf(runId); + return data.map((event) => slotFromId(event.eventId) ?? -1); +} + +function eventsOf(runId: string) { + return storage.events.list({ runId, pagination: { limit: 500 } }); +} + +async function createStep( + runId: string, + stepId: string, + eventId?: string +): Promise { + const result = await storage.events.create( + runId, + { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: stepId, + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }, + eventId === undefined ? undefined : { eventId } + ); + if (!result.event) { + throw new Error('Expected an event'); + } + return result.event.eventId; +} + +async function createWait( + runId: string, + waitId: string, + eventId?: string +): Promise { + const result = await storage.events.create( + runId, + { + eventType: 'wait_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: waitId, + eventData: { resumeAt: new Date('2030-01-01T00:00:00.000Z') }, + }, + eventId === undefined ? undefined : { eventId } + ); + if (!result.event) { + throw new Error('Expected an event'); + } + return result.event.eventId; +} + +describe('numbering', () => { + it('puts run_created in the first slot', async () => { + const runId = await newSlotRun(); + await expect(slotsOf(runId)).resolves.toEqual([FIRST_SLOT]); + }); + + it('allocates dense slots for writers that hold no log', async () => { + // A step completion reporting in, a cancellation from an API call: the + // caller has no event log, so the world numbers the event for it. + const runId = await newSlotRun(); + await createStep(runId, 'step_a'); + await createStep(runId, 'step_b'); + await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3]); + }); + + it('honours a slot the caller claims', async () => { + const runId = await newSlotRun(); + const eventId = await createStep(runId, 'step_a', slotEventId(2)); + expect(eventId).toBe(slotEventId(2)); + await expect(slotsOf(runId)).resolves.toEqual([1, 2]); + }); + + it('lists the log in slot order, not in the order writes started', async () => { + // A writer that loses its slot re-proposes above the winner while keeping + // the wall-clock stamp it started with, so `createdAt` order and slot order + // disagree. Replay consumes the log in list order, so list order has to be + // slot order — what the sort key gives the other backends for free. + const runId = await newSlotRun(); + await createStep(runId, 'step_late', slotEventId(3)); + await createStep(runId, 'step_early', slotEventId(2)); + await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3]); + }); + + it('keeps a burst of concurrent writers dense', async () => { + // The suspension flush issues every op at once. Density is what lets a + // reader prove its log is complete, so a burst must not leave holes. + const runId = await newSlotRun(); + const ids = await Promise.all( + Array.from({ length: 20 }, (_, index) => + createStep(runId, `step_${index}`) + ) + ); + expect(new Set(ids).size).toBe(ids.length); + const slots = await slotsOf(runId); + expect([...slots].sort((a, b) => a - b)).toEqual( + Array.from({ length: ids.length + 1 }, (_, index) => FIRST_SLOT + index) + ); + }); + + it('proves completeness: the highest slot is the event count', async () => { + const runId = await newSlotRun(); + await Promise.all( + Array.from({ length: 5 }, (_, index) => + createStep(runId, `step_${index}`) + ) + ); + const { data } = await eventsOf(runId); + expect(maxSlotOf(data)).toBe(data.length); + }); + + it('leaves a rejected write’s position unused instead of recycling it', async () => { + // The rejected op's position sits below its concurrent sibling's, so handing + // it to the next writer would order that writer's event below one that + // already published. The hole costs a reader the density proof; the + // inversion would cost the run. + const runId = await newSlotRun(); + const [rejected, accepted] = await Promise.allSettled([ + storage.events.create(runId, { + eventType: 'step_completed', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_never_created', + eventData: { output: new Uint8Array() }, + }), + createStep(runId, 'step_a'), + ]); + expect(rejected.status).toBe('rejected'); + expect(accepted.status).toBe('fulfilled'); + await createStep(runId, 'step_b'); + const slots = await slotsOf(runId); + expect(slots).toHaveLength(3); + expect(slots[0]).toBe(FIRST_SLOT); + // Both concurrent writers took a position, one abandoned its own, and the + // third write went above them both. + expect(slots[2]).toBe(FIRST_SLOT + 3); + expect(slots[1]).toBeGreaterThan(slots[0]); + expect(slots[1]).toBeLessThan(slots[2]); + }); +}); + +/** + * A lazy step start: a `step_started` carrying the step's creation data, which + * the world materializes into a step plus the `step_created` event the caller + * deferred — one request, two events. + */ +async function startStepLazily( + runId: string, + stepId: string, + eventId?: string +): Promise { + const result = await storage.events.create( + runId, + { + eventType: 'step_started', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: stepId, + eventData: { stepName: 'a-step', input: new Uint8Array(), attempt: 0 }, + }, + eventId === undefined ? undefined : { eventId } + ); + if (!result.event) { + throw new Error('Expected an event'); + } + return result.event.eventId; +} + +describe('a write that publishes two events', () => { + it('numbers the deferred step_created below the claim', async () => { + // The caller reserves both positions and names only the top one, so the + // pair is fixed before either lands — which is what keeps it off the slot + // the next write of the same batch is holding. + const runId = await newSlotRun(); + const startedEventId = await startStepLazily( + runId, + 'step_a', + slotEventId(3) + ); + expect(startedEventId).toBe(slotEventId(3)); + await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3]); + }); + + it('allocates both positions for a start that claims neither', async () => { + const runId = await newSlotRun(); + await startStepLazily(runId, 'step_a'); + await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3]); + }); + + it('keeps every claim in a burst of lazy starts', async () => { + // The suspension flush issues its lazy starts at once, each having reserved + // two positions. A second event numbered off the log as this world sees it + // would take the slot the next start in the batch claimed, and cost every + // start after the first its claim — collapsing the fan-out to one step. + const runId = await newSlotRun(); + const claims = Array.from({ length: 10 }, (_, index) => + slotEventId(FIRST_SLOT + 2 * (index + 1)) + ); + const ids = await Promise.all( + claims.map((eventId, index) => + startStepLazily(runId, `step_${index}`, eventId) + ) + ); + expect(ids).toEqual(claims); + const slots = await slotsOf(runId); + expect([...slots].sort((a, b) => a - b)).toEqual( + Array.from({ length: 2 * claims.length + 1 }, (_, i) => FIRST_SLOT + i) + ); + }); + + it('rejects a claim that leaves no room for the second event', async () => { + // The run's own run_created holds the first slot, so a claim of the second + // means the caller reserved one position for a write that publishes two. + const runId = await newSlotRun(); + await expect( + startStepLazily(runId, 'step_a', slotEventId(FIRST_SLOT + 1)) + ).rejects.toThrow(/leaves no slot below it/); + }); +}); + +describe('mode is pinned to the run', () => { + it('rejects a slot id claimed on a ULID-numbered run', async () => { + const created = await storage.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'dpl_test', + workflowName: 'test-workflow', + input: new Uint8Array(), + }, + }); + const runId = created.run?.runId as string; + await expect(createStep(runId, 'step_a', slotEventId(2))).rejects.toThrow( + /not numbered by slot/ + ); + }); + + it('rejects a ULID id claimed on a slot-numbered run', async () => { + const runId = await newSlotRun(); + await expect( + createStep(runId, 'step_a', 'evnt_01K5Z0000000000000000000AA') + ).rejects.toThrow(/not a slot id/); + }); + + it('ignores the spec version of later requests', async () => { + // A run is in exactly one mode for life; only what was persisted decides. + const runId = await newSlotRun(); + const result = await storage.events.create(runId, { + eventType: 'step_created', + specVersion: SPEC_VERSION_CURRENT, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }); + expect(slotFromId(result.event?.eventId ?? '')).toBe(2); + }); +}); + +describe('conflict', () => { + it('reports the events the loser is missing', async () => { + const runId = await newSlotRun(); + // Out of band: something else takes the slot this caller was about to + // claim, so the caller's log is provably missing an event. + await createStep(runId, 'step_out_of_band'); + + const conflict = await storage.events + .create( + runId, + { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }, + { eventId: slotEventId(2), maxSlot: 1 } + ) + .catch((error: unknown) => error); + + expect(SlotConflictError.is(conflict)).toBe(true); + const slotConflict = conflict as SlotConflictError; + expect(slotConflict.status).toBe(409); + expect(slotConflict.eventId).toBe(slotEventId(2)); + expect(slotConflict.events?.map((event) => event.eventId)).toEqual([ + slotEventId(2), + ]); + }); + + it('lets the loser re-propose at the next free slot', async () => { + const runId = await newSlotRun(); + await createStep(runId, 'step_out_of_band'); + await expect(createStep(runId, 'step_a', slotEventId(2))).rejects.toThrow( + SlotConflictError + ); + // Merging the delta moves the caller's own numbering forward by one. + const eventId = await createStep(runId, 'step_a', slotEventId(3)); + expect(eventId).toBe(slotEventId(3)); + await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3]); + }); + + it('excludes events the loser already holds from the delta', async () => { + const runId = await newSlotRun(); + await createStep(runId, 'step_one'); + await createStep(runId, 'step_two'); + + const conflict = await storage.events + .create( + runId, + { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }, + { eventId: slotEventId(2), maxSlot: 2 } + ) + .catch((error: unknown) => error); + + // Slots 1 and 2 are at or below what the caller had; only 3 is news. + expect( + (conflict as SlotConflictError).events?.map((event) => event.eventId) + ).toEqual([slotEventId(3)]); + }); + + it('conflicts when another instance takes a claimed slot', async () => { + // Two instances keep independent books, so the exclusive write — not the + // book — is what decides who owns a slot. A claim asserts a complete log, + // so its loser has to reload rather than move over. + const runId = await newSlotRun(); + const other = createStorage(testDir); + await other.events.create(runId, { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_b', + eventData: { stepName: 'b-step', input: new Uint8Array() }, + }); + await expect(createStep(runId, 'step_a', slotEventId(2))).rejects.toThrow( + SlotConflictError + ); + await expect(slotsOf(runId)).resolves.toEqual([1, 2]); + }); + + it('lets a lost claim re-propose an entity it had already materialized', async () => { + // A claim only reaches its exclusive write after the entity it describes + // exists, so a claim that loses leaves that entity behind. The caller's + // whole answer to a conflict is to merge, replay and propose the same + // operation one position higher — which it cannot do if its own leftover + // entity is what rejects the retry. + const runId = await newSlotRun(); + // Seed this instance's book, then let another instance take the position + // the book will hand out next. The claim below passes the book's + // "is it written?" check because the book has not seen that write. + await createStep(runId, 'step_seed'); + const other = createStorage(testDir); + await other.events.create(runId, { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_out_of_band', + eventData: { stepName: 'b-step', input: new Uint8Array() }, + }); + + await expect(createWait(runId, 'wait_a', slotEventId(3))).rejects.toThrow( + SlotConflictError + ); + const eventId = await createWait(runId, 'wait_a', slotEventId(4)); + expect(eventId).toBe(slotEventId(4)); + await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3, 4]); + }); + + it('lets a lazy start lose the position of the event it defers', async () => { + // The deferred `step_created` is published on the same terms as the start + // itself, so it is the pair's first position that can be lost. The retry has + // to be able to start the step lazily all over again — its own claim file + // and step entity would otherwise answer for a write that never landed. + const runId = await newSlotRun(); + await createStep(runId, 'step_seed'); + const other = createStorage(testDir); + await other.events.create(runId, { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_out_of_band', + eventData: { stepName: 'b-step', input: new Uint8Array() }, + }); + + await expect( + startStepLazily(runId, 'step_a', slotEventId(4)) + ).rejects.toThrow(SlotConflictError); + const eventId = await startStepLazily(runId, 'step_a', slotEventId(5)); + expect(eventId).toBe(slotEventId(5)); + await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3, 4, 5]); + }); + + it('reallocates around another instance holding the slot it picked', async () => { + // Neither writer holds a log, so neither has anything to reconcile: the + // loser takes the next free position instead of surfacing a conflict its + // caller could not act on. + const runId = await newSlotRun(); + const other = createStorage(testDir); + const outcomes = await Promise.allSettled([ + storage.events.create(runId, { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }), + other.events.create(runId, { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_b', + eventData: { stepName: 'b-step', input: new Uint8Array() }, + }), + ]); + expect(outcomes.map((outcome) => outcome.status)).toEqual([ + 'fulfilled', + 'fulfilled', + ]); + await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3]); + }); +}); diff --git a/packages/world-local/src/storage/slots.test.ts b/packages/world-local/src/storage/slots.test.ts new file mode 100644 index 0000000000..d7b87f9e10 --- /dev/null +++ b/packages/world-local/src/storage/slots.test.ts @@ -0,0 +1,274 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { + SPEC_VERSION_CURRENT, + SPEC_VERSION_SLOT_IDENTITY, + slotEventId, +} from '@workflow/world'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { createSlotBook, RUN_CREATED_SLOT } from './slots.js'; + +let basedir: string; + +beforeEach(async () => { + basedir = await fs.mkdtemp(path.join(os.tmpdir(), 'slot-book-')); +}); + +afterEach(async () => { + await fs.rm(basedir, { recursive: true, force: true }); +}); + +const RUN_ID = 'wrun_01K0000000000000000000TEST'; + +async function writeRun(specVersion: number): Promise { + await fs.mkdir(path.join(basedir, 'runs'), { recursive: true }); + await fs.writeFile( + path.join(basedir, 'runs', `${RUN_ID}.json`), + JSON.stringify({ + runId: RUN_ID, + deploymentId: 'dpl_test', + status: 'running', + workflowName: 'test', + specVersion, + input: [], + attributes: {}, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + }) + ); +} + +async function writeEvents(...slots: number[]): Promise { + await fs.mkdir(path.join(basedir, 'events'), { recursive: true }); + for (const slot of slots) { + await fs.writeFile( + path.join(basedir, 'events', `${RUN_ID}-${slotEventId(slot)}.json`), + '{}' + ); + } +} + +describe('usesSlots', () => { + it('reads the mode off the persisted run, not the build', async () => { + await writeRun(SPEC_VERSION_SLOT_IDENTITY); + await expect(createSlotBook(basedir).usesSlots(RUN_ID)).resolves.toBe(true); + + await writeRun(SPEC_VERSION_CURRENT); + await expect(createSlotBook(basedir).usesSlots(RUN_ID)).resolves.toBe( + false + ); + }); + + it('re-reads until the run exists', async () => { + // The resilient-start path writes run_started before the run entity, so a + // cached "no" taken from the missing run would strand a slot-numbered run + // on ULID ids for the rest of the process's life. + const book = createSlotBook(basedir); + await expect(book.usesSlots(RUN_ID)).resolves.toBe(false); + await writeRun(SPEC_VERSION_SLOT_IDENTITY); + await expect(book.usesSlots(RUN_ID)).resolves.toBe(true); + }); + + it('prefers its own tagged run over the untagged one', async () => { + await writeRun(SPEC_VERSION_SLOT_IDENTITY); + await fs.rename( + path.join(basedir, 'runs', `${RUN_ID}.json`), + path.join(basedir, 'runs', `${RUN_ID}.mine.json`) + ); + await writeRun(SPEC_VERSION_CURRENT); + + await expect( + createSlotBook(basedir, 'mine').usesSlots(RUN_ID) + ).resolves.toBe(true); + await expect( + createSlotBook(basedir, 'other').usesSlots(RUN_ID) + ).resolves.toBe(false); + }); +}); + +describe('reserve', () => { + it('starts above the slot the run’s own creation event owns', async () => { + // `run_created` takes the first slot outright — nothing can precede it — so + // an allocation never hands that position out. + await expect(createSlotBook(basedir).reserve(RUN_ID)).resolves.toBe( + RUN_CREATED_SLOT + 1 + ); + }); + + it('continues above the highest slot already on disk', async () => { + await writeEvents(1, 2, 3); + await expect(createSlotBook(basedir).reserve(RUN_ID)).resolves.toBe(4); + }); + + it('leaves a hole in the persisted log unfilled', async () => { + // Allocation is append-only: the free position sits below a published event, + // and an event placed there would order before one that already happened. + await writeEvents(1, 3); + const book = createSlotBook(basedir); + await expect(book.reserve(RUN_ID)).resolves.toBe(4); + await expect(book.reserve(RUN_ID)).resolves.toBe(5); + }); + + it('stays above published events when a lower position comes free', async () => { + // The corruption this rules out: a step completion allocating late, dropping + // into a hole, and landing below the step_started it reports on. Replay reads + // the log in slot order and cannot consume that. + const book = createSlotBook(basedir); + const abandoned = await book.reserve(RUN_ID); + const published = await book.reserve(RUN_ID); + book.observe(RUN_ID, slotEventId(published)); + book.release(RUN_ID, abandoned); + await expect(book.reserve(RUN_ID)).resolves.toBeGreaterThan(published); + }); + + it('hands a synchronous burst distinct consecutive slots', async () => { + // The suspension flush issues every op concurrently; a book that only moved + // on publish would give them all the same position and fail all but one. + const book = createSlotBook(basedir); + const slots = await Promise.all( + Array.from({ length: 20 }, () => book.reserve(RUN_ID)) + ); + expect([...slots].sort((a, b) => a - b)).toEqual( + Array.from({ length: 20 }, (_, index) => RUN_CREATED_SLOT + 1 + index) + ); + }); + + it('shares one disk scan across concurrent first callers', async () => { + await writeEvents(1); + const book = createSlotBook(basedir); + const slots = await Promise.all([ + book.reserve(RUN_ID), + book.reserve(RUN_ID), + ]); + expect([...slots].sort((a, b) => a - b)).toEqual([2, 3]); + }); + + it('honours a floor above where the book has reached', async () => { + // start() publishes the run entity before its `run_created` event and issues + // the queue send in parallel, so the delivery's `run_started` can allocate + // while slot 1 is still in flight. + const book = createSlotBook(basedir); + await expect(book.reserve(RUN_ID, 5)).resolves.toBe(5); + await expect(book.reserve(RUN_ID)).resolves.toBe(6); + }); + + it('keeps runs independent', async () => { + await writeEvents(1, 2); + const book = createSlotBook(basedir); + await expect(book.reserve(RUN_ID)).resolves.toBe(3); + await expect(book.reserve('wrun_01K0000000000000000000OTHR')).resolves.toBe( + RUN_CREATED_SLOT + 1 + ); + }); +}); + +describe('release', () => { + it('does not recycle an abandoned interior slot', async () => { + // The position may already sit below a sibling that published, and no + // caller can tell from here. A hole costs a reader its completeness proof; + // an inversion costs the run. + const book = createSlotBook(basedir); + const [first, second] = await Promise.all([ + book.reserve(RUN_ID), + book.reserve(RUN_ID), + ]); + book.release(RUN_ID, first); + await expect(book.reserve(RUN_ID)).resolves.toBe(second + 1); + }); + + it('does not resurrect a slot that was published', async () => { + const book = createSlotBook(basedir); + const slot = await book.reserve(RUN_ID); + book.observe(RUN_ID, slotEventId(slot)); + book.release(RUN_ID, slot); + await expect(book.reserve(RUN_ID)).resolves.toBe(slot + 1); + }); +}); + +describe('isWritten', () => { + it('reads the log to answer for a run it has never seen', async () => { + await writeEvents(1, 2); + const book = createSlotBook(basedir); + await expect(book.isWritten(RUN_ID, 2)).resolves.toBe(true); + await expect(book.isWritten(RUN_ID, 3)).resolves.toBe(false); + }); + + it('is false for a slot that is only reserved', async () => { + // A reservation is not a publish, so a caller claiming the slot has to be + // allowed through to the write that actually decides it. + const book = createSlotBook(basedir); + const slot = await book.reserve(RUN_ID); + await expect(book.isWritten(RUN_ID, slot)).resolves.toBe(false); + }); +}); + +describe('claim', () => { + it('holds a slot claimed before anything allocated for the run', async () => { + // A claim is synchronous and the first allocation's log scan is not, so a + // claim that only registered against an existing book would be invisible to + // the very allocation it races — and a single-process app would hand the + // caller's own position away. + await writeEvents(1); + const book = createSlotBook(basedir); + book.claim(RUN_ID, 2); + await expect(book.reserve(RUN_ID)).resolves.toBe(3); + }); + + it('stops holding back a claim that resolved', async () => { + // Released before anything allocated for the run, so no position in this + // instance was ever handed out above it and the log — which is the authority + // on what published — reaches only slot 1. Nothing can be inverted by + // seeding the book from disk alone. + await writeEvents(1); + const book = createSlotBook(basedir); + book.claim(RUN_ID, 2); + book.release(RUN_ID, 2); + await expect(book.reserve(RUN_ID)).resolves.toBe(2); + }); + + it('keeps holding a claim across a forget', async () => { + // `forget` follows a lost publish: the book is behind another writer, but + // the claims other writes in this instance still hold are not. + await writeEvents(1); + const book = createSlotBook(basedir); + book.claim(RUN_ID, 2); + book.forget(RUN_ID); + await expect(book.reserve(RUN_ID)).resolves.toBe(3); + }); +}); + +describe('observe', () => { + it('moves allocation above a position the client claimed', async () => { + const book = createSlotBook(basedir); + await book.reserve(RUN_ID); + book.observe(RUN_ID, slotEventId(5)); + await expect(book.reserve(RUN_ID)).resolves.toBe(6); + }); + + it('ignores ULID event ids', async () => { + const book = createSlotBook(basedir); + const slot = await book.reserve(RUN_ID); + book.observe(RUN_ID, 'evnt_01K5Z0000000000000000000AA'); + await expect(book.reserve(RUN_ID)).resolves.toBe(slot + 1); + }); +}); + +describe('forget', () => { + it("re-reads the log, picking up another writer's events", async () => { + const book = createSlotBook(basedir); + await expect(book.reserve(RUN_ID)).resolves.toBe(RUN_CREATED_SLOT + 1); + await writeEvents(1, 2, 3); + book.forget(RUN_ID); + await expect(book.reserve(RUN_ID)).resolves.toBe(4); + }); + + it('clear() forgets every run', async () => { + await writeEvents(1); + const book = createSlotBook(basedir); + await expect(book.reserve(RUN_ID)).resolves.toBe(2); + await writeEvents(2, 3); + book.clear(); + await expect(book.reserve(RUN_ID)).resolves.toBe(4); + }); +}); diff --git a/packages/world-local/src/storage/slots.ts b/packages/world-local/src/storage/slots.ts new file mode 100644 index 0000000000..25b2b8e522 --- /dev/null +++ b/packages/world-local/src/storage/slots.ts @@ -0,0 +1,316 @@ +/** + * Slot allocation for the Local World. + * + * A slot-numbered run names its events by position: `evnt_…001` is the first + * event of the run, `evnt_…002` the second. A replay reads the log in slot + * order, so the order slots are handed out in has to be an order some execution + * could have produced — which makes allocation strictly *append-only*: a slot is + * only ever handed out above every position this book has seen. + * + * Filling a hole is what that rules out, and it is worth naming why, because the + * alternative looks appealing (it keeps the log dense). A position left unwritten + * by an abandoned reservation sits below events that are already published. Hand + * it to the next caller and a `step_completed` lands below its own + * `step_started`; the replay reaches a completion for a step it has not started + * and diverges, and every later replay diverges the same way. A hole costs a + * reader the ability to prove its copy of the log is complete. An inversion + * costs the run. + * + * Three properties do the work: + * + * - Handing out a slot is a *synchronous* set operation, so concurrent + * callers in one process get distinct slots with no lock. The only await is + * seeding from disk, which is memoized per run. + * - An allocation picks the position above the highest one the book knows of, + * written or outstanding, and that ceiling never descends. A reservation that + * is abandoned (its create threw a validation error) leaves its position + * unused rather than being recycled below a sibling that already published. + * - The event publish is `writeExclusive`, which is the authority. The book is + * a hint: when it turns out to be stale (another process wrote the slot), + * the publish fails and the caller is told so, rather than a duplicate being + * written or a slot being skipped. + * + * The book is per storage instance, and two instances may share a data + * directory (the cross-process convergence tests rely on exactly that). Their + * books are then independent, and the loser of a collision gets a conflict it + * has to resolve by reloading — the same contract as the networked worlds. + */ + +import type { WorkflowRun } from '@workflow/world'; +import { + FIRST_SLOT, + slotFromId, + usesSlotIdentity, + WorkflowRunSchema, +} from '@workflow/world'; +import { readJSONWithFallback } from '../fs.js'; +import { listRunEventIds } from './helpers.js'; + +interface RunSlots { + /** Slots proven to be on disk. */ + written: Set; + /** Slots handed out whose publish has not resolved yet. */ + outstanding: Set; + /** + * The highest position this book has ever seen written, claimed or handed out. + * Allocation goes above it and it never descends, which is what keeps a + * released position from being recycled below events already published. + */ + ceiling: number; +} + +export interface SlotBook { + /** + * Whether `runId`'s events are numbered by slot, read from the run's + * persisted `specVersion` — never from the build, so a run stays in the mode + * it was created in for life. A run that does not exist yet is not + * slot-numbered, and that answer is not cached: the resilient-start path + * creates the run moments later, and caching "no" would strand it on ULIDs + * for the rest of this process's life. + */ + usesSlots(runId: string): Promise; + /** + * Reserves the position above every one this book knows of for `runId`, and at + * or above `minSlot`. Distinct for every concurrent caller; the publish still + * has to prove the position was actually free. + * + * `minSlot` defaults to the position above the run's first slot, which is + * reserved for its own `run_created`: that event needs no allocation, and it + * may not be on disk yet when a concurrent `run_started` allocates (start() + * issues the creation and the queue send in parallel, and the run entity is + * published before its event). + */ + reserve(runId: string, minSlot?: number): Promise; + /** + * Records that a caller claimed `slot` itself, so an allocation running + * alongside it picks a different one. Reserved and released on the same terms + * as {@link reserve}: the claim is only a hint until the publish proves it. + */ + claim(runId: string, slot: number): void; + /** + * Whether `slot` is already occupied by a published event, seeding from disk + * if this run has not been read yet. + * + * Lets a doomed claim be rejected *before* the create materializes its step, + * hook or wait: the entity mutation runs ahead of the event publish, so a + * claim that only fails at the publish leaves an entity behind with no event, + * and the caller's re-proposal at the next slot then collides with its own + * orphan. A `false` here is not a promise — the publish is still the + * authority — but it turns the case that actually happens (a caller numbering + * from a stale log) into a clean conflict. + */ + isWritten(runId: string, slot: number): Promise; + /** + * Forgets a reserved or claimed slot whose publish is never going to happen, + * so nothing waits on it. The position itself is not handed out again: it may + * already sit below a sibling that published, and recycling it there would put + * a later event below an earlier one. + */ + release(runId: string, slot: number): void; + /** Records a published event id, so it is never handed out again. */ + observe(runId: string, eventId: string): void; + /** + * Merges the run's published positions from disk into the book kept for it, + * leaving the reservations other writers in this instance still hold. + * + * A writer whose publish lost its position calls this before trying again: + * the book is demonstrably behind another instance's writes, and dropping it + * wholesale ({@link SlotBook.forget}) would hand a sibling's outstanding + * position to the next caller and cost that sibling its own publish. + */ + refresh(runId: string): Promise; + /** Drops what is cached for `runId`, so the next reservation re-reads disk. */ + forget(runId: string): void; + /** Drops everything cached (the data directory was cleared out from under us). */ + clear(): void; +} + +export function createSlotBook(basedir: string, tag?: string): SlotBook { + /** runId → whether the run is slot-numbered, memoized once it exists. */ + const modes = new Map(); + const books = new Map(); + /** runId → in-flight seed scan, so concurrent first callers share one scan. */ + const seeds = new Map>(); + /** + * runId → slots claimed while the run had no book yet, so the book the next + * allocation seeds starts out holding them. A claim is synchronous and a seed + * scan is not: without this, the first allocation of a run would read the log + * from disk and hand out a position a caller in this very instance had already + * claimed — the case that makes a claim lose in a single-process app. + */ + const claims = new Map>(); + + async function readMode(runId: string): Promise { + const run = await readJSONWithFallback( + basedir, + 'runs', + runId, + WorkflowRunSchema, + tag + ); + return run ? usesSlotIdentity(run.specVersion) : false; + } + + async function seed(runId: string): Promise { + const eventIds = await listRunEventIds(basedir, runId, tag); + const written = new Set(); + for (const eventId of eventIds) { + const slot = slotFromId(eventId); + if (slot !== undefined) { + written.add(slot); + } + } + const outstanding = new Set(claims.get(runId)); + const book: RunSlots = { + written, + outstanding, + ceiling: Math.max(FIRST_SLOT - 1, ...written, ...outstanding), + }; + books.set(runId, book); + return book; + } + + /** The run's book, seeding it from disk once for all concurrent callers. */ + function open(runId: string): RunSlots | Promise { + const known = books.get(runId); + if (known) { + return known; + } + let pending = seeds.get(runId); + if (!pending) { + pending = seed(runId).finally(() => seeds.delete(runId)); + seeds.set(runId, pending); + } + return pending; + } + + /** + * Drops a claim once its publish resolved, either way: a claim left behind + * would be handed to no one and become a hole in a log seeded later. + */ + function forgetClaim(runId: string, slot: number): void { + const claimed = claims.get(runId); + if (!claimed) { + return; + } + claimed.delete(slot); + if (claimed.size === 0) { + claims.delete(runId); + } + } + + function take(book: RunSlots, minSlot: number): number { + const slot = Math.max(book.ceiling + 1, minSlot); + book.outstanding.add(slot); + book.ceiling = slot; + return slot; + } + + return { + async usesSlots(runId) { + const cached = modes.get(runId); + if (cached !== undefined) { + return cached; + } + const mode = await readMode(runId); + // `false` here can mean "run not created yet" as well as "ULID run", and + // only the run's own absence is transient — so remember the positive + // answer eagerly and re-read until the run exists. + if (mode) { + modes.set(runId, true); + } + return mode; + }, + + async reserve(runId, minSlot = RUN_CREATED_SLOT + 1) { + const opened = open(runId); + // Awaiting a book that is already in hand would yield to the microtask + // queue and let a concurrent caller take the same slot. + return take(opened instanceof Promise ? await opened : opened, minSlot); + }, + + claim(runId, slot) { + const claimed = claims.get(runId); + if (claimed) { + claimed.add(slot); + } else { + claims.set(runId, new Set([slot])); + } + const book = books.get(runId); + if (book) { + book.outstanding.add(slot); + book.ceiling = Math.max(book.ceiling, slot); + } + }, + + async isWritten(runId, slot) { + const book = await open(runId); + return book.written.has(slot); + }, + + release(runId, slot) { + forgetClaim(runId, slot); + const book = books.get(runId); + if (!book) { + return; + } + // The ceiling stays where it is: this position may already sit below one a + // sibling published, and handing it out again would order a later event + // before an earlier one. + book.outstanding.delete(slot); + }, + + observe(runId, eventId) { + const slot = slotFromId(eventId); + if (slot === undefined) { + return; + } + forgetClaim(runId, slot); + const book = books.get(runId); + if (!book) { + // Nothing to keep consistent: the slot is on disk by the time this is + // called, so the eventual seed scan picks it up. + return; + } + book.written.add(slot); + book.outstanding.delete(slot); + book.ceiling = Math.max(book.ceiling, slot); + }, + + async refresh(runId) { + const book = books.get(runId); + if (!book) { + // Nothing cached to correct; the next reservation seeds from disk. + return; + } + for (const eventId of await listRunEventIds(basedir, runId, tag)) { + const slot = slotFromId(eventId); + if (slot !== undefined) { + book.written.add(slot); + book.outstanding.delete(slot); + book.ceiling = Math.max(book.ceiling, slot); + } + } + }, + + forget(runId) { + modes.delete(runId); + books.delete(runId); + // Claims outlive the book on purpose: they belong to writes still in + // flight, and the book a later allocation seeds has to hold them back. + }, + + clear() { + modes.clear(); + books.clear(); + claims.clear(); + }, + }; +} + +/** + * The slot a run's first event occupies. A run's own `run_created` is the only + * event that can be numbered without consulting the log, because there is + * provably nothing before it. + */ +export const RUN_CREATED_SLOT = FIRST_SLOT; diff --git a/packages/world-postgres/src/drizzle/migrations/0018_run_scoped_event_and_step_keys.sql b/packages/world-postgres/src/drizzle/migrations/0018_run_scoped_event_and_step_keys.sql new file mode 100644 index 0000000000..0764dee02d --- /dev/null +++ b/packages/world-postgres/src/drizzle/migrations/0018_run_scoped_event_and_step_keys.sql @@ -0,0 +1,13 @@ +-- Event ids and step ids are unique per run, not globally. Under slot identity +-- (spec 6) every run numbers its own log from 1, so "evnt_0...001" and +-- "step_0...001" exist once per run and the old global primary keys would make +-- the second run to reach slot 1 collide with the first. +-- +-- The run leads both keys so the existing run-scoped range scans stay a single +-- index seek; that also makes the standalone run_id indexes redundant. +ALTER TABLE "workflow"."workflow_events" DROP CONSTRAINT IF EXISTS "workflow_events_pkey";--> statement-breakpoint +ALTER TABLE "workflow"."workflow_events" ADD CONSTRAINT "workflow_events_run_id_id_pk" PRIMARY KEY("run_id","id");--> statement-breakpoint +DROP INDEX IF EXISTS "workflow"."workflow_events_run_id_index";--> statement-breakpoint +ALTER TABLE "workflow"."workflow_steps" DROP CONSTRAINT IF EXISTS "workflow_steps_pkey";--> statement-breakpoint +ALTER TABLE "workflow"."workflow_steps" ADD CONSTRAINT "workflow_steps_run_id_step_id_pk" PRIMARY KEY("run_id","step_id");--> statement-breakpoint +DROP INDEX IF EXISTS "workflow"."workflow_steps_run_id_index"; diff --git a/packages/world-postgres/src/drizzle/migrations/meta/_journal.json b/packages/world-postgres/src/drizzle/migrations/meta/_journal.json index b7fb5d8215..9dca967a5b 100644 --- a/packages/world-postgres/src/drizzle/migrations/meta/_journal.json +++ b/packages/world-postgres/src/drizzle/migrations/meta/_journal.json @@ -127,6 +127,13 @@ "when": 1785283200000, "tag": "0017_add_hook_resume_context", "breakpoints": true + }, + { + "idx": 18, + "version": "7", + "when": 1785801600000, + "tag": "0018_run_scoped_event_and_step_keys", + "breakpoints": true } ] } diff --git a/packages/world-postgres/src/drizzle/schema.ts b/packages/world-postgres/src/drizzle/schema.ts index 6ffb21abcb..df58f73eb8 100644 --- a/packages/world-postgres/src/drizzle/schema.ts +++ b/packages/world-postgres/src/drizzle/schema.ts @@ -133,7 +133,7 @@ export const runs = schema.table( export const events = schema.table( 'workflow_events', { - eventId: varchar('id').primaryKey(), + eventId: varchar('id').notNull(), eventType: varchar('type').$type().notNull(), correlationId: varchar('correlation_id'), createdAt: timestamp('created_at').defaultNow().notNull(), @@ -146,7 +146,11 @@ export const events = schema.table( Cborized & { eventData?: undefined }, 'eventData'> >, (tb) => [ - index().on(tb.runId), + // Event ids are only unique within their run: under slot identity every run + // numbers its own log from 1, so `evnt_0…001` exists once per run. The run + // leads the key so the range scans in `list` stay a single index seek, and + // it subsumes the plain `run_id` index the table used to carry. + primaryKey({ columns: [tb.runId, tb.eventId] }), index().on(tb.correlationId), // Runtime-correlated one-shot events must be unique per (run, correlation) // — without @@ -167,7 +171,7 @@ export const steps = schema.table( 'workflow_steps', { runId: varchar('run_id').notNull(), - stepId: varchar('step_id').primaryKey(), + stepId: varchar('step_id').notNull(), stepName: varchar('step_name').notNull(), status: stepStatus('status').notNull(), /** @deprecated */ @@ -203,7 +207,13 @@ export const steps = schema.table( 'output' | 'input' | 'error' > >, - (tb) => [index().on(tb.runId), index().on(tb.status)] + (tb) => [ + // A step id is a correlation id, which under slot identity is only unique + // within its run — same reasoning as `workflow_events`. Every step query in + // this world is already run-scoped, so the run leads the key. + primaryKey({ columns: [tb.runId, tb.stepId] }), + index().on(tb.status), + ] ); export const hooks = schema.table( diff --git a/packages/world-postgres/src/index.ts b/packages/world-postgres/src/index.ts index 430cea0812..926d752d3b 100644 --- a/packages/world-postgres/src/index.ts +++ b/packages/world-postgres/src/index.ts @@ -1,5 +1,5 @@ import type { Storage, World } from '@workflow/world'; -import { reenqueueActiveRuns, SPEC_VERSION_CURRENT } from '@workflow/world'; +import { mintedSpecVersion, reenqueueActiveRuns } from '@workflow/world'; import { Pool } from 'pg'; import type { PostgresWorldConfig } from './config.js'; import { createClient, type Drizzle } from './drizzle/index.js'; @@ -63,7 +63,10 @@ export function createWorld( const streamer = createStreamer(pool, drizzle); return { - specVersion: SPEC_VERSION_CURRENT, + // What this world stamps on new runs: slot identity, unless + // WORKFLOW_SLOT_IDENTITY switches it off. Every world reads both schemes + // whatever this says. + specVersion: mintedSpecVersion(), ...storage, ...streamer, ...queue, diff --git a/packages/world-postgres/src/slots.ts b/packages/world-postgres/src/slots.ts new file mode 100644 index 0000000000..6a0366a636 --- /dev/null +++ b/packages/world-postgres/src/slots.ts @@ -0,0 +1,231 @@ +/** + * Slot identity for the postgres world. + * + * A slot-numbered run names its events by position: `evnt_…001` is the first + * event of the run, `evnt_…002` the second. Contention on a position is the + * point — it is what makes a concurrent write detectable rather than silent — + * so the two things this module has to get right are that a position is written + * at most once and that a position this allocator loses is retried rather than + * abandoned as a hole. + * + * The authority for both is the events table's primary key, `(run_id, id)`: the + * INSERT either lands or raises a unique violation, and a writer that loses the + * race is retried at a position that is still free rather than abandoning the + * one it lost. The probe below is only ever a hint about where to try next. + */ + +import { WorkflowWorldError } from '@workflow/errors'; +import { + FIRST_SLOT, + SLOT_RETRY_BUDGET_MS, + slotEventId, + slotFromId, + slotRetryDelay, +} from '@workflow/world'; +import { and, desc, eq } from 'drizzle-orm'; +import { type Drizzle, Schema } from './drizzle/index.js'; + +/** + * The slot a run's own `run_created` occupies. Nothing in a run precedes its + * creation, so this one position needs no allocation, and every other event of + * the run searches above it — including the event that happens to reach storage + * first, which on the start path is routinely `run_started`. + */ +export const RUN_CREATED_SLOT = FIRST_SLOT; + +/** Postgres unique-violation code. */ +const UNIQUE_VIOLATION = '23505'; + +/** + * Whether an error says the position a write aimed at is already occupied. + * + * Drizzle wraps the pg error, so the code can sit on the error or on its cause. + * Both the name drizzle generates for the composite key and the name postgres + * gives an inline `PRIMARY KEY` are accepted, so a database whose key predates + * the run-scoped migration still classifies correctly. + */ +export function isEventKeyViolation(error: unknown): boolean { + const pg = (error as { code?: string; constraint?: string }).code + ? (error as { code?: string; constraint?: string }) + : ((error as { cause?: { code?: string; constraint?: string } }).cause ?? + {}); + return ( + pg.code === UNIQUE_VIOLATION && + (pg.constraint === 'workflow_events_run_id_id_pk' || + pg.constraint === 'workflow_events_pkey') + ); +} + +/** + * The highest event id in a run's log, or undefined when the log is empty. + * + * One backwards scan of the `(run_id, id)` primary key. Ids are fixed-width + * within a scheme, so for a slot-numbered run the highest id names the highest + * written position — and because a log holds ids of exactly one scheme, that id + * also reports which scheme the run was created with. + */ +export async function highestEventId( + drizzle: Drizzle, + runId: string +): Promise { + const [row] = await drizzle + .select({ eventId: Schema.events.eventId }) + .from(Schema.events) + .where(eq(Schema.events.runId, runId)) + .orderBy(desc(Schema.events.eventId)) + .limit(1); + return row?.eventId; +} + +/** The position an id names, or 0 for an empty log or a ULID-numbered one. */ +export function highestSlotOf(eventId: string | undefined): number { + return eventId === undefined ? 0 : (slotFromId(eventId) ?? 0); +} + +/** Whether a run's log already holds `eventId`. */ +export async function eventExists( + drizzle: Drizzle, + runId: string, + eventId: string +): Promise { + const [row] = await drizzle + .select({ eventId: Schema.events.eventId }) + .from(Schema.events) + .where( + and(eq(Schema.events.runId, runId), eq(Schema.events.eventId, eventId)) + ) + .limit(1); + return row !== undefined; +} + +/** The event ids a single create publishes. */ +export interface EventIds { + /** + * The id of the event this create returns. Taken on demand: a ULID-numbered + * write mints it inside its transaction, once the row lock that orders it is + * held. + */ + primary: () => string; + /** + * An additional event written in the same breath — the synthetic + * `step_created` of a lazy step start. + * + * A claim names the *top* of the pair, so this event takes the position + * immediately below it: the caller reserved both and named one. Numbering it + * off the log instead would hand it a position another write of the same + * concurrent batch is already holding, and cost that write its claim. + */ + extra: () => Promise; +} + +export interface PlaceEventOptions { + /** + * Position the caller named, when it holds the log and claimed one. A claim + * asserts the log is complete up to that position, so losing it is a conflict + * the caller has to resolve rather than something to retry here. + */ + claimedSlot?: number; + /** Lowest position this write may take when allocating. */ + minSlot: number; + /** + * Result of a probe the caller has already made, used for the first attempt + * instead of probing again. Later rounds always re-probe: the log has + * demonstrably moved. + */ + seedHighestEventId?: string | undefined; + /** The conflict raised when a claimed position turns out to be taken. */ + onClaimTaken: () => Promise; + /** Performs the write with the ids it should publish under. */ + write: (ids: EventIds) => Promise; +} + +/** + * Writes an event of a slot-numbered run, at the position the caller claimed or + * at the next free one. + * + * Every round re-probes rather than incrementing a local counter: each round at + * least one writer wins, so re-probing guarantees progress under any amount of + * contention. `write` must leave nothing behind when it raises a unique + * violation — the callers here either write only the event row or wrap their + * materialization in the same transaction, so a lost round rolls back whole. + */ +export async function placeEvent( + drizzle: Drizzle, + runId: string, + options: PlaceEventOptions +): Promise { + const deadline = Date.now() + SLOT_RETRY_BUDGET_MS; + for (let round = 0; ; round++) { + let cursor: number | undefined; + /** + * Positions for this attempt, consecutive from one probe. Deferred so a + * claimed write with no extra event never probes at all. + */ + const take = async (): Promise => { + if (cursor === undefined) { + const highest = + round === 0 && options.seedHighestEventId !== undefined + ? options.seedHighestEventId + : await highestEventId(drizzle, runId); + cursor = Math.max( + highestSlotOf(highest) + 1, + options.minSlot, + // The claimed position is this write's own; an extra event must not + // be handed it. + (options.claimedSlot ?? 0) + 1 + ); + } + return cursor++; + }; + + const primary = + options.claimedSlot === undefined + ? slotEventId(await take()) + : slotEventId(options.claimedSlot); + /** Positions the caller named, which are the caller's to resolve. */ + const claimed = options.claimedSlot === undefined ? [] : [primary]; + try { + return await options.write({ + primary: () => primary, + extra: async () => { + if (options.claimedSlot === undefined) { + return slotEventId(await take()); + } + const slot = options.claimedSlot - 1; + if (slot <= FIRST_SLOT) { + // The run's own `run_created` holds the first slot, so a claim of + // the second leaves nowhere for a second event to go: the caller + // reserved one position for a write that publishes two. + throw new WorkflowWorldError( + `Event id "${primary}" leaves no slot below it in run "${runId}" for the second event published alongside it`, + { status: 400 } + ); + } + const id = slotEventId(slot); + claimed.push(id); + return id; + }, + }); + } catch (error) { + if (!isEventKeyViolation(error)) { + throw error; + } + // Only a position the caller named is the caller's problem; one this + // world allocated is reallocated below without ever surfacing. + for (const id of claimed) { + if (await eventExists(drizzle, runId, id)) { + throw await options.onClaimTaken(); + } + } + if (Date.now() >= deadline) { + throw new WorkflowWorldError( + `Could not place an event in run "${runId}" within ${SLOT_RETRY_BUDGET_MS}ms of contention`, + { status: 503 } + ); + } + await new Promise((resolve) => + setTimeout(resolve, slotRetryDelay(round)) + ); + } + } +} diff --git a/packages/world-postgres/src/storage.ts b/packages/world-postgres/src/storage.ts index 140597a6a9..b62a17ec6f 100644 --- a/packages/world-postgres/src/storage.ts +++ b/packages/world-postgres/src/storage.ts @@ -3,12 +3,14 @@ import { HookNotFoundError, RunExpiredError, RunNotSupportedError, + SlotConflictError, TooEarlyError, WorkflowRunNotFoundError, WorkflowWorldError, } from '@workflow/errors'; import type { AttributeChange, + CreateEventParams, Event, EventResult, ExperimentalSetAttributesResult, @@ -35,15 +37,20 @@ import { isChildEntityCreationEventType, isHookEventRequiringExistence, isLegacySpecVersion, + isSlotId, isTerminalRunEventType, isTerminalStepStatus, isTerminalWorkflowRunStatus, requiresNewerWorld, SPEC_VERSION_CURRENT, + SPEC_VERSION_MAX_SUPPORTED, StepSchema, + slotEventId, + slotFromId, stripEventDataRefs, TERMINAL_STEP_STATUSES, TERMINAL_WORKFLOW_RUN_STATUSES, + usesSlotIdentity, validateAttributeChanges, validateUlidTimestamp, WorkflowRunSchema, @@ -62,6 +69,13 @@ import { import { monotonicFactory } from 'ulid'; import { type Drizzle, Schema } from './drizzle/index.js'; import type { SerializedContent } from './drizzle/schema.js'; +import { + type EventIds, + eventExists, + highestEventId, + placeEvent, + RUN_CREATED_SLOT, +} from './slots.js'; import { compact } from './util.js'; /** @@ -442,7 +456,7 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { // hook row left behind by a process / database interruption between // the hook INSERT and the events INSERT below (see the recovery // logic in the hook_created branch). - const getHookCreatedEvent = drizzle + const getCorrelatedEvent = drizzle .select({ eventId: events.eventId }) .from(events) .where( @@ -453,7 +467,7 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { ) ) .limit(1) - .prepare('events_get_hook_created_for_run_correlation'); + .prepare('events_get_correlated_event'); const getWaitForValidation = drizzle .select({ @@ -464,6 +478,64 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { .limit(1) .prepare('events_get_wait_for_validation'); + /** + * The events a caller that just lost a slot is missing: one ascending page of + * the run's log after the cursor it wrote from, minus anything at or below the + * highest slot it already held. Because slots are dense, that second filter is + * exact — a caller cannot be missing an event whose position it can name. + * + * Returned inline with the conflict so the common case (a handful of events + * arrived out of band) costs the caller no extra round-trip. `hasMore` is + * forwarded verbatim: an overflowing delta is the caller's signal to page from + * `cursor` instead of treating this as the whole story. + */ + async function eventsAfterClaim( + runId: string, + params: CreateEventParams | undefined + ): Promise<{ events: Event[]; cursor: string | null; hasMore: boolean }> { + const limit = 100; + const all = await drizzle + .select() + .from(events) + .where( + and( + eq(events.runId, runId), + map(params?.sinceCursor, (c) => gt(events.eventId, c)) + ) + ) + .orderBy(events.eventId) + .limit(limit + 1); + const page = all.slice(0, limit); + const maxSlot = params?.maxSlot ?? 0; + const resolveData = params?.resolveData ?? 'all'; + return { + events: page + .filter((v) => (slotFromId(v.eventId) ?? 0) > maxSlot) + .map((v) => { + v.eventData ||= v.eventDataJson; + return stripEventDataRefs(EventSchema.parse(compact(v)), resolveData); + }), + cursor: page.at(-1)?.eventId ?? null, + hasMore: all.length > limit, + }; + } + + /** + * The 409 a caller gets when the slot it named turns out to belong to someone + * else, carrying the events it is missing so it can merge, replay and + * re-propose at a free position. + */ + async function slotConflict( + runId: string, + eventId: string, + params: CreateEventParams | undefined + ): Promise { + return new SlotConflictError( + `Slot ${slotFromId(eventId)} of run "${runId}" is already taken`, + { eventId, ...(await eventsAfterClaim(runId, params)) } + ); + } + return { async create(runId, data, params): Promise { let eventId: string | undefined; @@ -490,6 +562,20 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { // specVersion is always sent by the runtime, but we provide a fallback for safety const effectiveSpecVersion = data.specVersion ?? SPEC_VERSION_CURRENT; + // Whether this run numbers its events by slot. Decided from what was + // persisted, never from this request or this build, so a run stays in the + // mode it was created in for life — a run whose log holds ULID ids must + // never be handed a slot id, and vice versa. `run_created` is the one + // event that decides the mode instead of reading it; the resilient-start + // path below decides it too, on the request that creates the run. + let slotMode: boolean | undefined = + data.eventType === 'run_created' + ? usesSlotIdentity(effectiveSpecVersion) + : undefined; + // The run's highest event id, when it was read before the write. Seeds the + // allocator's first attempt so a probe is never made twice. + let seedHighestEventId: string | undefined; + // Track entity created/updated for EventResult let run: WorkflowRun | undefined; let step: Step | undefined; @@ -585,7 +671,13 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { .returning(); if (inserted) { - const runCreatedEventId = `wevt_${ulid()}`; + // We created the run, so this request also decided its mode. + slotMode = usesSlotIdentity(effectiveSpecVersion); + // A run's own `run_created` provably has nothing before it, so its + // slot needs no allocation. + const runCreatedEventId = slotMode + ? slotEventId(RUN_CREATED_SLOT) + : `wevt_${ulid()}`; await drizzle.insert(events).values({ runId: effectiveRunId, eventId: runCreatedEventId, @@ -631,7 +723,7 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { if (requiresNewerWorld(currentRun.specVersion)) { throw new RunNotSupportedError( currentRun.specVersion!, - SPEC_VERSION_CURRENT + SPEC_VERSION_MAX_SUPPORTED ); } @@ -651,6 +743,92 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { throw new WorkflowRunNotFoundError(effectiveRunId); } + // The run entity is the authority on the mode, and it can appear between + // the resilient-start insert above and this point: start() issues + // `run_created` and the queue send concurrently, so a delivery's + // `run_started` can arrive while the run is still being published. A stale + // "no" would number that one event with a ULID on an otherwise + // slot-numbered run, and the hole it leaves in the numbering costs the log + // its completeness proof for life. + if (currentRun && data.eventType !== 'run_created') { + slotMode = usesSlotIdentity(currentRun.specVersion); + } + if (slotMode === undefined) { + // step_completed and step_retrying skip the run read above. The log's + // own highest id reports the scheme, since a log holds ids of exactly + // one, and it is the probe the allocator needs anyway — so a + // slot-numbered run pays nothing extra for this query. + seedHighestEventId = await highestEventId(drizzle, effectiveRunId); + slotMode = isSlotId(seedHighestEventId ?? ''); + } + + // ============================================================ + // EVENT ID: the caller's slot claim, an allocated slot, or a ULID + // ============================================================ + // A slot-numbered run's ids name positions in its log, so an id is either + // claimed by a caller that holds the log (and is therefore asserting the + // log is complete up to that position) or allocated at write time for a + // caller that has no log — a step completion reporting in, a cancellation + // from an API call. + let claimedSlot: number | undefined; + if (params?.eventId !== undefined) { + if (!slotMode) { + throw new WorkflowWorldError( + `Event id "${params.eventId}" was supplied for run "${effectiveRunId}", whose events are not numbered by slot`, + { status: 400 } + ); + } + claimedSlot = slotFromId(params.eventId); + if (claimedSlot === undefined) { + throw new WorkflowWorldError( + `Event id "${params.eventId}" is not a slot id, and run "${effectiveRunId}" numbers its events by slot`, + { status: 400 } + ); + } + eventId = params.eventId; + if (await eventExists(drizzle, effectiveRunId, eventId)) { + // Reject a doomed claim before the materialization below creates the + // step, hook or wait this event will now never accompany. A caller + // that re-proposes at the next slot would otherwise collide with its + // own orphan and read that as "my write already landed". + throw await slotConflict(effectiveRunId, eventId, params); + } + } + + /** + * Runs one of the event writes below under this run's id discipline: in + * slot mode it places the event at the position the caller claimed or at + * the next free one, retrying a position lost to a concurrent writer; + * otherwise it mints a ULID. + */ + const publish = async ( + write: (ids: EventIds) => Promise + ): Promise => + slotMode + ? placeEvent(drizzle, effectiveRunId, { + ...(claimedSlot !== undefined ? { claimedSlot } : {}), + // A run's own `run_created` owns the first slot — provably, since + // nothing precedes it — so every other event allocates above it, + // even when that event is the first to arrive here. + minSlot: + data.eventType === 'run_created' + ? RUN_CREATED_SLOT + : RUN_CREATED_SLOT + 1, + seedHighestEventId, + onClaimTaken: () => + slotConflict(effectiveRunId, eventId as string, params), + write: (ids) => { + // Each attempt publishes under its own id, and the result and + // error messages below read it back from here. + eventId = ids.primary(); + return write(ids); + }, + }) + : write({ + primary: getEventId, + extra: async () => `wevt_${ulid()}`, + }); + // Lazy step start: a step_started carrying step-creation data // (stepName + input) may arrive with no prior step_created — it creates // the step on the fly (see the materialization block below). This @@ -676,17 +854,19 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { .limit(1); // Create the event (still record it) - const [value] = await drizzle - .insert(Schema.events) - .values({ - runId: effectiveRunId, - eventId: getEventId(), - correlationId: data.correlationId, - eventType: data.eventType, - eventData: 'eventData' in data ? data.eventData : undefined, - specVersion: effectiveSpecVersion, - }) - .returning({ createdAt: Schema.events.createdAt }); + const [value] = await publish((ids) => + drizzle + .insert(Schema.events) + .values({ + runId: effectiveRunId, + eventId: ids.primary(), + correlationId: data.correlationId, + eventType: data.eventType, + eventData: 'eventData' in data ? data.eventData : undefined, + specVersion: effectiveSpecVersion, + }) + .returning({ createdAt: Schema.events.createdAt }) + ); const result = { ...data, @@ -1192,41 +1372,51 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { // event INSERT behind that lock prevents a late step_started from being // ordered after a concurrent terminal event that already won the row. if (data.eventType === 'step_started') { - value = await drizzle.transaction(async (tx) => { - // Lazy step start: no prior step_created exists, but this - // step_started carries the step-creation data. The step INSERT is - // the ownership claim: only the caller that inserts the row gets to - // run the step body inline. - if (lazyStepStart && !validatedStep) { - const lazyData = data.eventData; - const [inserted] = await tx - .insert(Schema.steps) - .values({ - runId: effectiveRunId, - stepId: data.correlationId, - stepName: lazyData.stepName, - input: lazyData.input as SerializedContent, - status: 'pending', - attempt: 0, - specVersion: effectiveSpecVersion, - }) - .onConflictDoNothing() - .returning({ stepId: Schema.steps.stepId }); - - if (!inserted) { - throw new EntityConflictError( - `Step "${data.correlationId}" already created` - ); - } + // The whole transaction is the retry unit here: a step_started that + // loses its slot has to roll the step row and the synthetic + // step_created back with it, or the next attempt would trip its own + // orphaned step and read that as "a concurrent handler won the create". + value = await publish((ids) => + drizzle.transaction(async (tx) => { + // Lazy step start: no prior step_created exists, but this + // step_started carries the step-creation data. The step INSERT is + // the ownership claim: only the caller that inserts the row gets to + // run the step body inline. + if (lazyStepStart && !validatedStep) { + const lazyData = data.eventData; + const [inserted] = await tx + .insert(Schema.steps) + .values({ + runId: effectiveRunId, + stepId: data.correlationId, + stepName: lazyData.stepName, + input: lazyData.input as SerializedContent, + status: 'pending', + attempt: 0, + specVersion: effectiveSpecVersion, + }) + .onConflictDoNothing() + .returning({ stepId: Schema.steps.stepId }); + + if (!inserted) { + throw new EntityConflictError( + `Step "${data.correlationId}" already created` + ); + } - // Replay still needs to observe step_created before - // step_started. Because this synthetic event is in the same - // transaction as the lazy step row and step_started event, we - // cannot leave behind only one side of that materialization. - const stepCreatedEventId = `wevt_${ulid()}`; - await tx - .insert(events) - .values({ + // Replay still needs to observe a step_created at all: the + // client's step consumer sets hasCreatedEvent only on that event + // type. Which of the pair sorts first does not matter — the + // step_started consumer is a no-op — but leaving behind only one + // side of the materialization would, hence the shared + // transaction. + // + // It takes a position of its own — the one below the claim on a + // claimed write, the next free one otherwise. Losing that position + // rolls the transaction back for a retry, so the insert must not + // swallow the collision. + const stepCreatedEventId = await ids.extra(); + const insertStepCreated = tx.insert(events).values({ runId: effectiveRunId, eventId: stepCreatedEventId, correlationId: data.correlationId, @@ -1236,99 +1426,104 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { input: lazyData.input, }, specVersion: effectiveSpecVersion, - }) - .onConflictDoNothing(); - stepCreatedLazily = true; - } - - // Retried steps may be scheduled for later. Keep this check inside - // the transaction so the step_started write cannot slip past it. - if ( - validatedStep?.retryAfter && - validatedStep.retryAfter.getTime() > Date.now() - ) { - throw new TooEarlyError( - `Cannot start step "${data.correlationId}": retryAfter timestamp has not been reached yet`, - { - retryAfter: Math.ceil( - (validatedStep.retryAfter.getTime() - Date.now()) / 1000 - ), - } - ); - } + }); + await (slotMode + ? insertStepCreated + : insertStepCreated.onConflictDoNothing()); + stepCreatedLazily = true; + } - // The terminal-state guard is part of the UPDATE, not just the - // earlier validation read. That closes the race where another - // writer completes/fails the step between validation and start. - const [stepValue] = await tx - .update(Schema.steps) - .set({ - status: 'running', - attempt: sql`${Schema.steps.attempt} + 1`, - // Preserve the original first-start timestamp across retries or - // overlapping starts. - startedAt: sql`COALESCE(${Schema.steps.startedAt}, ${now.toISOString()})`, - retryAfter: null, - }) - .where( - and( - eq(Schema.steps.runId, effectiveRunId), - eq(Schema.steps.stepId, data.correlationId!), - notInArray(Schema.steps.status, terminalStepStatuses) - ) - ) - .returning(); + // Retried steps may be scheduled for later. Keep this check inside + // the transaction so the step_started write cannot slip past it. + if ( + validatedStep?.retryAfter && + validatedStep.retryAfter.getTime() > Date.now() + ) { + throw new TooEarlyError( + `Cannot start step "${data.correlationId}": retryAfter timestamp has not been reached yet`, + { + retryAfter: Math.ceil( + (validatedStep.retryAfter.getTime() - Date.now()) / 1000 + ), + } + ); + } - if (stepValue) { - step = deserializeStepError(compact(stepValue)); - } else { - const [existing] = await tx - .select({ status: Schema.steps.status }) - .from(Schema.steps) + // The terminal-state guard is part of the UPDATE, not just the + // earlier validation read. That closes the race where another + // writer completes/fails the step between validation and start. + const [stepValue] = await tx + .update(Schema.steps) + .set({ + status: 'running', + attempt: sql`${Schema.steps.attempt} + 1`, + // Preserve the original first-start timestamp across retries or + // overlapping starts. + startedAt: sql`COALESCE(${Schema.steps.startedAt}, ${now.toISOString()})`, + retryAfter: null, + }) .where( and( eq(Schema.steps.runId, effectiveRunId), - eq(Schema.steps.stepId, data.correlationId!) + eq(Schema.steps.stepId, data.correlationId!), + notInArray(Schema.steps.status, terminalStepStatuses) ) ) - .limit(1); - if (!existing) { - throw new WorkflowWorldError( - `Step "${data.correlationId}" not found` - ); + .returning(); + + if (stepValue) { + step = deserializeStepError(compact(stepValue)); + } else { + const [existing] = await tx + .select({ status: Schema.steps.status }) + .from(Schema.steps) + .where( + and( + eq(Schema.steps.runId, effectiveRunId), + eq(Schema.steps.stepId, data.correlationId!) + ) + ) + .limit(1); + if (!existing) { + throw new WorkflowWorldError( + `Step "${data.correlationId}" not found` + ); + } + if (isTerminalStepStatus(existing.status)) { + throw new EntityConflictError( + `Cannot modify step in terminal state "${existing.status}"` + ); + } } - if (isTerminalStepStatus(existing.status)) { + + // A ULID-numbered step_started takes its id only after the guarded + // step UPDATE has acquired and passed the row lock. Without a + // sequence, this is the local ordering guarantee we can provide: a + // writer blocked on the step row will not carry an older event id + // into a later insert. A slot id is exempt — it names a position in + // the log, not a time — and the caller may have claimed it already. + const stepStartedEventId = ids.primary(); + eventId = stepStartedEventId; + const [eventValue] = await tx + .insert(events) + .values({ + runId: effectiveRunId, + eventId: stepStartedEventId, + correlationId: data.correlationId, + eventType: data.eventType, + eventData: storedEventData, + specVersion: effectiveSpecVersion, + }) + .returning({ createdAt: events.createdAt }); + + if (!eventValue) { throw new EntityConflictError( - `Cannot modify step in terminal state "${existing.status}"` + `Event ${stepStartedEventId} could not be created` ); } - } - - // Allocate the step_started ULID only after the guarded step UPDATE - // has acquired and passed the row lock. Without a sequence, this is - // the local ordering guarantee we can provide: a writer blocked on - // the step row will not carry an older event id into a later insert. - const stepStartedEventId = `wevt_${ulid()}`; - eventId = stepStartedEventId; - const [eventValue] = await tx - .insert(events) - .values({ - runId: effectiveRunId, - eventId: stepStartedEventId, - correlationId: data.correlationId, - eventType: data.eventType, - eventData: storedEventData, - specVersion: effectiveSpecVersion, - }) - .returning({ createdAt: events.createdAt }); - - if (!eventValue) { - throw new EntityConflictError( - `Event ${stepStartedEventId} could not be created` - ); - } - return eventValue; - }); + return eventValue; + }) + ); } // Handle step_completed event: update step status @@ -1496,7 +1691,7 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { existingHook.runId === effectiveRunId && existingHook.hookId === data.correlationId ) { - const [existingEvent] = await getHookCreatedEvent.execute({ + const [existingEvent] = await getCorrelatedEvent.execute({ runId: effectiveRunId, correlationId: data.correlationId, eventType: 'hook_created', @@ -1531,20 +1726,21 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { token: eventData.token, conflictingRunId: existingHook.runId, }; + const [conflictValue] = await publish((ids) => + drizzle + .insert(events) + .values({ + runId: effectiveRunId, + eventId: ids.primary(), + correlationId: data.correlationId, + eventType: 'hook_conflict', + eventData: conflictEventData, + specVersion: effectiveSpecVersion, + }) + .returning({ createdAt: events.createdAt }) + ); const conflictEventId = getEventId(); - const [conflictValue] = await drizzle - .insert(events) - .values({ - runId: effectiveRunId, - eventId: conflictEventId, - correlationId: data.correlationId, - eventType: 'hook_conflict', - eventData: conflictEventData, - specVersion: effectiveSpecVersion, - }) - .returning({ createdAt: events.createdAt }); - if (!conflictValue) { throw new EntityConflictError( `Event ${conflictEventId} could not be created` @@ -1622,47 +1818,49 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { // guarded UPDATE linearizes against a concurrent terminal step // event. if (data.eventType === 'hook_received') { - value = await drizzle.transaction(async (tx) => { - const [runRow] = await tx - .select({ status: Schema.runs.status }) - .from(Schema.runs) - .where(eq(Schema.runs.runId, effectiveRunId)) - .for('update') - .limit(1); - if (!runRow) { - throw new WorkflowRunNotFoundError(effectiveRunId); - } - if (isTerminalWorkflowRunStatus(runRow.status)) { - throw new RunExpiredError( - `Workflow run "${effectiveRunId}" is already in terminal state "${runRow.status}"` - ); - } + value = await publish((ids) => + drizzle.transaction(async (tx) => { + const [runRow] = await tx + .select({ status: Schema.runs.status }) + .from(Schema.runs) + .where(eq(Schema.runs.runId, effectiveRunId)) + .for('update') + .limit(1); + if (!runRow) { + throw new WorkflowRunNotFoundError(effectiveRunId); + } + if (isTerminalWorkflowRunStatus(runRow.status)) { + throw new RunExpiredError( + `Workflow run "${effectiveRunId}" is already in terminal state "${runRow.status}"` + ); + } - // Allocate the ULID only after the row lock is acquired, - // matching step_started's ordering guarantee: a writer blocked - // on the run row must not carry an older event id into a later - // insert. - const hookReceivedEventId = `wevt_${ulid()}`; - eventId = hookReceivedEventId; - const [eventValue] = await tx - .insert(events) - .values({ - runId: effectiveRunId, - eventId: hookReceivedEventId, - correlationId: data.correlationId, - eventType: data.eventType, - eventData: storedEventData, - specVersion: effectiveSpecVersion, - }) - .returning({ createdAt: events.createdAt }); + // Take the ULID only after the row lock is acquired, matching + // step_started's ordering guarantee: a writer blocked on the run + // row must not carry an older event id into a later insert. A slot + // id names a position rather than a time, so it is exempt. + const hookReceivedEventId = ids.primary(); + eventId = hookReceivedEventId; + const [eventValue] = await tx + .insert(events) + .values({ + runId: effectiveRunId, + eventId: hookReceivedEventId, + correlationId: data.correlationId, + eventType: data.eventType, + eventData: storedEventData, + specVersion: effectiveSpecVersion, + }) + .returning({ createdAt: events.createdAt }); - if (!eventValue) { - throw new EntityConflictError( - `Event ${hookReceivedEventId} could not be created` - ); - } - return eventValue; - }); + if (!eventValue) { + throw new EntityConflictError( + `Event ${hookReceivedEventId} could not be created` + ); + } + return eventValue; + }) + ); } // Handle wait_created event: create wait entity @@ -1694,9 +1892,43 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { specVersion: waitValue.specVersion ?? undefined, }; } else { - throw new EntityConflictError( - `Wait "${data.correlationId}" already exists` - ); + // The wait row exists but this call did not write it. Which of the two + // reasons it is decides everything, and only the event log knows: + // - the `wait_created` event exists → a real duplicate, so throw and + // let the runtime's concurrent-replay catch path swallow it. + // - it does not → an orphaned row from an attempt that materialized + // the wait and then lost its event write (a crash, or a slot + // claimed by someone else). The caller re-proposing the same + // operation one position higher is exactly what has to succeed + // here, so adopt the row and complete the partial write. Mirrors + // hook_created's handling of the same window. + const [existingEvent] = await getCorrelatedEvent.execute({ + runId: effectiveRunId, + correlationId: data.correlationId, + eventType: 'wait_created', + }); + if (existingEvent) { + throw new EntityConflictError( + `Wait "${data.correlationId}" already exists` + ); + } + const [orphan] = await drizzle + .select() + .from(Schema.waits) + .where(eq(Schema.waits.waitId, waitId)) + .limit(1); + if (orphan) { + wait = { + waitId: orphan.waitId, + runId: orphan.runId, + status: orphan.status, + resumeAt: orphan.resumeAt ?? undefined, + completedAt: orphan.completedAt ?? undefined, + createdAt: orphan.createdAt, + updatedAt: orphan.updatedAt, + specVersion: orphan.specVersion ?? undefined, + }; + } } } @@ -1748,17 +1980,23 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { try { if (!value) { - [value] = await drizzle - .insert(events) - .values({ - runId: effectiveRunId, - eventId: getEventId(), - correlationId: data.correlationId, - eventType: data.eventType, - eventData: storedEventData, - specVersion: effectiveSpecVersion, - }) - .returning({ createdAt: events.createdAt }); + // Only the event row is retried here: the entity this event describes + // was materialized above, outside any transaction, and re-inserting + // the event at a higher position leaves the log dense and still + // consistent with that entity. + [value] = await publish((ids) => + drizzle + .insert(events) + .values({ + runId: effectiveRunId, + eventId: ids.primary(), + correlationId: data.correlationId, + eventType: data.eventType, + eventData: storedEventData, + specVersion: effectiveSpecVersion, + }) + .returning({ createdAt: events.createdAt }) + ); } } catch (err) { // Translate unique-violation on the correlated-event partial index diff --git a/packages/world-postgres/test/slot-identity.test.ts b/packages/world-postgres/test/slot-identity.test.ts new file mode 100644 index 0000000000..a6cf498651 --- /dev/null +++ b/packages/world-postgres/test/slot-identity.test.ts @@ -0,0 +1,426 @@ +import { execSync } from 'node:child_process'; +import { PostgreSqlContainer } from '@testcontainers/postgresql'; +import { SlotConflictError } from '@workflow/errors'; +import { + FIRST_SLOT, + maxSlotOf, + SPEC_VERSION_CURRENT, + SPEC_VERSION_SLOT_IDENTITY, + slotEventId, + slotFromId, +} from '@workflow/world'; +import { Pool } from 'pg'; +import { + afterAll, + beforeAll, + beforeEach, + describe, + expect, + test, +} from 'vitest'; +import { createClient } from '../src/drizzle/index.js'; +import { createEventsStorage } from '../src/storage.js'; + +describe('Slot identity (Postgres integration)', () => { + if (process.platform === 'win32') { + test.skip('skipped on Windows since it relies on a docker container', () => {}); + return; + } + + let container: Awaited>; + let pool: Pool; + let events: ReturnType; + + beforeAll(async () => { + container = await new PostgreSqlContainer('postgres:15-alpine').start(); + const dbUrl = container.getConnectionUri(); + process.env.DATABASE_URL = dbUrl; + process.env.WORKFLOW_POSTGRES_URL = dbUrl; + execSync('pnpm db:push', { + stdio: 'inherit', + cwd: process.cwd(), + env: process.env, + }); + // Contention is the point of these tests, so the pool has to be able to + // hold every writer of a burst at once. + pool = new Pool({ connectionString: dbUrl, max: 20 }); + events = createEventsStorage(createClient(pool)); + }, 120_000); + + beforeEach(async () => { + await pool.query( + 'TRUNCATE TABLE workflow.workflow_events, workflow.workflow_steps, workflow.workflow_hooks, workflow.workflow_runs RESTART IDENTITY CASCADE' + ); + }); + + afterAll(async () => { + await pool.end(); + await container.stop(); + }); + + /** Start a run whose events are numbered by slot, and return its id. */ + async function newSlotRun(): Promise { + const result = await events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + eventData: { + deploymentId: 'dpl_test', + workflowName: 'test-workflow', + input: new Uint8Array(), + }, + }); + if (!result.run) { + throw new Error('Expected run to be created'); + } + return result.run.runId; + } + + function eventsOf(runId: string) { + // The page size is explicit: the default would silently truncate a fan-out + // and make a dense log look sparse. + return events.list({ runId, pagination: { limit: 500 } }); + } + + /** The slots of the run's log, in list order. */ + async function slotsOf(runId: string): Promise { + const { data } = await eventsOf(runId); + return data.map((event) => slotFromId(event.eventId) ?? -1); + } + + function ascending(slots: number[]): number[] { + return [...slots].sort((a, b) => a - b); + } + + function denseFrom(count: number): number[] { + return Array.from({ length: count }, (_, index) => FIRST_SLOT + index); + } + + async function createStep( + runId: string, + stepId: string, + eventId?: string + ): Promise { + const result = await events.create( + runId, + { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: stepId, + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }, + eventId === undefined ? undefined : { eventId } + ); + if (!result.event) { + throw new Error('Expected an event'); + } + return result.event.eventId; + } + + describe('numbering', () => { + test('puts run_created in the first slot', async () => { + const runId = await newSlotRun(); + await expect(slotsOf(runId)).resolves.toEqual([FIRST_SLOT]); + }); + + test('allocates dense slots for writers that hold no log', async () => { + // A step completion reporting in, a cancellation from an API call: the + // caller has no event log, so the world numbers the event for it. + const runId = await newSlotRun(); + await createStep(runId, 'step_a'); + await createStep(runId, 'step_b'); + await expect(slotsOf(runId)).resolves.toEqual(denseFrom(3)); + }); + + test('honours a slot the caller claims', async () => { + const runId = await newSlotRun(); + const eventId = await createStep(runId, 'step_a', slotEventId(2)); + expect(eventId).toBe(slotEventId(2)); + await expect(slotsOf(runId)).resolves.toEqual(denseFrom(2)); + }); + + test('numbers a ULID-mode run the way it always did', async () => { + const created = await events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'dpl_test', + workflowName: 'test-workflow', + input: new Uint8Array(), + }, + }); + const runId = created.run?.runId as string; + const eventId = await createStep(runId, 'step_a'); + expect(eventId).toMatch(/^wevt_/); + expect(slotFromId(eventId)).toBeUndefined(); + }); + + test('gives a lazy step start two consecutive slots', async () => { + // One request, two events: the step_started the caller sent and the + // step_created it deferred. Which sorts first does not matter — only + // step_created flips the client's hasCreatedEvent — but both have to + // land, and neither may leave a hole. + const runId = await newSlotRun(); + const started = await events.create(runId, { + eventType: 'step_started', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }); + expect(slotFromId(started.event?.eventId ?? '')).toBe(2); + const { data } = await eventsOf(runId); + expect( + data.map((event) => `${slotFromId(event.eventId)} ${event.eventType}`) + ).toEqual(['1 run_created', '2 step_started', '3 step_created']); + }); + + test('numbers the deferred step_created below a claimed slot', async () => { + // A claim names the top of the pair: the caller reserved both positions + // before either landed, which is what keeps the second event off the slot + // the next write of the same batch claimed. + const runId = await newSlotRun(); + const started = await events.create( + runId, + { + eventType: 'step_started', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }, + { eventId: slotEventId(3) } + ); + expect(started.event?.eventId).toBe(slotEventId(3)); + const { data } = await eventsOf(runId); + expect( + data.map((event) => `${slotFromId(event.eventId)} ${event.eventType}`) + ).toEqual(['1 run_created', '2 step_created', '3 step_started']); + }); + + test('keeps every claim in a burst of lazy starts', async () => { + // The suspension flush issues its lazy starts at once, each having + // reserved two positions. A second event numbered off the log as this + // world sees it would take the slot the next start in the batch claimed, + // costing every start after the first its claim. + const runId = await newSlotRun(); + const claims = Array.from({ length: 10 }, (_, index) => + slotEventId(FIRST_SLOT + 2 * (index + 1)) + ); + const started = await Promise.all( + claims.map((eventId, index) => + events.create( + runId, + { + eventType: 'step_started', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: `step_${index}`, + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }, + { eventId } + ) + ) + ); + expect(started.map((result) => result.event?.eventId)).toEqual(claims); + expect(ascending(await slotsOf(runId))).toEqual( + denseFrom(2 * claims.length + 1) + ); + }); + + test('rejects a claim that leaves no room for the second event', async () => { + // The run's own run_created holds the first slot, so a claim of the second + // means the caller reserved one position for a write that publishes two. + const runId = await newSlotRun(); + await expect( + events.create( + runId, + { + eventType: 'step_started', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }, + { eventId: slotEventId(FIRST_SLOT + 1) } + ) + ).rejects.toThrow(/leaves no slot below it/); + }); + + test('numbers events of runs it never created', async () => { + // `step_completed` and `step_retrying` deliberately skip the run read, so + // the mode comes from the log rather than from a run row in hand. + const runId = await newSlotRun(); + await createStep(runId, 'step_a'); + const completed = await events.create(runId, { + eventType: 'step_completed', + specVersion: SPEC_VERSION_CURRENT, + correlationId: 'step_a', + eventData: { output: new Uint8Array() }, + }); + expect(slotFromId(completed.event?.eventId ?? '')).toBe(3); + }); + }); + + describe('contention', () => { + // The primary key is the authority, and a writer that loses a position is + // retried at one that is still free rather than abandoning the one it lost, + // so a burst of concurrent writers still numbers itself densely. + for (const writers of [2, 8, 50]) { + test(`keeps ${writers} concurrent writers dense`, async () => { + const runId = await newSlotRun(); + const ids = await Promise.all( + Array.from({ length: writers }, (_, index) => + createStep(runId, `step_${index}`) + ) + ); + expect(new Set(ids).size).toBe(writers); + expect(ascending(await slotsOf(runId))).toEqual(denseFrom(writers + 1)); + }, 60_000); + } + + test('numbers a burst so the highest slot is the event count', async () => { + const runId = await newSlotRun(); + await Promise.all( + Array.from({ length: 5 }, (_, index) => + createStep(runId, `step_${index}`) + ) + ); + const { data } = await eventsOf(runId); + expect(maxSlotOf(data)).toBe(data.length); + }); + + test('leaves no hole behind a rejected write', async () => { + // The rejected op's slot sits below its concurrent sibling's, and a hole + // below a published event can never be filled. + const runId = await newSlotRun(); + const [rejected, accepted] = await Promise.allSettled([ + events.create(runId, { + eventType: 'step_completed', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_never_created', + eventData: { output: new Uint8Array() }, + }), + createStep(runId, 'step_a'), + ]); + expect(rejected.status).toBe('rejected'); + expect(accepted.status).toBe('fulfilled'); + await createStep(runId, 'step_b'); + expect(ascending(await slotsOf(runId))).toEqual(denseFrom(3)); + }); + }); + + describe('mode is pinned to the run', () => { + test('rejects a slot id claimed on a ULID-numbered run', async () => { + const created = await events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'dpl_test', + workflowName: 'test-workflow', + input: new Uint8Array(), + }, + }); + const runId = created.run?.runId as string; + await expect(createStep(runId, 'step_a', slotEventId(2))).rejects.toThrow( + /not numbered by slot/ + ); + }); + + test('rejects a ULID id claimed on a slot-numbered run', async () => { + const runId = await newSlotRun(); + await expect( + createStep(runId, 'step_a', 'evnt_01K5Z0000000000000000000AA') + ).rejects.toThrow(/not a slot id/); + }); + + test('ignores the spec version of later requests', async () => { + // A run is in exactly one mode for life; only what was persisted decides. + const runId = await newSlotRun(); + const result = await events.create(runId, { + eventType: 'step_created', + specVersion: SPEC_VERSION_CURRENT, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }); + expect(slotFromId(result.event?.eventId ?? '')).toBe(2); + }); + }); + + describe('conflict', () => { + test('reports the events the loser is missing', async () => { + const runId = await newSlotRun(); + // Out of band: something else takes the slot this caller was about to + // claim, so the caller's log is provably missing an event. + await createStep(runId, 'step_out_of_band'); + + const conflict = await events + .create( + runId, + { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }, + { eventId: slotEventId(2), maxSlot: 1 } + ) + .catch((error: unknown) => error); + + expect(SlotConflictError.is(conflict)).toBe(true); + const slotConflict = conflict as SlotConflictError; + expect(slotConflict.status).toBe(409); + expect(slotConflict.eventId).toBe(slotEventId(2)); + expect(slotConflict.events?.map((event) => event.eventId)).toEqual([ + slotEventId(2), + ]); + }); + + test('excludes events the loser already holds from the delta', async () => { + const runId = await newSlotRun(); + await createStep(runId, 'step_one'); + await createStep(runId, 'step_two'); + + const conflict = await events + .create( + runId, + { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }, + { eventId: slotEventId(2), maxSlot: 2 } + ) + .catch((error: unknown) => error); + + // Slots 1 and 2 are at or below what the caller had; only 3 is news. + expect( + (conflict as SlotConflictError).events?.map((event) => event.eventId) + ).toEqual([slotEventId(3)]); + }); + + test('lets the loser re-propose at the next free slot', async () => { + const runId = await newSlotRun(); + await createStep(runId, 'step_out_of_band'); + await expect(createStep(runId, 'step_a', slotEventId(2))).rejects.toThrow( + SlotConflictError + ); + // Merging the delta moves the caller's own numbering forward by one. + const eventId = await createStep(runId, 'step_a', slotEventId(3)); + expect(eventId).toBe(slotEventId(3)); + await expect(slotsOf(runId)).resolves.toEqual(denseFrom(3)); + }); + + test('materializes nothing for a claim that is already taken', async () => { + // The re-post is what the guard protects: a step row left behind by the + // losing attempt would make the retry trip its own orphan and read that + // as "a concurrent handler won the create". + const runId = await newSlotRun(); + await createStep(runId, 'step_out_of_band'); + await expect(createStep(runId, 'step_a', slotEventId(2))).rejects.toThrow( + SlotConflictError + ); + const { rows } = await pool.query( + 'SELECT step_id FROM workflow.workflow_steps WHERE run_id = $1', + [runId] + ); + expect(rows.map((row) => row.step_id)).toEqual(['step_out_of_band']); + }); + }); +}); diff --git a/packages/world-vercel/src/event-retry.test.ts b/packages/world-vercel/src/event-retry.test.ts index d514576c7b..c0b7f65491 100644 --- a/packages/world-vercel/src/event-retry.test.ts +++ b/packages/world-vercel/src/event-retry.test.ts @@ -1,6 +1,7 @@ import { EntityConflictError, RunExpiredError, + SlotConflictError, ThrottleError, TooEarlyError, WorkflowWorldError, @@ -94,6 +95,16 @@ describe('isRetryableEventPostError', () => { expect(isRetryableEventPostError(new ThrottleError('429'))).toBe(false); }); + it('does not retry a lost event slot', () => { + // Re-issuing the same write is guaranteed to lose the slot again; only a + // merge and a replay can produce a write that lands. + expect( + isRetryableEventPostError( + new SlotConflictError('taken', { eventId: 'evnt_x' }) + ) + ).toBe(false); + }); + it('retries a body-parse failure (write may have landed)', () => { expect( isRetryableEventPostError( diff --git a/packages/world-vercel/src/event-retry.ts b/packages/world-vercel/src/event-retry.ts index fc0d3bd87f..25985a6c5c 100644 --- a/packages/world-vercel/src/event-retry.ts +++ b/packages/world-vercel/src/event-retry.ts @@ -40,6 +40,7 @@ import { EntityConflictError, RunExpiredError, + SlotConflictError, ThrottleError, TooEarlyError, WorkflowWorldError, @@ -203,9 +204,13 @@ function collectErrorMarkers(err: unknown, depth = 0): string[] { export function isRetryableEventPostError(err: unknown): boolean { // Definitive, server-considered outcomes — never retried in-process. // (425/429 are intentionally left to the runtime's retry-after handling.) + // A slot conflict is doubly definitive: the write lost a race for its event + // id, so re-issuing it unchanged is guaranteed to lose again. Only a merge and + // a replay can produce a write that can land. if ( EntityConflictError.is(err) || RunExpiredError.is(err) || + SlotConflictError.is(err) || TooEarlyError.is(err) || ThrottleError.is(err) ) { diff --git a/packages/world-vercel/src/events-v4.test.ts b/packages/world-vercel/src/events-v4.test.ts index a9d3980dc5..09bdf3a716 100644 --- a/packages/world-vercel/src/events-v4.test.ts +++ b/packages/world-vercel/src/events-v4.test.ts @@ -1,10 +1,16 @@ import { EntityConflictError, RunExpiredError, + SlotConflictError, ThrottleError, TooEarlyError, WorkflowWorldError, } from '@workflow/errors'; +import { + SPEC_VERSION_SLOT_IDENTITY, + slotEventId, + slotIdBody, +} from '@workflow/world'; import { decode, encode } from 'cbor-x'; import { MockAgent } from 'undici'; import { afterEach, describe, expect, it, vi } from 'vitest'; @@ -27,7 +33,7 @@ import { WORKFLOW_SERVER_URL_OVERRIDE } from './utils.js'; describe('throwForErrorResponse', () => { const call = ( status: number, - body = '{"message":"boom"}', + body: string | Uint8Array = '{"message":"boom"}', headers: Record = {} ) => throwForErrorResponse(status, headers, body, 'createEvent', 'http://x'); @@ -107,6 +113,109 @@ describe('throwForErrorResponse', () => { /createEvent failed: HTTP 500 plain text oops/ ); }); + + /** + * The slot-conflict 409 is the only v4 error body that arrives as CBOR: it + * carries the event-log delta the client replays from, whose payloads are + * byte strings that JSON cannot represent. Decoding it as JSON would lose the + * delta silently and mis-type the error as an entity conflict, which the + * runtime reads as "my write already landed". + */ + describe('slot conflict', () => { + const PAYLOAD = new Uint8Array([1, 2, 3]); + const conflictBody = ( + overrides: Record = {} + ): Uint8Array => + new Uint8Array( + encode({ + success: false, + error: 'slot-conflict', + message: "Event slot 'evnt_…003' is already taken", + details: { eventId: 'evnt_from_details' }, + events: [{ eventId: 'evnt_x', eventData: { output: PAYLOAD } }], + cursor: 'eid:evnt_x', + hasMore: false, + ...overrides, + }) + ); + + it('decodes a CBOR body into SlotConflictError with the delta intact', () => { + try { + call(409, conflictBody(), { + 'content-type': 'application/cbor', + 'x-wf-event-id': 'evnt_from_header', + }); + expect.unreachable(); + } catch (err) { + expect(SlotConflictError.is(err)).toBe(true); + const conflict = err as SlotConflictError; + expect(conflict.eventId).toBe('evnt_from_header'); + expect(conflict.cursor).toBe('eid:evnt_x'); + expect(conflict.hasMore).toBe(false); + // Binary payloads survive: this is what a JSON error path destroys. + expect(conflict.events).toHaveLength(1); + expect( + (conflict.events[0] as { eventData: { output: Uint8Array } }) + .eventData.output + ).toEqual(PAYLOAD); + // Not the 409 → EntityConflictError mapping, which the runtime reads as + // "the write I am retrying already landed". + expect(EntityConflictError.is(err)).toBe(false); + } + }); + + it('falls back to the eventId in details when the header is absent', () => { + try { + call(409, conflictBody(), { 'content-type': 'application/cbor' }); + expect.unreachable(); + } catch (err) { + expect((err as SlotConflictError).eventId).toBe('evnt_from_details'); + } + }); + + it('reports an empty delta when the backend could not read one', () => { + try { + call(409, conflictBody({ events: undefined, cursor: null }), { + 'content-type': 'application/cbor', + }); + expect.unreachable(); + } catch (err) { + const conflict = err as SlotConflictError; + expect(conflict.events).toEqual([]); + expect(conflict.cursor).toBeNull(); + } + }); + + it('reads a JSON-encoded slot conflict too', () => { + // Nothing in the protocol forbids a JSON encoding of the same body; only + // the delta's binary payloads require CBOR. + try { + call( + 409, + JSON.stringify({ + error: 'slot-conflict', + message: 'taken', + events: [], + cursor: 'eid:evnt_y', + hasMore: true, + }), + { 'x-wf-event-id': 'evnt_j' } + ); + expect.unreachable(); + } catch (err) { + expect(SlotConflictError.is(err)).toBe(true); + expect((err as SlotConflictError).hasMore).toBe(true); + } + }); + + it('leaves an ordinary 409 as EntityConflictError', () => { + // Entity materialization conflicts share the status and are how the + // runtime recognizes a duplicate write. + expect(() => + call(409, JSON.stringify({ error: 'conflict', message: 'exists' })) + ).toThrowError(EntityConflictError); + }); + }); }); /** @@ -271,7 +380,8 @@ describe('getWorkflowRunEventsV4 over HTTP', () => { */ describe('getEventV4 over HTTP', () => { it('returns the first frame and stops reading the rest', async () => { - const origin = 'https://vercel-workflow.com'; + const origin = + WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; const agent = new MockAgent(); agent.disableNetConnect(); @@ -508,7 +618,8 @@ describe('createWorkflowRunEventV4 over HTTP', () => { }); it('forwards stateUpdatedAt in the frame meta (precondition guard)', async () => { - const origin = 'https://vercel-workflow.com'; + const origin = + WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; const agent = new MockAgent(); agent.disableNetConnect(); @@ -560,7 +671,8 @@ describe('createWorkflowRunEventV4 over HTTP', () => { }); it('omits stateUpdatedAt from the frame meta when not set', async () => { - const origin = 'https://vercel-workflow.com'; + const origin = + WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; const agent = new MockAgent(); agent.disableNetConnect(); @@ -609,6 +721,117 @@ describe('createWorkflowRunEventV4 over HTTP', () => { expect('stateUpdatedAt' in (capturedMeta ?? {})).toBe(false); agent.assertNoPendingInterceptors(); }); + + it('sends the claimed eventId and maxSlot in the frame meta', async () => { + // A slot-numbered run names its own event ids, so the id has to reach the + // wire: the backend reads it from the frame meta and inserts it + // conditionally. Dropped, the backend mints a ULID instead and the run + // silently reverts to server-assigned identity mid-log. + const origin = + WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; + const agent = new MockAgent(); + agent.disableNetConnect(); + + let capturedMeta: Record | undefined; + agent + .get(origin) + .intercept({ + path: '/api/v4/runs/wrun_1/events/wait_created', + method: 'POST', + }) + .reply( + 200, + (opts: { body?: unknown }) => { + const bytes = new Uint8Array(opts.body as ArrayBufferLike); + const metaLen = new DataView( + bytes.buffer, + bytes.byteOffset, + bytes.byteLength + ).getUint32(0, false); + capturedMeta = decode(bytes.subarray(4, 4 + metaLen)) as Record< + string, + unknown + >; + return encode({ wait: { waitId: 'wait_1' } }); + }, + { + headers: { + 'x-wf-event-id': 'evnt_1', + 'x-wf-run-id': 'wrun_1', + 'x-wf-created-at': '2026-06-10T00:00:00.000Z', + }, + } + ); + + const eventId = slotEventId(4); + await createWorkflowRunEventV4( + { + runId: 'wrun_1', + eventType: 'wait_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: `wait_${slotIdBody(1)}`, + eventId, + maxSlot: 3, + }, + { token: 'test-token', dispatcher: agent } + ); + + expect(capturedMeta?.eventId).toBe(eventId); + expect(capturedMeta?.maxSlot).toBe(3); + agent.assertNoPendingInterceptors(); + }); + + it('omits eventId and maxSlot from the frame meta for a ULID-numbered run', async () => { + const origin = + WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; + const agent = new MockAgent(); + agent.disableNetConnect(); + + let capturedMeta: Record | undefined; + agent + .get(origin) + .intercept({ + path: '/api/v4/runs/wrun_1/events/wait_created', + method: 'POST', + }) + .reply( + 200, + (opts: { body?: unknown }) => { + const bytes = new Uint8Array(opts.body as ArrayBufferLike); + const metaLen = new DataView( + bytes.buffer, + bytes.byteOffset, + bytes.byteLength + ).getUint32(0, false); + capturedMeta = decode(bytes.subarray(4, 4 + metaLen)) as Record< + string, + unknown + >; + return encode({ wait: { waitId: 'wait_1' } }); + }, + { + headers: { + 'x-wf-event-id': 'evnt_1', + 'x-wf-run-id': 'wrun_1', + 'x-wf-created-at': '2026-06-10T00:00:00.000Z', + }, + } + ); + + await createWorkflowRunEventV4( + { + runId: 'wrun_1', + eventType: 'wait_created', + specVersion: 5, + correlationId: 'wait_1', + }, + { token: 'test-token', dispatcher: agent } + ); + + expect('eventId' in (capturedMeta ?? {})).toBe(false); + expect('maxSlot' in (capturedMeta ?? {})).toBe(false); + agent.assertNoPendingInterceptors(); + }); }); /** diff --git a/packages/world-vercel/src/events-v4.ts b/packages/world-vercel/src/events-v4.ts index 716dc35e50..0e6b11037e 100644 --- a/packages/world-vercel/src/events-v4.ts +++ b/packages/world-vercel/src/events-v4.ts @@ -21,6 +21,7 @@ * bytes — this module stays at the wire-bytes layer. */ +import { SlotConflictError } from '@workflow/errors'; import { decode } from 'cbor-x'; import { decodeFrames, encodeFrame, V4_FRAME_CONTENT_TYPE } from './frames.js'; import { getEventsDispatcher } from './http-client.js'; @@ -72,13 +73,33 @@ async function fetchV4( errorFromV4Response( response.status, headersToRecord(response.headers), - await response.text(), + await readErrorBody(response), opName, url ), }); } +/** + * The error body as bytes when it is CBOR, as text otherwise. + * + * Most v4 error responses are JSON, because the client sends no `Accept` header + * and the backend's error encoder defaults to it. A slot conflict is the + * exception: its body carries the event-log delta the client needs, whose + * payloads are byte strings that JSON cannot represent, so the backend encodes + * that one as CBOR regardless of the `Accept` header. + */ +async function readErrorBody(response: Response): Promise { + if (isCborContentType(response.headers.get('content-type'))) { + return new Uint8Array(await response.arrayBuffer()); + } + return await response.text(); +} + +function isCborContentType(contentType: string | null | undefined): boolean { + return contentType?.toLowerCase().includes('application/cbor') ?? false; +} + /** Flatten a fetch `Headers` into the record shape throwForErrorResponse * expects (it mirrors the v3 `makeRequest` error contract). */ function headersToRecord(headers: Headers): Record { @@ -209,6 +230,22 @@ export interface CreateEventV4Input { * without a loaded event log; older servers ignore it entirely. */ stateUpdatedAt?: number; + /** + * The event's id, claimed by the client instead of minted by the server. + * Sent only for a run that numbers its events by slot, where the id encodes + * the event's position in the log. The server inserts it conditionally and + * answers 409 `slot-conflict` when the slot is already taken; a run on the + * older numbering that sends one is rejected with 400. Older servers ignore + * the field and mint an id as before — which is why only runs stamped with + * slot identity ever send it. + */ + eventId?: string; + /** + * The highest slot the client has seen in the run's event log (0 when it has + * seen none). Observability only: slots are dense, so a persisted slot more + * than one past this is a permanent hole in the log. Ignored by older servers. + */ + maxSlot?: number; /** Number of consecutive replay divergences resolved by this write. */ replayDivergenceCount?: number; } @@ -311,12 +348,81 @@ function buildPostFrameMeta( if (input.stateUpdatedAt !== undefined) { meta.stateUpdatedAt = input.stateUpdatedAt; } + if (input.eventId !== undefined) meta.eventId = input.eventId; + if (input.maxSlot !== undefined) meta.maxSlot = input.maxSlot; if (input.replayDivergenceCount !== undefined) { meta.replayDivergenceCount = input.replayDivergenceCount; } return meta; } +/** + * The backend's machine-readable code for a lost event slot. Paired with 409 + * rather than 412 so a slot conflict stays distinguishable from the + * `stateUpdatedAt` watermark's staleness rejection while both are live. + */ +const V4_SLOT_CONFLICT_CODE = 'slot-conflict'; + +/** The fields a v4 error body may carry, whatever encoding it arrived in. */ +interface V4ErrorBody { + message?: unknown; + /** Machine-readable code. The backend names this field `error`. */ + error?: unknown; + code?: unknown; + events?: unknown; + cursor?: unknown; + hasMore?: unknown; + details?: unknown; +} + +/** Decode an error body as CBOR or JSON, or `undefined` if it is neither. */ +function decodeErrorBody( + errorBody: string | Uint8Array +): V4ErrorBody | undefined { + try { + const value = + typeof errorBody === 'string' + ? (JSON.parse(errorBody) as unknown) + : (decode(errorBody) as unknown); + return value && typeof value === 'object' + ? (value as V4ErrorBody) + : undefined; + } catch { + return undefined; + } +} + +/** + * Build the `SlotConflictError` for a 409 whose body names a taken slot. + * + * The conflicting event id comes from the response header rather than the body + * so the error is still actionable when the body failed to decode; the delta is + * best-effort in the other direction — an absent or malformed `events` leaves + * the runtime to reload the log itself, which is always correct. + */ +function slotConflictFromBody( + message: string, + responseHeaders: Record, + body: V4ErrorBody | undefined +): SlotConflictError { + const details = body?.details; + const detailEventId = + details && typeof details === 'object' && 'eventId' in details + ? (details as { eventId?: unknown }).eventId + : undefined; + const headerEventId = readHeader( + responseHeaders, + V4_RESPONSE_HEADERS.eventId + ); + return new SlotConflictError(message, { + eventId: + headerEventId ?? (typeof detailEventId === 'string' ? detailEventId : ''), + events: Array.isArray(body?.events) ? body.events : [], + cursor: typeof body?.cursor === 'string' ? body.cursor : null, + hasMore: body?.hasMore === true, + }); +} + /** * Build the typed error for a non-2xx v4 response. Reuses the shared * `errorForResponse` status → error-type contract (409→EntityConflictError, @@ -329,19 +435,27 @@ function buildPostFrameMeta( function errorFromV4Response( statusCode: number, responseHeaders: Record, - errorBody: string, + errorBody: string | Uint8Array, opName: string, url: string ): Error { let message = `v4 ${opName} failed: HTTP ${statusCode}`; let code: string | undefined; - try { - const json = JSON.parse(errorBody) as { message?: string; code?: string }; - if (typeof json.message === 'string') message = json.message; - if (typeof json.code === 'string') code = json.code; - } catch { - // body wasn't JSON — keep the default message, append raw text below - if (errorBody) message += ` ${errorBody}`; + const decoded = decodeErrorBody(errorBody); + if (decoded) { + if (typeof decoded.message === 'string') message = decoded.message; + if (typeof decoded.code === 'string') code = decoded.code; + } else if (typeof errorBody === 'string' && errorBody) { + // Body was neither JSON nor CBOR — keep the default message and append the + // raw text so the response is still diagnosable. + message += ` ${errorBody}`; + } + + // A lost event slot is the one 409 that is not an entity conflict. The + // backend names its machine-readable code `error`; that field is read only + // here, so every other error keeps the status → type mapping below unchanged. + if (statusCode === 409 && decoded?.error === V4_SLOT_CONFLICT_CODE) { + return slotConflictFromBody(message, responseHeaders, decoded); } const retryAfter = parseRetryAfter( @@ -364,7 +478,7 @@ function errorFromV4Response( export function throwForErrorResponse( statusCode: number, responseHeaders: Record, - errorBody: string, + errorBody: string | Uint8Array, opName: string, url: string ): never { diff --git a/packages/world-vercel/src/events.ts b/packages/world-vercel/src/events.ts index 4169c61a35..27be450ca1 100644 --- a/packages/world-vercel/src/events.ts +++ b/packages/world-vercel/src/events.ts @@ -32,7 +32,11 @@ * the v3 path. */ -import { HookNotFoundError, WorkflowWorldError } from '@workflow/errors'; +import { + HookNotFoundError, + SlotConflictError, + WorkflowWorldError, +} from '@workflow/errors'; import { type AnyEventRequest, type CreateEventParams, @@ -465,6 +469,35 @@ function coerceNormalizedEvent(raw: Record): Event { return coerceEventDates(normalizeEventData(raw)); } +/** + * Runs an event create, normalizing the event-log delta a slot conflict carries + * into the same `Event` shape every other read path produces. + * + * The delta arrives as raw CBOR off the error response, so its nested dates are + * still ISO strings; the runtime merges these events into its loaded log and + * calls `.getTime()` on them, exactly as it does for the inline delta on the + * success path. Doing the coercion here rather than in the runtime keeps the + * wire's shape a concern of this adapter, and keeps `SlotConflictError.events` + * meaning the same thing for every World that raises it. + */ +async function withCoercedSlotConflictDelta( + op: () => Promise +): Promise { + try { + return await op(); + } catch (error) { + if (!SlotConflictError.is(error) || error.events.length === 0) { + throw error; + } + throw new SlotConflictError(error.message, { + eventId: error.eventId, + events: (error.events as Record[]).map(coerceEventDates), + cursor: error.cursor, + hasMore: error.hasMore, + }); + } +} + function decodeLegacyStructuredError(payload: Uint8Array): unknown { if (hasSerializedDataFormatPrefix(payload)) { return payload; @@ -618,9 +651,11 @@ export async function createWorkflowRunEvent( // the next queue delivery. Non-retryable // types (step_started, step_retrying, hook_received) run once. See // ./event-retry for the validated per-event classification. - return await withEventPostRetry( - () => createWorkflowRunEventInner(id, data, params, config), - data.eventType + return await withCoercedSlotConflictDelta(() => + withEventPostRetry( + () => createWorkflowRunEventInner(id, data, params, config), + data.eventType + ) ); } catch (err) { // 404 on hook_disposed / hook_received → already-disposed hook. @@ -727,6 +762,13 @@ async function createWorkflowRunEventInner( // skip the list+resolve. The server only acts on it for run_started; // older servers ignore it and simply preload as before. ...(params?.skipPreload ? { skipPreload: true } : {}), + // Slot identity: the runtime names the event's own id, claiming that + // position in the run's event log. The server inserts it conditionally + // and answers 409 slot-conflict when another writer got there first. + // `maxSlot` rides along so the server can spot a gap, which slots being + // dense makes an unrecoverable corruption. + ...(params?.eventId ? { eventId: params.eventId } : {}), + ...(params?.maxSlot !== undefined ? { maxSlot: params.maxSlot } : {}), remoteRefBehavior, payload, ...meta, diff --git a/packages/world-vercel/src/index.ts b/packages/world-vercel/src/index.ts index 42ff155fda..6a68e6845a 100644 --- a/packages/world-vercel/src/index.ts +++ b/packages/world-vercel/src/index.ts @@ -1,5 +1,5 @@ import type { World } from '@workflow/world'; -import { SPEC_VERSION_SUPPORTS_COMPRESSION } from '@workflow/world'; +import { mintedSpecVersion } from '@workflow/world'; import { createAnalytics } from './analytics.js'; import { createRunId, describeRun } from './create-run-id.js'; import { createGetEncryptionKeyForRun } from './encryption.js'; @@ -29,9 +29,12 @@ export function createWorld(config?: APIConfig): World { config?.projectConfig?.projectId || process.env.VERCEL_PROJECT_ID; return { - // Spec v5 adds client-side zstd/gzip payload compression. The server stores - // those payloads opaquely, and v5 remains a superset of v4 attributes. - specVersion: SPEC_VERSION_SUPPORTS_COMPRESSION, + // What this world stamps on new runs: slot identity (spec v6) unless + // WORKFLOW_SLOT_IDENTITY switches it off, in which case v5 — client-side + // zstd/gzip payload compression over a superset of the v4 attributes. + // Either way this world reads both, so the stamp only decides how the runs + // it creates from here on are numbered. + specVersion: mintedSpecVersion(), capabilities: { // workflow-server enforces the `stateUpdatedAt` optimistic-concurrency // guard: creations carrying a stale snapshot are rejected with 412 diff --git a/packages/world-vercel/src/utils.ts b/packages/world-vercel/src/utils.ts index 60157de448..efe9ba0fca 100644 --- a/packages/world-vercel/src/utils.ts +++ b/packages/world-vercel/src/utils.ts @@ -32,7 +32,10 @@ import { version } from './version.js'; * `main` — rewritten by external CI for branch-deployment testing. * Prefer `VERCEL_WORKFLOW_SERVER_URL` for deployment-time configuration. */ -export const WORKFLOW_SERVER_URL_OVERRIDE = ''; +// TEMPORARY — revert to '' before merge. Points e2e at the slot-identity +// backend branch deployment (top of the paired backend stack). +export const WORKFLOW_SERVER_URL_OVERRIDE = + 'https://workflow-server-git-peter-slot-ids-6-guards.vercel.sh'; /** * HTTP methods that are safe to transparently re-issue inside the adapter. diff --git a/packages/world/package.json b/packages/world/package.json index 681c1d3327..84d5f1b1a0 100644 --- a/packages/world/package.json +++ b/packages/world/package.json @@ -20,6 +20,7 @@ "scripts": { "build": "tsc", "dev": "tsc --watch", + "test": "vitest run src", "clean": "tsc --build --clean && rm -rf dist" }, "dependencies": { diff --git a/packages/world/src/events.ts b/packages/world/src/events.ts index 9f04e2fc9d..f87af16d54 100644 --- a/packages/world/src/events.ts +++ b/packages/world/src/events.ts @@ -788,6 +788,37 @@ export interface CreateEventParams { * across the SDK and the backend. */ skipPreload?: boolean; + /** + * The event's id, chosen by the client rather than the World. + * + * Only sent for a run whose spec version numbers events by slot + * (`SPEC_VERSION_SLOT_IDENTITY`), where the id encodes the event's position in + * the log and is therefore the client's claim on that position. + * + * Backend contract (for World implementers who want to support slot + * identity): treat the id as a claim to be won, not a hint. Insert it under a + * uniqueness constraint on `(runId, eventId)` and, when the id is already + * taken, reject the write with `SlotConflictError` (HTTP 409) instead of + * minting a different id — a lost slot means the client replayed against an + * event log missing at least one event, so its whole proposed event, not just + * its id, is suspect. Reject a mismatch in either direction with a 400: an id + * of this shape on a run that does not use slot identity, or an absent or + * ULID-shaped id on a run that does, would leave the log unable to prove its + * own completeness. + * + * A World that ignores this field keeps minting ids itself, which is correct + * only for runs that were never stamped with slot identity in the first + * place. + */ + eventId?: string; + /** + * The highest slot the client has observed in the run's event log, or 0 for a + * log with no slot-numbered events. Sent alongside {@link eventId} purely as + * an observability signal: because slots are dense, a persisted slot more + * than one past this is a hole, which is unrecoverable and worth alerting on. + * Worlds MAY ignore it. + */ + maxSlot?: number; } /** diff --git a/packages/world/src/index.ts b/packages/world/src/index.ts index 3d13620de8..1ecae0457d 100644 --- a/packages/world/src/index.ts +++ b/packages/world/src/index.ts @@ -101,16 +101,34 @@ export { PaginatedResponseSchema, StructuredErrorSchema, } from './shared.js'; +export { + FIRST_SLOT, + isSlotId, + maxSlotOf, + SLOT_ID_WIDTH, + SLOT_RETRY_BASE_MS, + SLOT_RETRY_BUDGET_MS, + SLOT_RETRY_MAX_DELAY_MS, + slotEventId, + slotFromId, + slotIdBody, + slotRetryDelay, +} from './slot-identity.js'; export type { SpecVersion } from './spec-version.js'; export { isLegacySpecVersion, + mintedSpecVersion, requiresNewerWorld, + SLOT_IDENTITY_ENV_VAR, SPEC_VERSION_CURRENT, SPEC_VERSION_LEGACY, + SPEC_VERSION_MAX_SUPPORTED, + SPEC_VERSION_SLOT_IDENTITY, SPEC_VERSION_SUPPORTS_ATTRIBUTES, SPEC_VERSION_SUPPORTS_CBOR_QUEUE_TRANSPORT, SPEC_VERSION_SUPPORTS_COMPRESSION, SPEC_VERSION_SUPPORTS_EVENT_SOURCING, + usesSlotIdentity, } from './spec-version.js'; export type * from './steps.js'; export { diff --git a/packages/world/src/slot-identity.test.ts b/packages/world/src/slot-identity.test.ts new file mode 100644 index 0000000000..acdfada627 --- /dev/null +++ b/packages/world/src/slot-identity.test.ts @@ -0,0 +1,107 @@ +import { ulid } from 'ulid'; +import { describe, expect, it } from 'vitest'; +import { z } from 'zod'; +import { + FIRST_SLOT, + isSlotId, + maxSlotOf, + SLOT_ID_WIDTH, + slotEventId, + slotFromId, + slotIdBody, +} from './slot-identity.js'; +import { ulidToDate } from './ulid.js'; + +describe('slotIdBody', () => { + it('pads to ULID width so a slot is accepted wherever a ULID is', () => { + const body = slotIdBody(FIRST_SLOT); + expect(body).toHaveLength(SLOT_ID_WIDTH); + expect(`evnt_${body}`).toHaveLength(`evnt_${ulid()}`.length); + // Crockford base32 starts with the decimal digits, so the padded body + // satisfies the ULID syntax — this is what keeps every existing schema, + // sort key and range fence working unchanged. + expect(z.string().ulid().safeParse(body).success).toBe(true); + }); + + it('orders lexicographically by slot at a fixed width', () => { + const ascending = [1, 2, 9, 10, 100].map(slotIdBody); + expect([...ascending].sort()).toEqual(ascending); + }); + + it('rejects slots outside the dense numbering', () => { + expect(() => slotIdBody(0)).toThrow(); + expect(() => slotIdBody(-1)).toThrow(); + expect(() => slotIdBody(1.5)).toThrow(); + }); +}); + +describe('slotFromId', () => { + it('round-trips a prefixed id', () => { + expect(slotFromId(`step_${slotIdBody(42)}`)).toBe(42); + }); + + it('round-trips a bare body', () => { + expect(slotFromId(slotIdBody(42))).toBe(42); + }); + + it('reads no slot out of a ULID id', () => { + expect(slotFromId(`evnt_${ulid()}`)).toBeUndefined(); + expect(isSlotId(`evnt_${ulid()}`)).toBe(false); + }); + + it('reads no slot out of the all-zero range fence', () => { + // Slot 0 is the inclusive lower fence for range queries over a run's + // events, never an event. + expect(slotFromId('0'.repeat(SLOT_ID_WIDTH))).toBeUndefined(); + }); + + it('reads no slot out of a body of the wrong width', () => { + expect(slotFromId('evnt_1')).toBeUndefined(); + expect(slotFromId(`evnt_${'1'.repeat(SLOT_ID_WIDTH + 1)}`)).toBeUndefined(); + }); +}); + +describe('a slot carries no timestamp', () => { + it('reports no time rather than epoch 0', () => { + // Passing the ULID syntax check is what makes a slot portable; decoding a + // *time* out of one is always a bug. Two that this guards: the sandbox + // clock is set from the events it consumes, so epoch 0 would rewind a + // replaying workflow's `Date.now()` to 1970; and world-local prefilters + // cursor pagination on the time in the filename, so epoch 0 would hide + // every slot-numbered event from an ascending page. + expect(ulidToDate(slotIdBody(FIRST_SLOT))).toBeNull(); + expect(ulidToDate(slotEventId(FIRST_SLOT))).toBeNull(); + expect(ulidToDate(slotIdBody(123_456))).toBeNull(); + }); + + it('still reads the time out of a ULID', () => { + expect(ulidToDate(ulid())?.getTime()).toBeGreaterThan(0); + }); +}); + +describe('maxSlotOf', () => { + it('finds the highest slot regardless of position', () => { + // A log is merged from several loads and is not sorted, so the last element + // is not necessarily the highest slot. + expect( + maxSlotOf([slotEventId(3), slotEventId(7), slotEventId(1)].map(toEvent)) + ).toBe(7); + }); + + it('reports 0 for an empty or ULID-numbered log', () => { + expect(maxSlotOf([])).toBe(0); + expect(maxSlotOf([toEvent(`evnt_${ulid()}`)])).toBe(0); + }); + + it('ignores ULID ids mixed in with slots', () => { + // A mixed log violates slot identity's purity invariant, but the scan must + // still report the highest slot rather than throwing or returning 0. + expect( + maxSlotOf([toEvent(`evnt_${ulid()}`), toEvent(slotEventId(2))]) + ).toBe(2); + }); +}); + +function toEvent(eventId: string): { eventId: string } { + return { eventId }; +} diff --git a/packages/world/src/slot-identity.ts b/packages/world/src/slot-identity.ts new file mode 100644 index 0000000000..203005c719 --- /dev/null +++ b/packages/world/src/slot-identity.ts @@ -0,0 +1,117 @@ +/** + * Slot identity: dense, per-run numbering for event ids and correlation ids. + * + * A slot id's body is a decimal counter zero-padded to ULID width. Crockford's + * base32 alphabet begins with the ten decimal digits, so that body is a + * syntactically valid ULID body — every schema, sort key, range fence and + * cursor that accepted a ULID keeps accepting a slot — and because the width is + * fixed, lexicographic order is numeric order. + * + * Slots start at 1 and are allocated contiguously, which is what makes + * contention explicit: two writers proposing one position cannot both win, and + * the loser is told which events it was missing. Zero is left unused because + * the inclusive lower fence for range queries over a run's events is the + * all-zero id. + * + * Allocation being contiguous does not make a published log gap-free, so + * `events.length === maxSlot` is not a completeness proof. A slot claimed by an + * operation that then fails for a reason of its own is never filled, and if a + * later slot has already been published the gap is permanent. Nothing may treat + * a missing slot as an event still on its way. + * + * A slot body decodes as a ULID *timestamp* of epoch 0 without erroring, so + * nothing may read a time out of one. Use the event's own `createdAt` / + * `occurredAt`. + */ + +/** Width of a slot id's body: ULID width, so a slot is accepted wherever a ULID is. */ +export const SLOT_ID_WIDTH = 26; + +/** First slot in a run. Slot 0 is unused — it is the inclusive range fence. */ +export const FIRST_SLOT = 1; + +const SLOT_BODY_PATTERN = new RegExp(`^[0-9]{${SLOT_ID_WIDTH}}$`); + +/** + * The id body naming `slot`, e.g. `1` → `00000000000000000000000001`. Callers + * prepend their own prefix (`evnt_`, `step_`, `wait_`). + */ +export function slotIdBody(slot: number): string { + if (!Number.isInteger(slot) || slot < FIRST_SLOT) { + throw new Error( + `Slot must be an integer >= ${FIRST_SLOT}, received ${slot}` + ); + } + const body = String(slot).padStart(SLOT_ID_WIDTH, '0'); + if (body.length > SLOT_ID_WIDTH) { + throw new Error(`Slot ${slot} does not fit in ${SLOT_ID_WIDTH} digits`); + } + return body; +} + +/** + * The slot named by an id, or undefined if the id is not a slot id. Accepts + * both a prefixed id (`step_0…001`) and a bare body. + */ +export function slotFromId(id: string): number | undefined { + const underscore = id.indexOf('_'); + const body = underscore === -1 ? id : id.slice(underscore + 1); + if (!SLOT_BODY_PATTERN.test(body)) { + return undefined; + } + const slot = Number(body); + return slot >= FIRST_SLOT ? slot : undefined; +} + +/** Whether an id numbers itself by slot rather than by ULID. */ +export function isSlotId(id: string): boolean { + return slotFromId(id) !== undefined; +} + +/** The event id occupying `slot`. */ +export function slotEventId(slot: number): string { + return `evnt_${slotIdBody(slot)}`; +} + +/** First backoff after losing a position; doubled each round. */ +export const SLOT_RETRY_BASE_MS = 5; + +/** Ceiling for a single backoff, so a contended run keeps making attempts. */ +export const SLOT_RETRY_MAX_DELAY_MS = 250; + +/** + * How long a writer that allocates its own position keeps looking for a free + * one before giving up. Exhausting it is a retryable failure for the caller — + * in practice a queue delivery — rather than something the run stalls on. + */ +export const SLOT_RETRY_BUDGET_MS = 30_000; + +/** + * Full jitter over an exponentially growing, capped window. Shared by every + * world that allocates positions, so contention behaves the same wherever a run + * is stored. + */ +export function slotRetryDelay(round: number): number { + return ( + Math.random() * + Math.min(SLOT_RETRY_BASE_MS * 2 ** round, SLOT_RETRY_MAX_DELAY_MS) + ); +} + +/** + * The highest slot named by any of `events`, or 0 when none is slot-numbered. + * + * Scans rather than reading the last element: a log is merged from several + * loads and is not necessarily sorted, and callers use this value to pick the + * next free slot. + */ +export function maxSlotOf(events: readonly { eventId: string }[]): number { + let max = 0; + for (const event of events) { + const slot = slotFromId(event.eventId); + if (slot !== undefined && slot > max) { + max = slot; + } + } + return max; +} diff --git a/packages/world/src/spec-version.test.ts b/packages/world/src/spec-version.test.ts index 880c43180d..0125e2fb1c 100644 --- a/packages/world/src/spec-version.test.ts +++ b/packages/world/src/spec-version.test.ts @@ -1,9 +1,13 @@ import { describe, expect, it } from 'vitest'; import { isLegacySpecVersion, + mintedSpecVersion, requiresNewerWorld, + SLOT_IDENTITY_ENV_VAR, SPEC_VERSION_CURRENT, SPEC_VERSION_LEGACY, + SPEC_VERSION_MAX_SUPPORTED, + SPEC_VERSION_SLOT_IDENTITY, SPEC_VERSION_SUPPORTS_ATTRIBUTES, SPEC_VERSION_SUPPORTS_COMPRESSION, } from './spec-version.js'; @@ -13,10 +17,21 @@ describe('spec version constants', () => { expect(SPEC_VERSION_CURRENT).toBe(SPEC_VERSION_SUPPORTS_COMPRESSION); expect(SPEC_VERSION_SUPPORTS_COMPRESSION).toBe(5); }); + + it('can read a newer spec version than it mints', () => { + // Slot identity is readable by every world before any world mints it, so + // that turning it on for new runs cannot make those same worlds reject + // them. Once slots are the default the two constants coincide again. + expect(SPEC_VERSION_MAX_SUPPORTED).toBe(SPEC_VERSION_SLOT_IDENTITY); + expect(SPEC_VERSION_MAX_SUPPORTED).toBeGreaterThanOrEqual( + SPEC_VERSION_CURRENT + ); + }); }); describe('requiresNewerWorld', () => { - it('accepts runs at or below the current spec version', () => { + it('accepts runs at or below the newest readable spec version', () => { + expect(requiresNewerWorld(SPEC_VERSION_MAX_SUPPORTED)).toBe(false); expect(requiresNewerWorld(SPEC_VERSION_CURRENT)).toBe(false); expect(requiresNewerWorld(SPEC_VERSION_SUPPORTS_ATTRIBUTES)).toBe(false); expect(requiresNewerWorld(SPEC_VERSION_LEGACY)).toBe(false); @@ -24,13 +39,20 @@ describe('requiresNewerWorld', () => { expect(requiresNewerWorld(null)).toBe(false); }); - it('rejects runs newer than the current spec version', () => { + it('accepts a slot-identity run', () => { + // Gates the flag rollout: a world that rejected spec-6 would reject the + // runs it had just stamped spec-6 itself, at their first event after + // run_created. + expect(requiresNewerWorld(SPEC_VERSION_SLOT_IDENTITY)).toBe(false); + }); + + it('rejects runs newer than the newest readable spec version', () => { // This is the contract that protects older SDKs from compressed // payloads they cannot decode: a spec-5 run read by an SDK whose - // SPEC_VERSION_CURRENT is 4 fails this check up front (with - // RunNotSupportedError at the storage layer) instead of failing on - // individual compressed payloads. - expect(requiresNewerWorld(SPEC_VERSION_CURRENT + 1)).toBe(true); + // ceiling is 4 fails this check up front (with RunNotSupportedError at + // the storage layer) instead of failing on individual compressed + // payloads. + expect(requiresNewerWorld(SPEC_VERSION_MAX_SUPPORTED + 1)).toBe(true); }); it('simulates a v4 reader rejecting a compression-era run', () => { @@ -51,3 +73,32 @@ describe('isLegacySpecVersion', () => { expect(isLegacySpecVersion(5)).toBe(false); }); }); + +describe('mintedSpecVersion', () => { + it('mints slot identity by default', () => { + expect(mintedSpecVersion({})).toBe(SPEC_VERSION_SLOT_IDENTITY); + }); + + it('mints the previous version when the flag is switched off', () => { + for (const value of ['0', 'false']) { + expect(mintedSpecVersion({ [SLOT_IDENTITY_ENV_VAR]: value })).toBe( + SPEC_VERSION_CURRENT + ); + } + }); + + it('treats any other value as on', () => { + // An unset-but-present variable is the shape a shell leaves behind, and it + // must not silently switch a deployment's event identity scheme. Opting + // out takes an explicit `0`/`false`. + for (const value of ['', '1', 'true', 'yes']) { + expect(mintedSpecVersion({ [SLOT_IDENTITY_ENV_VAR]: value })).toBe( + SPEC_VERSION_SLOT_IDENTITY + ); + } + }); + + it('mints nothing a world cannot read', () => { + expect(requiresNewerWorld(mintedSpecVersion({}))).toBe(false); + }); +}); diff --git a/packages/world/src/spec-version.ts b/packages/world/src/spec-version.ts index cf516b772b..4b830fd4a8 100644 --- a/packages/world/src/spec-version.ts +++ b/packages/world/src/spec-version.ts @@ -31,13 +31,70 @@ export const SPEC_VERSION_SUPPORTS_ATTRIBUTES = 4 as SpecVersion; */ export const SPEC_VERSION_SUPPORTS_COMPRESSION = 5 as SpecVersion; +/** + * Runs at this spec version or later number their events, and their step and + * wait correlation ids, by dense per-run slots (`evnt_…001`, `step_…001`) + * instead of ULIDs. Ids are minted by the client, and a write that proposes an + * event id already taken is rejected rather than renumbered — which is what + * lets a client prove its loaded log is complete. + * + * A run is in exactly one mode for life: the mode is read from the run's + * persisted `specVersion`, never from the build. A run started under ULID + * correlation ids and replayed by a slot-capable build would otherwise propose + * `step_…001` where its log holds `step_01K…`, matching no existing entity. + */ +export const SPEC_VERSION_SLOT_IDENTITY = 6 as SpecVersion; + /** * Current spec version (event-sourced architecture with native attributes * and compressed payloads). + * + * The floor a world stamps on new runs, and a *lower* bar than the newest + * version this build can read — see {@link SPEC_VERSION_MAX_SUPPORTED}. What a + * world actually stamps comes from {@link mintedSpecVersion}; this is what it + * falls back to when slot identity is switched off. */ export const SPEC_VERSION_CURRENT = SPEC_VERSION_SUPPORTS_COMPRESSION as SpecVersion; +/** + * Newest spec version this build can read. Runs above it are rejected outright + * by {@link requiresNewerWorld} rather than misread. + * + * Distinct from {@link SPEC_VERSION_CURRENT} because a world has to be able to + * read a version before anything may mint it, and because a world that mints + * slot identity still has to read the spec-5 runs it created before the switch. + * Worlds opt into minting individually, via the `specVersion` they declare. + */ +export const SPEC_VERSION_MAX_SUPPORTED = + SPEC_VERSION_SLOT_IDENTITY as SpecVersion; + +/** + * Environment variable that opts new runs out of slot identity. + * + * Read per `createWorld()` call rather than at module load, so a test or a + * single process can create worlds in both modes. + */ +export const SLOT_IDENTITY_ENV_VAR = 'WORKFLOW_SLOT_IDENTITY'; + +/** + * The spec version a world should stamp on the runs it creates: slot identity + * unless {@link SLOT_IDENTITY_ENV_VAR} disables it, in which case + * {@link SPEC_VERSION_CURRENT}. + * + * Every world reads runs up to {@link SPEC_VERSION_MAX_SUPPORTED} whatever this + * returns, so turning the flag off in one place does not make the runs another + * process created unreadable here. + */ +export function mintedSpecVersion( + env: Record = process.env +): SpecVersion { + const value = env[SLOT_IDENTITY_ENV_VAR]; + return value === '0' || value === 'false' + ? SPEC_VERSION_CURRENT + : SPEC_VERSION_SLOT_IDENTITY; +} + /** * Check if a spec version is legacy (<= SPEC_VERSION_LEGACY or undefined). * Legacy runs require different handling - they use direct entity mutation @@ -55,7 +112,7 @@ export function isLegacySpecVersion(v: number | undefined | null): boolean { } /** - * Check if a spec version requires a newer world (> SPEC_VERSION_CURRENT). + * Check if a spec version requires a newer world (> SPEC_VERSION_MAX_SUPPORTED). * This happens when a run was created by a newer SDK version. * * @param v - The spec version number, or undefined/null for legacy runs @@ -63,5 +120,17 @@ export function isLegacySpecVersion(v: number | undefined | null): boolean { */ export function requiresNewerWorld(v: number | undefined | null): boolean { if (v === undefined || v === null) return false; - return v > SPEC_VERSION_CURRENT; + return v > SPEC_VERSION_MAX_SUPPORTED; +} + +/** + * Whether a run numbers its events and correlation ids by slot. Always pass the + * run's persisted `specVersion`; see `SPEC_VERSION_SLOT_IDENTITY`. + * + * @param v - The spec version number, or undefined/null for legacy runs + * @returns true if the run uses slot identity + */ +export function usesSlotIdentity(v: number | undefined | null): boolean { + if (v === undefined || v === null) return false; + return v >= SPEC_VERSION_SLOT_IDENTITY; } diff --git a/packages/world/src/ulid.ts b/packages/world/src/ulid.ts index 1ee1b7b47c..cc893834c3 100644 --- a/packages/world/src/ulid.ts +++ b/packages/world/src/ulid.ts @@ -1,5 +1,6 @@ import { decodeTime } from 'ulid'; import { z } from 'zod'; +import { isSlotId } from './slot-identity.js'; const UlidSchema = z.string().ulid(); @@ -36,8 +37,19 @@ export const DEFAULT_TIMESTAMP_THRESHOLD_MS = /** * Extracts a Date from a ULID string, or null if the string is not a valid ULID. + * + * Slot ids are not ULIDs even though they pass the ULID *syntax* check: their + * body is all decimal digits, which Crockford base32 accepts, and it would + * decode to a timestamp of epoch 0 instead of failing. A slot encodes a + * position, not a time, so it is reported here as having no time at all — + * callers must read the object's own `createdAt`. Silently returning 1970 + * instead would, among other things, rewind a replaying workflow's clock and + * make cursor pagination skip every slot-numbered event. */ export function ulidToDate(maybeUlid: string): Date | null { + if (isSlotId(maybeUlid)) { + return null; + } const ulid = UlidSchema.safeParse(maybeUlid); if (!ulid.success) { return null;