From 8411c69eb246f26798f96e24eb00695e9367834d Mon Sep 17 00:00:00 2001 From: Pranay Prakash Date: Tue, 18 Aug 2026 15:23:44 -0700 Subject: [PATCH 01/10] =?UTF-8?q?feat(world,core):=20sealed-log=20spec=207?= =?UTF-8?q?=20=E2=80=94=20sequenced=20positions,=20replay=20skips=20server?= =?UTF-8?q?=20noops?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Client half of the sealed-log design (server: workflow-server pgp/sealed-log-sequencer). Spec 7 runs get their slot positions from a per-run sequencer on the backend, so concurrent writers never race for a position — and a position whose writer died is filled by the backend with a server-written `noop` event ("sealing") to keep the log a dense prefix. - @workflow/world: SPEC_VERSION_SUPPORTS_SEALED_LOG = 7; SPEC_VERSION_CURRENT and SPEC_VERSION_MAX_SUPPORTED move to it together (what we stamp and what we can read). `noop` joins the read union (AllEventsSchema) but stays out of CreateEventSchema — it is never user-creatable. - @workflow/core: EventsConsumer steps over noops without offering them to any consumer and WITHOUT advancing the deterministic clock — a noop's createdAt is the sealer's wall clock and can postdate later slots, so a log containing one must produce the same timestamps as a log whose hole was filled by the real writer (same rule as skipDuplicateEvent). - @workflow/world-vercel: noop entries in the exhaustive retry-policy and v4 response-schema maps (never POSTed; server-originated). - CI (DO NOT MERGE as-is): e2e VERCEL_WORKFLOW_SERVER_URL pinned to the sealed-log server preview so this PR's suite exercises spec-7 sequencing + sealing end-to-end; revert to the secret before landing. Co-Authored-By: Claude Fable 5 --- .changeset/sealed-log-spec-seven.md | 7 ++ .github/workflows/tests.yml | 18 ++++- packages/core/src/events-consumer.test.ts | 93 +++++++++++++++++++++++ packages/core/src/events-consumer.ts | 22 ++++++ packages/core/src/runtime/start.test.ts | 17 +++-- packages/world-vercel/src/event-retry.ts | 6 ++ packages/world-vercel/src/events-v4.ts | 3 + packages/world/src/events.test.ts | 36 +++++++++ packages/world/src/events.ts | 24 ++++++ packages/world/src/index.ts | 1 + packages/world/src/spec-version.test.ts | 21 +++-- packages/world/src/spec-version.ts | 20 ++++- 12 files changed, 243 insertions(+), 25 deletions(-) create mode 100644 .changeset/sealed-log-spec-seven.md diff --git a/.changeset/sealed-log-spec-seven.md b/.changeset/sealed-log-spec-seven.md new file mode 100644 index 0000000000..cd52698930 --- /dev/null +++ b/.changeset/sealed-log-spec-seven.md @@ -0,0 +1,7 @@ +--- +'@workflow/world': minor +'@workflow/world-vercel': minor +'@workflow/core': minor +--- + +Sealed-log event identity (specVersion 7): runs are stamped at spec 7, whose slot positions come from a per-run sequencer on the backend instead of writers racing conditional creates. The backend may fill a position whose writer died with a server-written `noop` event ("sealing"); the runtime skips noops during replay without advancing the deterministic clock, and the read union accepts the new event type. `SPEC_VERSION_SUPPORTS_SEALED_LOG` is exported from `@workflow/world`. diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a9e702af56..f55c29edae 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -481,7 +481,12 @@ jobs: # unset for changeset-release PRs: they test a production deployment # that is wired to the production workflow-server, and the harness # has to read run state from the same server the app writes it to. - VERCEL_WORKFLOW_SERVER_URL: ${{ github.ref != 'refs/heads/main' && !startsWith(github.head_ref, 'changeset-release/') && secrets.VERCEL_WORKFLOW_SERVER_URL || '' }} + # TODO(sealed-log): DO NOT MERGE as-is — pinned to the sealed-log + # sequencer server preview (workflow-server PR) so this PR's e2e + # suite exercises spec-7 sequencing + noop sealing end-to-end. + # Revert to the secrets.VERCEL_WORKFLOW_SERVER_URL expression + # before landing. + VERCEL_WORKFLOW_SERVER_URL: https://workflow-server-git-pgp-sealed-log-sequencer.vercel.sh - name: Capture runtime logs on failure if: failure() @@ -595,7 +600,12 @@ jobs: WORKFLOW_VERCEL_PROJECT_SLUG: "example-nextjs-workflow-turbopack" # See the note on e2e-vercel-prod: PRs point at the protected # workflow-server preview; unset on main for production. - VERCEL_WORKFLOW_SERVER_URL: ${{ github.ref != 'refs/heads/main' && !startsWith(github.head_ref, 'changeset-release/') && secrets.VERCEL_WORKFLOW_SERVER_URL || '' }} + # TODO(sealed-log): DO NOT MERGE as-is — pinned to the sealed-log + # sequencer server preview (workflow-server PR) so this PR's e2e + # suite exercises spec-7 sequencing + noop sealing end-to-end. + # Revert to the secrets.VERCEL_WORKFLOW_SERVER_URL expression + # before landing. + VERCEL_WORKFLOW_SERVER_URL: https://workflow-server-git-pgp-sealed-log-sequencer.vercel.sh - name: Capture runtime logs on failure if: failure() @@ -769,7 +779,9 @@ jobs: WORKFLOW_VERCEL_TEAM: ${{ env.WS_TEAM_ID }} WORKFLOW_VERCEL_PROJECT: ${{ matrix.app.project-id }} WORKFLOW_VERCEL_PROJECT_SLUG: ${{ matrix.app.project-slug }} - VERCEL_WORKFLOW_SERVER_URL: ${{ github.ref != 'refs/heads/main' && secrets.VERCEL_WORKFLOW_SERVER_URL || '' }} + # TODO(sealed-log): DO NOT MERGE as-is — see the note on the + # e2e-vercel-prod job; pinned to the sealed-log server preview. + VERCEL_WORKFLOW_SERVER_URL: https://workflow-server-git-pgp-sealed-log-sequencer.vercel.sh - name: Capture runtime logs on failure if: failure() diff --git a/packages/core/src/events-consumer.test.ts b/packages/core/src/events-consumer.test.ts index 943fa2ef84..89ebe679e1 100644 --- a/packages/core/src/events-consumer.test.ts +++ b/packages/core/src/events-consumer.test.ts @@ -1167,3 +1167,96 @@ describe('EventsConsumer', () => { }); }); }); + +describe('sealed-log noop events (specVersion 7)', () => { + function logEvent(eventType: Event['eventType'], id: string): Event { + return createMockEvent({ id, eventId: id, eventType } as Partial); + } + + function consumerFor(ids: string[]) { + const seen: string[] = []; + const callback = (event: Event | null) => { + if (event && ids.includes(event.id) && !seen.includes(event.id)) { + seen.push(event.id); + return EventConsumerResult.Consumed; + } + return EventConsumerResult.NotConsumed; + }; + return { seen, callback }; + } + + it('steps over a noop without offering it to any consumer', async () => { + // The backend sealed an abandoned slot between two real events. The walk + // must pass through it as if the position never had a writer: both real + // events land, nothing is reported unconsumed, and the callback is never + // even offered the noop. + const noop = logEvent('noop' as Event['eventType'], 'noop-1'); + const before = logEvent('wait_created', 'wait-1'); + const after = logEvent('wait_completed', 'wait-2'); + const onUnconsumedEvent = vi.fn(); + const offered: (string | null)[] = []; + const consumer = new EventsConsumer([before, noop, after], { + onUnconsumedEvent, + getPromiseQueue: () => Promise.resolve(), + isDeliveryIdle: () => true, + }); + const reals = consumerFor(['wait-1', 'wait-2']); + consumer.subscribe((event) => { + offered.push(event === null ? null : event.id); + return reals.callback(event); + }); + + await vi.waitFor(() => { + expect(reals.seen).toEqual(['wait-1', 'wait-2']); + }); + expect(consumer.eventIndex).toBe(3); + expect(onUnconsumedEvent).not.toHaveBeenCalled(); + expect(offered).not.toContain('noop-1'); + }); + + it('never advances the deterministic clock off a noop', async () => { + // A noop's createdAt is the SEALER's wall clock — it can postdate every + // real event around it. Letting it reach onConsumedEvent would leak that + // timestamp into replay Date.now() and diverge from a log whose hole was + // filled by the real writer instead. + const noop = createMockEvent({ + id: 'noop-1', + eventId: 'noop-1', + eventType: 'noop', + createdAt: new Date(Date.now() + 60_000), + } as Partial); + const real = logEvent('wait_created', 'wait-1'); + const onConsumedEvent = vi.fn(); + const consumer = new EventsConsumer([noop, real], { + ...defaultOptions, + onConsumedEvent, + }); + const reals = consumerFor(['wait-1']); + consumer.subscribe(reals.callback); + + await vi.waitFor(() => { + expect(reals.seen).toEqual(['wait-1']); + }); + expect(onConsumedEvent).toHaveBeenCalledTimes(1); + expect(onConsumedEvent).toHaveBeenCalledWith(real); + }); + + it('handles a log that ends on a noop', async () => { + const real = logEvent('wait_created', 'wait-1'); + const noop = logEvent('noop' as Event['eventType'], 'noop-1'); + const onUnconsumedEvent = vi.fn(); + const consumer = new EventsConsumer([real, noop], { + onUnconsumedEvent, + getPromiseQueue: () => Promise.resolve(), + isDeliveryIdle: () => true, + }); + const reals = consumerFor(['wait-1']); + consumer.subscribe(reals.callback); + + await vi.waitFor(() => { + expect(reals.seen).toEqual(['wait-1']); + }); + expect(consumer.eventIndex).toBe(2); + expect(onUnconsumedEvent).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/core/src/events-consumer.ts b/packages/core/src/events-consumer.ts index a6686006b9..427694041f 100644 --- a/packages/core/src/events-consumer.ts +++ b/packages/core/src/events-consumer.ts @@ -331,6 +331,10 @@ export class EventsConsumer { // event's by the index it holds. this.drainParked(); const currentEvent = this.events[this.eventIndex] ?? null; + if (currentEvent !== null && currentEvent.eventType === 'noop') { + this.skipSealedNoop(currentEvent); + continue; + } const consumed = this.offer(currentEvent); if (consumed) { this.eventIndex++; @@ -540,6 +544,24 @@ export class EventsConsumer { return key === undefined ? undefined : this.seenEventClasses.get(key); } + /** + * Steps the walk over a sealed-log `noop` (specVersion >= 7): the World's + * backend wrote it to occupy a slot whose writer allocated the position and + * died, so the log's density arithmetic holds. It is invisible to the + * workflow: no consumer is offered it, no event class is recorded, and — + * exactly as with {@link skipDuplicateEvent} — the deterministic clock does + * not advance, so a log that happens to contain one produces the same + * timestamps as a log that does not. (Its `createdAt` is the seal time, + * which can even postdate later slots' events; letting it touch the clock + * would leak the sealer's wall clock into replay.) + */ + private skipSealedNoop(event: Event) { + this.eventIndex++; + eventsLogger.debug('Skipping sealed-log noop event', { + eventId: event.eventId, + }); + } + /** Steps the walk over a repeat of an already-consumed class. */ private skipDuplicateEvent(event: Event, firstType: Event['eventType']) { this.eventIndex++; diff --git a/packages/core/src/runtime/start.test.ts b/packages/core/src/runtime/start.test.ts index 28a62755ba..f810c26168 100644 --- a/packages/core/src/runtime/start.test.ts +++ b/packages/core/src/runtime/start.test.ts @@ -5,7 +5,6 @@ import { SPEC_VERSION_MAX_SUPPORTED, SPEC_VERSION_SUPPORTS_ATTRIBUTES, SPEC_VERSION_SUPPORTS_CBOR_QUEUE_TRANSPORT, - SPEC_VERSION_SUPPORTS_SLOT_IDENTITY, } from '@workflow/world'; import { afterEach, @@ -201,17 +200,19 @@ describe('start', () => { expect(mockQueue).not.toHaveBeenCalled(); }); - it('accepts a world that opts into a spec version above the default', async () => { - // `world-vercel` declares the slot-identity version so its new runs are - // created with slot event ids. An equality check against the default - // would make the runtime refuse the adapter shipped alongside it, and - // the failure surfaces only in e2e against that World. + it('accepts a world that declares the ceiling version', async () => { + // With the sealed-log bump the default and the ceiling coincide at 7, + // so "above the default" is momentarily unoccupiable — what this pins + // instead is that a World declaring the ceiling is admitted and its + // declaration is what gets stamped, not this runtime's default. (When + // the ceiling next moves ahead of the default, point the declaration + // between them again.) const validWorkflow = Object.assign(() => Promise.resolve('result'), { workflowId: 'test-workflow', }); setWorld({ - specVersion: SPEC_VERSION_SUPPORTS_SLOT_IDENTITY, + specVersion: SPEC_VERSION_MAX_SUPPORTED, getDeploymentId: vi.fn().mockResolvedValue('deploy_123'), events: { create: mockEventsCreate }, queue: mockQueue, @@ -225,7 +226,7 @@ describe('start', () => { expect.stringMatching(/^wrun_/), expect.objectContaining({ eventType: 'run_created', - specVersion: SPEC_VERSION_SUPPORTS_SLOT_IDENTITY, + specVersion: SPEC_VERSION_MAX_SUPPORTED, }), expect.anything() ); diff --git a/packages/world-vercel/src/event-retry.ts b/packages/world-vercel/src/event-retry.ts index 3228bd45b8..0310309e6f 100644 --- a/packages/world-vercel/src/event-retry.ts +++ b/packages/world-vercel/src/event-retry.ts @@ -172,6 +172,12 @@ export const EVENT_RETRY_ELIGIBILITY = { retryable: false, reason: 'server-originated; never POSTed by the SDK', }, + // Server-originated sealed-log filler (specVersion 7); the SDK never + // POSTs it — the server's read path writes it to seal an abandoned slot. + noop: { + retryable: false, + reason: 'server-originated; never POSTed by the SDK', + }, } satisfies Record; /** Up to this many retries after the initial attempt (3 attempts total). */ diff --git a/packages/world-vercel/src/events-v4.ts b/packages/world-vercel/src/events-v4.ts index 2b9f8f0fe2..4cd71ba695 100644 --- a/packages/world-vercel/src/events-v4.ts +++ b/packages/world-vercel/src/events-v4.ts @@ -361,6 +361,9 @@ const CreateEventV4BodySchemas: { hook_conflict: CreateEventV4BodySchema, wait_created: CreateEventV4BodySchema, wait_completed: CreateEventV4BodySchema, + // Never POSTed by the SDK (server-originated sealed-log filler); present + // only because the map is exhaustive over EventType. + noop: CreateEventV4BodySchema, }; const MaxEventsHeaderSchema = z.coerce.number().int().positive(); diff --git a/packages/world/src/events.test.ts b/packages/world/src/events.test.ts index 587d48535b..491c0d9e54 100644 --- a/packages/world/src/events.test.ts +++ b/packages/world/src/events.test.ts @@ -109,3 +109,39 @@ describe('run_cancelled cancelReason', () => { ).toBe('operator cancelled'); }); }); + +describe('sealed-log noop events', () => { + it('parses a noop event from the read union', () => { + // Written only by the World's backend when it seals an abandoned slot + // (specVersion >= 7); readers must accept it wherever events are parsed. + const parsed = EventSchema.parse({ + eventType: 'noop', + runId: 'wrun_123', + eventId: 'evnt_00000000000000000000000003', + createdAt: new Date().toISOString(), + specVersion: 7, + eventData: { sealed: true }, + }); + expect(parsed.eventType).toBe('noop'); + }); + + it('parses a noop with no eventData at all', () => { + const parsed = EventSchema.parse({ + eventType: 'noop', + runId: 'wrun_123', + eventId: 'evnt_00000000000000000000000003', + createdAt: new Date().toISOString(), + }); + expect(parsed.eventType).toBe('noop'); + }); + + it('is not user-creatable', () => { + // A client-minted noop would burn a slot it never allocated; only the + // backend's sealer writes them. + const result = CreateEventSchema.safeParse({ + eventType: 'noop', + eventData: { sealed: true }, + }); + expect(result.success).toBe(false); + }); +}); diff --git a/packages/world/src/events.ts b/packages/world/src/events.ts index 344462ea22..57085e034b 100644 --- a/packages/world/src/events.ts +++ b/packages/world/src/events.ts @@ -31,6 +31,10 @@ export const EventTypeSchema = z.enum([ // Wait lifecycle events 'wait_created', 'wait_completed', + // Sealed-log filler (specVersion >= 7): written ONLY by the World's backend + // to occupy a slot whose writer allocated it and died. Carries no workflow + // meaning; replay skips it (see EventsConsumer). Never user-creatable. + 'noop', ]); export type EventType = z.infer; @@ -496,6 +500,25 @@ const HookConflictEventSchema = BaseEventSchema.extend({ }), }); +/** + * Sealed-log filler event (specVersion >= 7). Written ONLY by the World's + * backend when it seals a slot whose writer allocated the position and died + * before committing (see `SPEC_VERSION_SUPPORTS_SEALED_LOG`). It occupies its + * slot — so density arithmetic and cursors count it — but carries no workflow + * meaning: replay steps over it without delivering it to any consumer and + * without advancing the deterministic clock. NOT user-creatable, and absent + * from `CreateEventSchema` for that reason. + */ +const NoopEventSchema = BaseEventSchema.extend({ + eventType: z.literal('noop'), + eventData: z + .object({ + sealed: z.boolean().optional(), + }) + .passthrough() + .optional(), +}); + const WaitCreatedEventSchema = BaseEventSchema.extend({ eventType: z.literal('wait_created'), correlationId: z.string(), @@ -690,6 +713,7 @@ const AllEventsSchema = z.discriminatedUnion('eventType', [ // Wait lifecycle events WaitCreatedEventSchema, WaitCompletedEventSchema, + NoopEventSchema, // World-only: sealed-log filler for an abandoned slot ]); // Server response includes runId, eventId, and createdAt diff --git a/packages/world/src/index.ts b/packages/world/src/index.ts index 701acb4ea3..2d54b7c7e1 100644 --- a/packages/world/src/index.ts +++ b/packages/world/src/index.ts @@ -144,6 +144,7 @@ export { SPEC_VERSION_SUPPORTS_CBOR_QUEUE_TRANSPORT, SPEC_VERSION_SUPPORTS_COMPRESSION, SPEC_VERSION_SUPPORTS_EVENT_SOURCING, + SPEC_VERSION_SUPPORTS_SEALED_LOG, SPEC_VERSION_SUPPORTS_SLOT_IDENTITY, } from './spec-version.js'; export type * from './steps.js'; diff --git a/packages/world/src/spec-version.test.ts b/packages/world/src/spec-version.test.ts index da9d9a3f79..b348714955 100644 --- a/packages/world/src/spec-version.test.ts +++ b/packages/world/src/spec-version.test.ts @@ -7,23 +7,22 @@ import { SPEC_VERSION_MAX_SUPPORTED, SPEC_VERSION_SUPPORTS_ATTRIBUTES, SPEC_VERSION_SUPPORTS_COMPRESSION, + SPEC_VERSION_SUPPORTS_SEALED_LOG, SPEC_VERSION_SUPPORTS_SLOT_IDENTITY, } from './spec-version.js'; describe('spec version constants', () => { - it('current spec version is the compression version', () => { - expect(SPEC_VERSION_CURRENT).toBe(SPEC_VERSION_SUPPORTS_COMPRESSION); - expect(SPEC_VERSION_SUPPORTS_COMPRESSION).toBe(5); + it('current spec version is the sealed-log version', () => { + expect(SPEC_VERSION_SUPPORTS_SLOT_IDENTITY).toBe(6); + expect(SPEC_VERSION_SUPPORTS_SEALED_LOG).toBe(7); + expect(SPEC_VERSION_CURRENT).toBe(SPEC_VERSION_SUPPORTS_SEALED_LOG); }); - it('the readable ceiling is the slot-identity version', () => { - // The default a World stamps and the highest version this SDK can read - // are separate dials. Slot identity is above the default on purpose: only - // a World that actually allocates slots opts into it. - expect(SPEC_VERSION_SUPPORTS_SLOT_IDENTITY).toBe(6); - expect(SPEC_VERSION_MAX_SUPPORTED).toBe( - SPEC_VERSION_SUPPORTS_SLOT_IDENTITY - ); + it('the readable ceiling moves with the version we stamp', () => { + // "What do we write?" and "what can we still read?" are separate dials, + // and the ceiling must never sit below the default: an SDK that stamps a + // version it cannot read back would reject its own runs. + expect(SPEC_VERSION_MAX_SUPPORTED).toBe(SPEC_VERSION_SUPPORTS_SEALED_LOG); expect(SPEC_VERSION_MAX_SUPPORTED).toBeGreaterThanOrEqual( SPEC_VERSION_CURRENT ); diff --git a/packages/world/src/spec-version.ts b/packages/world/src/spec-version.ts index b6dab76964..2909c6f38d 100644 --- a/packages/world/src/spec-version.ts +++ b/packages/world/src/spec-version.ts @@ -50,9 +50,23 @@ export const SPEC_VERSION_SUPPORTS_COMPRESSION = 5 as SpecVersion; */ export const SPEC_VERSION_SUPPORTS_SLOT_IDENTITY = 6 as SpecVersion; +/** + * Runs at this spec version or later live in a "sealed log": their slot + * positions are pre-assigned by a per-run sequencer on the World's backend, + * so concurrent writers never race each other for a position — and a position + * whose writer died is filled ("sealed") by the backend with a `noop` event. + * What the version gates is the READER contract that makes that safe: a + * reader at this version knows a `noop` occupies its slot and carries no + * workflow meaning, and skips it during replay without advancing the + * deterministic clock (see `EventsConsumer`). A reader below this version + * would fail to parse the unknown event type, which is exactly what + * `requiresNewerWorld` exists to catch. + */ +export const SPEC_VERSION_SUPPORTS_SEALED_LOG = 7 as SpecVersion; + /** * Current spec version: event-sourced architecture with native attributes, - * compressed payloads and slot-numbered event ids. + * compressed payloads, slot-numbered event ids, and sealed-log sequencing. * * This is both the version a World stamps on the runs it creates and the * *lowest* one this runtime accepts from a World (see @@ -72,7 +86,7 @@ export const SPEC_VERSION_SUPPORTS_SLOT_IDENTITY = 6 as SpecVersion; * run's identity scheme from what is stored rather than from this constant. */ export const SPEC_VERSION_CURRENT = - SPEC_VERSION_SUPPORTS_SLOT_IDENTITY as SpecVersion; + SPEC_VERSION_SUPPORTS_SEALED_LOG as SpecVersion; /** * The highest spec version this SDK can read. @@ -86,7 +100,7 @@ export const SPEC_VERSION_CURRENT = * impossible to express. */ export const SPEC_VERSION_MAX_SUPPORTED = - SPEC_VERSION_SUPPORTS_SLOT_IDENTITY as SpecVersion; + SPEC_VERSION_SUPPORTS_SEALED_LOG as SpecVersion; /** * Check if a spec version is legacy (<= SPEC_VERSION_LEGACY or undefined). From 2d4aaf500bd072300d00f869778d6d9e3f344402 Mon Sep 17 00:00:00 2001 From: Pranay Prakash Date: Tue, 18 Aug 2026 16:18:32 -0700 Subject: [PATCH 02/10] docs+test(sealed-log): document the noop event, pin scheduling neutrality, cover every world MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Docs: event-sourcing.mdx gains a System Events table entry and a "Sealed Positions (noop events)" section (skip semantics, clock rule, deterministic noop_ correlation id, seal-vs-writer race); building-a-world.mdx gains the spec-7 pre-assigned-positions + sealing contract for World authors (and why world-local/world-postgres never need it). - Scheduling neutrality, tested rather than asserted: the consumer skip is a synchronous `continue` inside the walk pass, so a noop consumes no extra micro- or macrotask. events-consumer.test.ts pins that a noop-riddled log finishes in the SAME single tick with the IDENTICAL offer sequence as its noop-free twin (plus all-noop and trailing-noop logs); step-delivery-ordering.test.ts injects noops into the exact microtask-hop-sensitive scenario from the production ordering incident (including the wait_completed/step_completed gap) and shows replay resolves branch order and ULID draws identically, on first replays and on payload-cache-sharing later replays. - Cross-world storage coverage: world-local and world-postgres (against the real testcontainer) store a noop at its slot, list it back in order, and keep numbering past it — spec-7 logs are legal residents of every World's storage even though only the vercel backend seals. Co-Authored-By: Claude Fable 5 --- .../docs/v5/how-it-works/event-sourcing.mdx | 20 ++ docs/content/worlds/v5/building-a-world.mdx | 6 + packages/core/src/events-consumer.test.ts | 68 +++++++ .../core/src/step-delivery-ordering.test.ts | 175 ++++++++++++++++++ .../src/storage/slot-identity.test.ts | 44 +++++ packages/world-postgres/test/spec.test.ts | 77 +++++++- 6 files changed, 389 insertions(+), 1 deletion(-) diff --git a/docs/content/docs/v5/how-it-works/event-sourcing.mdx b/docs/content/docs/v5/how-it-works/event-sourcing.mdx index 14c3bcf321..0aa5411f84 100644 --- a/docs/content/docs/v5/how-it-works/event-sourcing.mdx +++ b/docs/content/docs/v5/how-it-works/event-sourcing.mdx @@ -199,6 +199,12 @@ Events are categorized by the entity type they affect. Each event contains metad | `wait_created` | Creates a new wait in `waiting` state. Contains the timestamp when the wait should complete. | | `wait_completed` | Transitions the wait to `completed` state when the delay period has elapsed. | +### System Events + +| Event | Description | +|-------|-------------| +| `noop` | Seals an abandoned log position (specVersion 7 and above). Written only by the backend, never by workflow code — the create endpoints reject it. See [Sealed positions](#sealed-positions-noop-events). | + ## Terminal States Terminal states represent the end of an entity's lifecycle. Once an entity reaches a terminal state, no further events can transition it to another state. @@ -263,6 +269,20 @@ Both kinds of skip are logged at `debug`, so neither reaches the console unless The observability UI greys out the events it can identify this way, with the reason on hover. Its set is narrower than the runtime's: it reads the log without consumer state, and a consumer for an entity that is still open legitimately claims a repeat — each retry of a step writes another `step_started`. So it marks a repeat only once no consumer can remain for it: past a terminal event for the same entity, or a second `run_started`, of which the log records one per run. On a partial view of the log — one page of a paginated list, or search results — it marks nothing, since which copy came first is a property of the whole log. +## Sealed Positions (noop events) + +Runs at specVersion 7 and above live in a *sealed log*: the backend hands each write its position from a per-run sequencer **before** the write commits, so concurrent writers hold distinct positions and never race one another for a slot. The trade is that a writer can claim a position and then die — a crashed process, a cancelled transaction — leaving a hole that no writer will ever fill, and a hole reads exactly like an event the reader failed to load. + +The backend restores the dense log at read time by **sealing** such positions: once a hole is provably abandoned (bounded by the commit time of later positions — positions are handed out in order, so a committed later position proves how long the hole has been open), the backend writes a `noop` event into it. A `noop` occupies its position — length-based completeness checks, cursors, and pagination all count it — and means nothing: + +- It is **never offered to any consumer** during replay. The walk steps over it in the same synchronous pass that delivers the events around it, so its presence cannot perturb delivery order, promise scheduling, or which branch of a `Promise.all` resumes first. +- It **never advances the deterministic clock**. A `noop`'s `createdAt` is the *sealer's* wall clock — it can even postdate events at higher positions — and letting it feed the replay clock would make a log containing a seal replay differently from one whose hole was filled by its original writer. Same rule, and same mechanism, as skipped duplicates above. +- Its `correlationId` is `noop_` followed by the sealed position's zero-padded digits — deterministic, so any two sealers racing for the same hole mint the identical event, and recognizable at a glance in the log. + +A sealed position races its original writer at the same uniqueness fence as every other write, and losing that race is the good outcome: the real event landed first, and readers get it instead. A live writer that gets sealed over simply re-derives a fresh position and commits there — the same recovery as losing any other write race — so sealing can cost a retry, never a wrong log. + +`noop` is not user-creatable: it does not exist in the create schemas, and backends reject it on every create endpoint. Only a backend's own read path writes one. + ## Event Correlation Events use a `correlationId` to link related events together. For step, hook, and wait events, the correlation ID identifies the specific entity instance: diff --git a/docs/content/worlds/v5/building-a-world.mdx b/docs/content/worlds/v5/building-a-world.mdx index 0d0bfef497..4fde24ae98 100644 --- a/docs/content/worlds/v5/building-a-world.mdx +++ b/docs/content/worlds/v5/building-a-world.mdx @@ -144,6 +144,12 @@ Two properties have to hold, and both are about what a reader can conclude from Allocate the position **at the commit**, in the same operation that appends the event. That is what makes a reader's log a *prefix* of the run's log rather than a prefix with a hole in it: nothing can land behind a position a reader has already passed. Handing a position out earlier — in a request handler, say — and committing later breaks the property every replay depends on, and is the one case where you may need [a stale-write rejection](#optional-rejecting-a-stale-write) to compensate. +#### Optional: pre-assigned positions and `noop` sealing + +Spec version 7 legitimizes one alternative to allocate-at-commit, for Worlds whose store makes commit-time allocation a contention bottleneck: hand positions out from a per-run atomic counter **before** the commit, and restore density at read time. Pre-assignment means concurrent writers hold distinct positions and never race for one — but a writer that claims a position and dies leaves a permanent hole. A World that allocates this way MUST therefore **seal** provably abandoned positions by writing a `noop` event into them (racing the original writer at the same uniqueness fence — losing that race means the real event landed, which is success), and MUST NOT return a page with an interior hole: return the dense prefix below the hole and let the caller's next page pick up past it once the position resolves to an event or a seal. + +The runtime's side of the contract: it skips `noop` events during replay — never delivered to a consumer, never advancing the deterministic clock — so a sealed log replays identically to one whose holes were filled by their writers. `noop` is not user-creatable and is never sent to `events.create()`; only your own read path may write one. Worlds that allocate at the commit (a synchronous counter, a unique-constraint append) keep perfect density by construction and never need any of this — `world-local` and `world-postgres` never seal. + `events.create()` params carry `eventCount`: how many events the writer held in the log it replayed from, which is the position it expects to land on minus one. Attempt `eventCount + 1`. When that position is taken, **do not reject the write**. Advance to the next free position, commit there, and return the events occupying the positions you skipped over on the success response, in `events` with a matching `cursor` and `hasMore`. The writer merges them into its own log and replays once, rather than paying a second round trip to discover it was behind. A caller with a stale count is the normal case for a fan-out, and rejecting it would serialize writes the runtime issues in parallel. ### Optional: Rejecting a Stale Write diff --git a/packages/core/src/events-consumer.test.ts b/packages/core/src/events-consumer.test.ts index 89ebe679e1..1ad7b339f8 100644 --- a/packages/core/src/events-consumer.test.ts +++ b/packages/core/src/events-consumer.test.ts @@ -1241,6 +1241,74 @@ describe('sealed-log noop events (specVersion 7)', () => { expect(onConsumedEvent).toHaveBeenCalledWith(real); }); + it('is scheduling-neutral: same offers, same tick, as the log without noops', async () => { + // The skip is a synchronous `continue` inside the walk pass — it consumes + // no extra micro- or macrotask. This pins that: a log with noops + // interleaved at the head, middle, and tail is fully consumed after the + // SAME single tick as its noop-free twin, and the sequence of events + // offered to consumers is byte-for-byte identical. Deterministic + // scheduling is what keeps replay ULID draws (and therefore correlation + // ids) stable across branches racing in Promise.all. + async function offersAfterOneTick(events: Event[]) { + const offered: (string | null)[] = []; + const consumer = new EventsConsumer(events, { + onUnconsumedEvent: vi.fn(), + getPromiseQueue: () => Promise.resolve(), + isDeliveryIdle: () => true, + }); + consumer.subscribe((event) => { + offered.push(event === null ? null : event.id); + return event === null + ? EventConsumerResult.NotConsumed + : EventConsumerResult.Consumed; + }); + // subscribe() schedules exactly one nextTick; the walk drains + // synchronously inside it. One tick must therefore finish either log. + await waitForNextTick(); + return { offered, index: consumer.eventIndex, total: events.length }; + } + + const clean = await offersAfterOneTick([ + logEvent('wait_created', 'w1'), + logEvent('wait_completed', 'w2'), + ]); + const sealed = await offersAfterOneTick([ + logEvent('noop' as Event['eventType'], 'n0'), + logEvent('wait_created', 'w1'), + logEvent('noop' as Event['eventType'], 'n1'), + logEvent('noop' as Event['eventType'], 'n2'), + logEvent('wait_completed', 'w2'), + logEvent('noop' as Event['eventType'], 'n3'), + ]); + + expect(clean.index).toBe(clean.total); + expect(sealed.index).toBe(sealed.total); + // Identical offer sequences — the noops were never offered at all, and + // both logs finished inside the same single tick. + expect(sealed.offered).toEqual(clean.offered); + }); + + it('consumes an all-noop log to the end without divergence', async () => { + const onUnconsumedEvent = vi.fn(); + const consumer = new EventsConsumer( + [ + logEvent('noop' as Event['eventType'], 'n1'), + logEvent('noop' as Event['eventType'], 'n2'), + ], + { + onUnconsumedEvent, + getPromiseQueue: () => Promise.resolve(), + isDeliveryIdle: () => true, + } + ); + consumer.subscribe(() => EventConsumerResult.NotConsumed); + + await vi.waitFor(() => { + expect(consumer.eventIndex).toBe(2); + }); + expect(onUnconsumedEvent).not.toHaveBeenCalled(); + }); + it('handles a log that ends on a noop', async () => { const real = logEvent('wait_created', 'wait-1'); const noop = logEvent('noop' as Event['eventType'], 'noop-1'); diff --git a/packages/core/src/step-delivery-ordering.test.ts b/packages/core/src/step-delivery-ordering.test.ts index 300cb6dba4..93a21be71e 100644 --- a/packages/core/src/step-delivery-ordering.test.ts +++ b/packages/core/src/step-delivery-ordering.test.ts @@ -800,3 +800,178 @@ describe('step result delivery ordering across replays', () => { }); }); }); + +/** + * Sealed-log noops (specVersion 7) injected into the exact scenario above — + * the one this file exists for, where delivery ORDER between adjacent events + * decides which branch draws which correlation id. If skipping a noop cost an + * extra microtask hop, or shifted the walk relative to the promise queue, it + * would surface here as the same divergence the production incident produced. + * The noops are deliberately placed in the hop-count-sensitive gap (between + * `wait_completed` and `step_completed`) as well as at the head and tail. + */ +describe('sealed-log noop events in a scheduling-sensitive replay', () => { + const resumeAt = new Date('2026-07-27T12:00:05.000Z'); + + function noopAt(id: string): Event { + return { + eventId: id, + runId: 'wrun_test', + eventType: 'noop', + eventData: { sealed: true }, + // Deliberately far in the future: a noop's createdAt is the sealer's + // wall clock. It must not leak into the replay clock (asserted by the + // run completing identically; the clock rule itself is pinned in + // events-consumer.test.ts). + createdAt: new Date('2030-01-01T00:00:00.000Z'), + } as unknown as Event; + } + + async function buildEventLog(): Promise { + const ops: Promise[] = []; + const stepAResult = await dehydrateStepReturnValue( + 'ok', + 'wrun_test', + undefined, + ops + ); + + return [ + noopAt('evnt_n0'), + { + eventId: 'evnt_0', + runId: 'wrun_test', + eventType: 'step_created', + correlationId: `step_${CORR_IDS[0]}`, + eventData: { stepName: 'stepA' }, + createdAt: new Date(), + }, + { + eventId: 'evnt_1', + runId: 'wrun_test', + eventType: 'wait_created', + correlationId: `wait_${CORR_IDS[1]}`, + eventData: { resumeAt }, + createdAt: new Date(), + }, + { + eventId: 'evnt_2', + runId: 'wrun_test', + eventType: 'step_started', + correlationId: `step_${CORR_IDS[0]}`, + eventData: { stepName: 'stepA' }, + createdAt: new Date(), + }, + { + eventId: 'evnt_3', + runId: 'wrun_test', + eventType: 'wait_completed', + correlationId: `wait_${CORR_IDS[1]}`, + eventData: { resumeAt }, + createdAt: new Date(), + }, + // The sensitive gap: the wait branch's resume and the step branch's + // resume race on microtask hops from exactly this adjacency. + noopAt('evnt_n1'), + noopAt('evnt_n2'), + { + eventId: 'evnt_4', + runId: 'wrun_test', + eventType: 'step_completed', + correlationId: `step_${CORR_IDS[0]}`, + eventData: { stepName: 'stepA', result: stepAResult }, + createdAt: new Date(), + }, + { + eventId: 'evnt_5', + runId: 'wrun_test', + eventType: 'step_created', + correlationId: `step_${CORR_IDS[2]}`, + eventData: { stepName: 'afterSleep' }, + createdAt: new Date(), + }, + noopAt('evnt_n3'), + { + eventId: 'evnt_6', + runId: 'wrun_test', + eventType: 'step_created', + correlationId: `step_${CORR_IDS[3]}`, + eventData: { stepName: 'afterStep' }, + createdAt: new Date(), + }, + noopAt('evnt_n4'), + ]; + } + + function workflowBody(ctx: WorkflowOrchestratorContext) { + const useStep = createUseStep(ctx); + const sleep = createSleep(ctx); + + return async () => { + const stepA = useStep('stepA'); + const afterStep = useStep('afterStep'); + const afterSleep = useStep('afterSleep'); + + const branchStep = (async () => { + await stepA(); + await afterStep(); + })(); + const branchSleep = (async () => { + await sleep(resumeAt); + await afterSleep(); + })(); + + await Promise.all([branchStep, branchSleep]); + }; + } + + it('replays the noop-bearing log with the ordering the noop-free log encodes', async () => { + const hydration = delayHydration(); + const spy = await hydration.install(); + try { + const events = await buildEventLog(); + const ctx = setupWorkflowContext(events); + const { error } = await runWithDiscontinuation(ctx, workflowBody(ctx)); + + expect(error).toBeDefined(); + if (!WorkflowSuspension.is(error)) { + throw error; + } + // Identical outcome to the noop-free scenario above: the sleep branch + // resumed first and drew CORR_IDS[2], both follow-up steps are pending, + // and every event — noops included — was walked to the end. + expect(pendingStepNames(ctx).sort()).toEqual(['afterSleep', 'afterStep']); + expect(ctx.eventsConsumer.eventIndex).toBe(events.length); + } finally { + spy.mockRestore(); + } + }); + + it('keeps that ordering on a later replay sharing the payload cache', async () => { + const hydration = delayHydration(); + const spy = await hydration.install(); + try { + const cache = new ReplayPayloadCache(undefined); + + const first = setupWorkflowContext(await buildEventLog(), cache); + const firstRun = await runWithDiscontinuation(first, workflowBody(first)); + expect(WorkflowSuspension.is(firstRun.error)).toBe(true); + + // The second replay resolves the memoized primitive step result in + // fewer hops — the exact asymmetry the incident exploited. The noops + // must not tip it. + const second = setupWorkflowContext(await buildEventLog(), cache); + const secondRun = await runWithDiscontinuation( + second, + workflowBody(second) + ); + expect(WorkflowSuspension.is(secondRun.error)).toBe(true); + expect(pendingStepNames(second).sort()).toEqual([ + 'afterSleep', + 'afterStep', + ]); + } finally { + spy.mockRestore(); + } + }); +}); diff --git a/packages/world-local/src/storage/slot-identity.test.ts b/packages/world-local/src/storage/slot-identity.test.ts index 7ef644e724..38d8afc099 100644 --- a/packages/world-local/src/storage/slot-identity.test.ts +++ b/packages/world-local/src/storage/slot-identity.test.ts @@ -467,3 +467,47 @@ describe('skipped-slot report', () => { } }); }); + +describe('sealed-log noop events', () => { + // world-local allocates positions synchronously from its own counter, so it + // never needs to seal a hole itself — but spec 7 makes `noop` a legal + // resident of any slot log, and the storage layer must round-trip one: + // store it at its slot, list it back in order, and keep numbering past it. + // (Creating one through `events.create` stands in for a backend sealer; the + // public CreateEventSchema excludes `noop`, which is asserted in + // @workflow/world's own tests.) + it('stores, lists, and numbers past a noop event', async () => { + const runId = await startRun(); + + await storage.events.create(runId, { + eventType: 'noop', + specVersion: SPEC_VERSION_CURRENT, + eventData: { sealed: true }, + } as any); + await storage.events.create(runId, { + eventType: 'step_created', + correlationId: 'step_after_noop', + specVersion: SPEC_VERSION_CURRENT, + eventData: { stepName: 'afterNoop', input: serialized([]) }, + } as any); + + const result = await storage.events.list({ + runId, + pagination: { limit: 100 }, + }); + const types = result.data.map((event) => event.eventType); + expect(types).toEqual([ + 'run_created', + 'run_started', + 'noop', + 'step_created', + ]); + // The noop occupies a real position: slots stay dense through it. + expect(result.data.map((event) => event.eventId)).toEqual([ + slotId(FIRST_EVENT_SLOT), + slotId(FIRST_EVENT_SLOT + 1), + slotId(FIRST_EVENT_SLOT + 2), + slotId(FIRST_EVENT_SLOT + 3), + ]); + }); +}); diff --git a/packages/world-postgres/test/spec.test.ts b/packages/world-postgres/test/spec.test.ts index 1be4cb2636..39eeeb08e7 100644 --- a/packages/world-postgres/test/spec.test.ts +++ b/packages/world-postgres/test/spec.test.ts @@ -1,7 +1,12 @@ import { execSync } from 'node:child_process'; import { PostgreSqlContainer } from '@testcontainers/postgresql'; +import { + eventIdToSlot, + FIRST_EVENT_SLOT, + SPEC_VERSION_CURRENT, +} from '@workflow/world'; import { createTestSuite } from '@workflow/world-testing'; -import { afterAll, beforeAll, test } from 'vitest'; +import { afterAll, beforeAll, expect, test } from 'vitest'; // Skip these tests on Windows since it relies on a docker container if (process.platform === 'win32') { @@ -29,5 +34,75 @@ if (process.platform === 'win32') { }); test('smoke', () => {}); + + // Sealed-log noop tolerance (specVersion 7): world-postgres allocates + // positions from its own counter and never seals holes itself, but a + // `noop` is a legal resident of any spec-7 slot log, and the storage layer + // must round-trip one — store it at its slot, list it back in order, and + // keep numbering past it. Direct storage access (not the conformance + // server): only a backend sealer would ever write one, and the public + // CreateEventSchema excludes it. + test('stores, lists, and numbers past a noop event', async () => { + // Storage layer only — createWorld would also spin up the queue and the + // streamer's dedicated LISTEN client, which have no shutdown hook here + // and would die noisily when afterAll stops the container. + const { createClient } = await import('../dist/drizzle/index.js'); + const { createEventsStorage } = await import('../dist/storage.js'); + const { Pool } = await import('pg'); + const pool = new Pool({ + connectionString: process.env.WORKFLOW_POSTGRES_URL, + max: 2, + }); + const world = { events: createEventsStorage(createClient(pool)) }; + + const serialized = (value: unknown) => + ({ data: JSON.stringify(value), encoding: 'json' }) as any; + const created = await world.events.create('', { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'dpl_noop', + workflowName: 'noopWorkflow', + input: serialized([]), + }, + } as any); + const runId = created.event!.runId; + await world.events.create(runId, { + eventType: 'run_started', + specVersion: SPEC_VERSION_CURRENT, + } as any); + await world.events.create(runId, { + eventType: 'noop', + specVersion: SPEC_VERSION_CURRENT, + eventData: { sealed: true }, + } as any); + await world.events.create(runId, { + eventType: 'step_created', + correlationId: 'step_after_noop', + specVersion: SPEC_VERSION_CURRENT, + eventData: { stepName: 'afterNoop', input: serialized([]) }, + } as any); + + const result = await world.events.list({ + runId, + pagination: { limit: 100 }, + }); + expect(result.data.map((event: any) => event.eventType)).toEqual([ + 'run_created', + 'run_started', + 'noop', + 'step_created', + ]); + expect( + result.data.map((event: any) => eventIdToSlot(event.eventId)) + ).toEqual([ + FIRST_EVENT_SLOT, + FIRST_EVENT_SLOT + 1, + FIRST_EVENT_SLOT + 2, + FIRST_EVENT_SLOT + 3, + ]); + await pool.end(); + }, 60_000); + createTestSuite('./dist/index.js'); } From 71cd9b9e8221f35979c69abdf507950c239c4ef0 Mon Sep 17 00:00:00 2001 From: Pranay Prakash Date: Wed, 19 Aug 2026 16:34:05 -0700 Subject: [PATCH 03/10] feat(web-shared): render sealed-position noops like the log rows they are The o11y UI now understands spec-7 noop seals, mirroring how it already treats replay duplicates: real rows of the log, shown greyed in every event list with the reason on hover, and excluded from everything that charts the run's behavior. - trace-builder keeps noops out of span grouping AND the latest-known-time bound: a noop's createdAt is the SEALER's wall clock, which can postdate every real event around it, and letting it in would stretch spans and the trace's known duration to the sealer's schedule. - Event lists (main table + sidebar) grey noop rows and explain them via the new EventNoticeTooltip, the generalized duplicate tooltip. - noop groups and labels with the run itself, like attr_set: its correlationId (noop_) is positional, not a child entity. - Neutral gray palette + status dot, one step quieter than pending. Verified against a seeded world-local run in the real web UI: dimmed rows with tooltips in the Events tab, dense event ids straight through the seals, and a trace waterfall whose geometry ignores them. Co-Authored-By: Claude Fable 5 --- .../src/components/event-list-view.tsx | 32 +++++++++++---- .../src/components/sidebar/events-list.tsx | 18 ++++++-- .../components/ui/duplicate-event-tooltip.tsx | 41 ++++++++++++++----- .../workflow-traces/event-colors.ts | 11 +++++ packages/web-shared/src/index.ts | 4 ++ .../web-shared/src/lib/trace-builder.test.ts | 30 +++++++++++++- packages/web-shared/src/lib/trace-builder.ts | 15 ++++--- 7 files changed, 123 insertions(+), 28 deletions(-) diff --git a/packages/web-shared/src/components/event-list-view.tsx b/packages/web-shared/src/components/event-list-view.tsx index 19d57c418d..8bd874382d 100644 --- a/packages/web-shared/src/components/event-list-view.tsx +++ b/packages/web-shared/src/components/event-list-view.tsx @@ -10,7 +10,10 @@ import type { } from 'react'; import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { Virtuoso, type VirtuosoHandle } from 'react-virtuoso'; -import { findDuplicateEventIds } from '../lib/duplicate-events'; +import { + DUPLICATE_EVENT_MESSAGE, + findDuplicateEventIds, +} from '../lib/duplicate-events'; import { type ExactIdSearchResult, type ExactWorkflowSearchIdKind, @@ -18,13 +21,14 @@ import { parseExactWorkflowSearchId, } from '../lib/exact-event-search-id'; import { isEncryptedMarker } from '../lib/hydration'; +import { isSealedNoopEvent, SEALED_EVENT_MESSAGE } from '../lib/sealed-events'; import { useToast } from '../lib/toast'; import { formatDuration } from '../lib/utils'; import { AttrSetEventBlock } from './sidebar/attributes-block'; import { ContextCardProvider } from './ui/context-card'; import { DataInspector, DecryptClickContext } from './ui/data-inspector'; import { DecryptButton } from './ui/decrypt-button'; -import { DuplicateEventTooltip } from './ui/duplicate-event-tooltip'; +import { EventNoticeTooltip } from './ui/duplicate-event-tooltip'; import { ErrorStackBlock, isStructuredError, @@ -142,6 +146,11 @@ function getStatusDotColor(eventType: string): string { ) { return 'var(--ds-blue-700)'; } + // Sealed positions → dim gray, one step quieter than pending: the row is + // log filler the run never observed. + if (eventType === 'noop') { + return 'var(--ds-gray-500)'; + } // Created/pending → gray return 'var(--ds-gray-600)'; } @@ -297,9 +306,10 @@ function isRunLevel(eventType: string): boolean { eventType === 'workflow_started' || eventType === 'workflow_completed' || eventType === 'workflow_failed' || - // attr_set carries a dedup correlationId rather than a child entity ID, - // so it groups and labels with the run itself. - eventType === 'attr_set' + // attr_set and noop carry a dedup/positional correlationId rather than a + // child entity ID, so they group and label with the run itself. + eventType === 'attr_set' || + eventType === 'noop' ); } @@ -898,6 +908,12 @@ export function EventRow({ ? '__run__' : (event.correlationId ?? undefined); + const isSealed = isSealedNoopEvent(event); + const rowNotice = isDuplicate + ? DUPLICATE_EVENT_MESSAGE + : isSealed + ? SEALED_EVENT_MESSAGE + : undefined; const statusDotColor = getStatusDotColor(event.eventType); const createdAt = new Date(event.createdAt); const occurredAt = parseEventDate(event.occurredAt); @@ -1111,11 +1127,11 @@ export function EventRow({ {/* Event Type */}
- + {formatEventType(event.eventType)} - +
{/* Name */} diff --git a/packages/web-shared/src/components/sidebar/events-list.tsx b/packages/web-shared/src/components/sidebar/events-list.tsx index 7fd65bc10d..c6254b2493 100644 --- a/packages/web-shared/src/components/sidebar/events-list.tsx +++ b/packages/web-shared/src/components/sidebar/events-list.tsx @@ -2,7 +2,12 @@ import { type Event, getEventDataRefFields } from '@workflow/world'; import { useCallback, useLayoutEffect, useMemo, useRef, useState } from 'react'; +import { DUPLICATE_EVENT_MESSAGE } from '../../lib/duplicate-events'; import { hasEncryptedFields, isExpiredMarker } from '../../lib/hydration'; +import { + isSealedNoopEvent, + SEALED_EVENT_MESSAGE, +} from '../../lib/sealed-events'; import { Collapsible, CollapsibleContent, @@ -10,7 +15,7 @@ import { CollapsibleTrigger, } from '../ui/collapsible'; import { RunClickContext, StreamClickContext } from '../ui/data-inspector'; -import { DuplicateEventTooltip } from '../ui/duplicate-event-tooltip'; +import { EventNoticeTooltip } from '../ui/duplicate-event-tooltip'; import { ErrorCard } from '../ui/error-card'; import { ErrorStackBlock, isStructuredError } from '../ui/error-stack-block'; import { Skeleton } from '../ui/skeleton'; @@ -110,6 +115,11 @@ function EventItem({ void loadEventData(true); }, [encryptionKey, loadEventData]); + const rowNotice = isDuplicate + ? DUPLICATE_EVENT_MESSAGE + : isSealedNoopEvent(event) + ? SEALED_EVENT_MESSAGE + : undefined; const createdAt = new Date(event.createdAt); const occurredAt = parseDateValue(event.occurredAt); const displayedCreatedAt = showSeparateEventOccurrenceTimestamps @@ -139,15 +149,15 @@ function EventItem({ >
- + {event.eventType} - + {displayedCreatedAtTime} diff --git a/packages/web-shared/src/components/ui/duplicate-event-tooltip.tsx b/packages/web-shared/src/components/ui/duplicate-event-tooltip.tsx index 920576ae44..1487167b09 100644 --- a/packages/web-shared/src/components/ui/duplicate-event-tooltip.tsx +++ b/packages/web-shared/src/components/ui/duplicate-event-tooltip.tsx @@ -10,22 +10,23 @@ import { } from './tooltip'; /** - * Explains why an event is shown greyed out: it repeats a class the log - * already records for the same entity, after that entity finished. + * Explains why an event row is shown greyed out — a repeat the runtime read + * past ({@link DUPLICATE_EVENT_MESSAGE}), a backend seal for an abandoned + * position (`SEALED_EVENT_MESSAGE`), or any other notice a list attaches. * - * Renders `children` untouched when `isDuplicate` is false, so a call site can + * Renders `children` untouched when `notice` is absent, so a call site can * wrap an event label unconditionally. Mounts its own {@link TooltipProvider} - * so it works in the sidebar and the events table alike; nesting one inside an - * existing provider is harmless. + * so it works in the sidebar and the events table alike; nesting one inside + * an existing provider is harmless. */ -export function DuplicateEventTooltip({ - isDuplicate = false, +export function EventNoticeTooltip({ + notice, children, }: { - isDuplicate?: boolean; + notice?: string; children: ReactNode; }): ReactNode { - if (!isDuplicate) return children; + if (!notice) return children; return ( @@ -36,9 +37,29 @@ export function DuplicateEventTooltip({ collisionPadding={8} side="top" > - {DUPLICATE_EVENT_MESSAGE} + {notice} ); } + +/** + * The duplicate-specific wrapper kept for existing call sites: greys the + * event out as a repeat the runtime read past. + */ +export function DuplicateEventTooltip({ + isDuplicate = false, + children, +}: { + isDuplicate?: boolean; + children: ReactNode; +}): ReactNode { + return ( + + {children} + + ); +} diff --git a/packages/web-shared/src/components/workflow-traces/event-colors.ts b/packages/web-shared/src/components/workflow-traces/event-colors.ts index 3dd5243e3c..e8b02b871d 100644 --- a/packages/web-shared/src/components/workflow-traces/event-colors.ts +++ b/packages/web-shared/src/components/workflow-traces/event-colors.ts @@ -72,6 +72,17 @@ export function getEventColor( }; } + // Sealed positions - neutral gray: backend log filler, not run activity + if (eventType === 'noop') { + return { + color: 'var(--ds-gray-500)', + background: 'var(--ds-gray-100)', + border: 'var(--ds-gray-400)', + text: 'var(--ds-gray-900)', + secondary: 'var(--ds-gray-700)', + }; + } + // Default - Blue return { color: 'var(--ds-blue-600)', diff --git a/packages/web-shared/src/index.ts b/packages/web-shared/src/index.ts index cbfd776f5b..65d71ba4e7 100644 --- a/packages/web-shared/src/index.ts +++ b/packages/web-shared/src/index.ts @@ -59,6 +59,10 @@ export { STREAM_REF_TYPE, truncateId, } from './lib/hydration'; +export { + isSealedNoopEvent, + SEALED_EVENT_MESSAGE, +} from './lib/sealed-events'; export type { DecodedStreamChunkSource } from './lib/stream-display'; export type { ToastAdapter } from './lib/toast'; export { ToastProvider, useToast } from './lib/toast'; diff --git a/packages/web-shared/src/lib/trace-builder.test.ts b/packages/web-shared/src/lib/trace-builder.test.ts index 70ad41339a..8de5282d24 100644 --- a/packages/web-shared/src/lib/trace-builder.test.ts +++ b/packages/web-shared/src/lib/trace-builder.test.ts @@ -1,7 +1,7 @@ import type { Event, EventType, WorkflowRun } from '@workflow/world'; import { describe, expect, it } from 'vitest'; import { otelTimeToMs } from '../components/workflow-traces/trace-time-utils'; -import { buildTrace } from './trace-builder'; +import { buildTrace, filterSpanRawEvents } from './trace-builder'; const BASE_TIME = Date.parse('2026-01-01T00:00:00.000Z'); @@ -71,4 +71,32 @@ describe('buildTrace', () => { expect(otelTimeToMs(stepSpan?.endTime ?? [0, 0])).toBe(BASE_TIME + 20_000); expect(trace.duplicateEventIds.size).toBe(0); }); + + it('keeps sealed-position noops out of the geometry and its time bounds', () => { + const events = [ + event('run_created', { at: 0 }), + event('run_started', { at: 0 }), + event('step_created', { correlationId: 'step_a', at: 1 }), + event('step_started', { correlationId: 'step_a', at: 1 }), + event('step_completed', { correlationId: 'step_a', at: 4 }), + // A hole sealed by a reader long after the run went quiet. Its + // createdAt is the SEALER's clock — letting it into the geometry would + // stretch the trace to the sealer's schedule. + event('noop', { correlationId: 'noop_5', at: 300 }), + ]; + + const trace = buildTrace(run, events, new Date(BASE_TIME + 400_000), { + isCompleteHistory: true, + }); + + // No span for the noop's correlationId, and the trace's known duration + // ends at the last real event, not at the seal. + expect(trace.spans.some((span) => span.spanId === 'noop_5')).toBe(false); + expect(trace.knownDurationMs).toBe(4000); + + // The run span's raw event list still shows the seal (greyed in the UI), + // exactly like duplicate rows: real log rows, marked with the reason. + const runRaw = filterSpanRawEvents(events, 'run', 'run_1'); + expect(runRaw.some((e) => e.eventType === 'noop')).toBe(true); + }); }); diff --git a/packages/web-shared/src/lib/trace-builder.ts b/packages/web-shared/src/lib/trace-builder.ts index b8d1186efd..ec671ebc66 100644 --- a/packages/web-shared/src/lib/trace-builder.ts +++ b/packages/web-shared/src/lib/trace-builder.ts @@ -23,6 +23,7 @@ import { } from '../components/workflow-traces/trace-span-construction'; import { otelTimeToMs } from '../components/workflow-traces/trace-time-utils'; import { findDuplicateEventIds } from './duplicate-events'; +import { isSealedNoopEvent } from './sealed-events'; import type { Span } from './trace-types'; /** @@ -212,14 +213,18 @@ export function buildTrace( // Span geometry comes from what the run acted on. A repeat of a class the // log already records is read past by every replay, and letting one through // here would stretch a span to whenever a concurrent replay committed it. - // The event lists still show them, marked as repeats. + // Sealed-position noops are excluded for the same reason with a different + // clock: a noop's createdAt is the sealer's wall time, which can postdate + // every real event around it, so feeding it into span grouping or the + // latest-known-time bound would chart the sealer's schedule instead of the + // run's. The event lists still show both, marked with the reason. const duplicateEventIds = findDuplicateEventIds(events, { isCompleteHistory, }); - const actedOnEvents = - duplicateEventIds.size === 0 - ? events - : events.filter((event) => !duplicateEventIds.has(event.eventId)); + const actedOnEvents = events.filter( + (event) => + !duplicateEventIds.has(event.eventId) && !isSealedNoopEvent(event) + ); const groupedEvents = groupEventsByCorrelation(actedOnEvents); const latestKnownTime = computeLatestKnownTime(actedOnEvents, run); From 8d6b85d33f4a21a7899e9d784a37461d1544f257 Mon Sep 17 00:00:00 2001 From: Pranay Prakash Date: Wed, 19 Aug 2026 16:45:33 -0700 Subject: [PATCH 04/10] fix(web-shared): add the sealed-events module the noop-rendering commit imports MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 214dd090d referenced ../lib/sealed-events but the new file itself was never staged. Lands with the lightened tooltip copy: "No-op events may be added by Workflow SDK to ensure correctness" — the mechanism belongs in the event-sourcing docs, not a hover. Co-Authored-By: Claude Fable 5 --- packages/web-shared/src/lib/sealed-events.ts | 27 ++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 packages/web-shared/src/lib/sealed-events.ts diff --git a/packages/web-shared/src/lib/sealed-events.ts b/packages/web-shared/src/lib/sealed-events.ts new file mode 100644 index 0000000000..0eb777f455 --- /dev/null +++ b/packages/web-shared/src/lib/sealed-events.ts @@ -0,0 +1,27 @@ +import type { Event } from '@workflow/world'; + +/** + * Sealed-position `noop` events (specVersion 7). + * + * A sealed-log backend hands each write its position before the write + * commits, so a writer that dies after claiming a position leaves a hole. + * The backend closes a provably abandoned hole by writing a `noop` event + * into it — a log-only row the run itself never observes: replay steps over + * it without offering it to any consumer and without advancing the + * deterministic clock. + * + * The observability UI mirrors that treatment. A `noop` appears in event + * lists (greyed, with {@link SEALED_EVENT_MESSAGE} on hover) because it is a + * real row of the log, but it is excluded from span geometry and from + * trace-duration bounds: its `createdAt` is the *sealer's* wall clock, which + * can postdate every real event around it, and letting it stretch a span or + * the trace's known duration would chart the sealer's schedule rather than + * the run's. + */ +export const SEALED_EVENT_MESSAGE = + 'No-op events may be added by Workflow SDK to ensure correctness'; + +/** Whether `event` is a backend-written seal for an abandoned position. */ +export function isSealedNoopEvent(event: Pick): boolean { + return event.eventType === 'noop'; +} From d5b3a596e31ee6e778b9f4b3da6bed0c11c284cc Mon Sep 17 00:00:00 2001 From: Pranay Prakash Date: Wed, 19 Aug 2026 17:12:42 -0700 Subject: [PATCH 05/10] ci: unpin e2e from the sealed-log server preview MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Restores the three VERCEL_WORKFLOW_SERVER_URL expressions to main's secrets-based form, byte-identical to 37e1d9e5a. The pin existed so this PR's e2e exercised spec-7 sequencing + noop sealing against the workflow-server PR's preview; shipping the client against today's production server is safe — it treats spec-7 runs as slot identity (usesSlotIdentity is >= 6) and never writes a noop, so the client's new reading capability simply lies dormant until the server lands. Co-Authored-By: Claude Fable 5 --- .github/workflows/tests.yml | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f55c29edae..a9e702af56 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -481,12 +481,7 @@ jobs: # unset for changeset-release PRs: they test a production deployment # that is wired to the production workflow-server, and the harness # has to read run state from the same server the app writes it to. - # TODO(sealed-log): DO NOT MERGE as-is — pinned to the sealed-log - # sequencer server preview (workflow-server PR) so this PR's e2e - # suite exercises spec-7 sequencing + noop sealing end-to-end. - # Revert to the secrets.VERCEL_WORKFLOW_SERVER_URL expression - # before landing. - VERCEL_WORKFLOW_SERVER_URL: https://workflow-server-git-pgp-sealed-log-sequencer.vercel.sh + VERCEL_WORKFLOW_SERVER_URL: ${{ github.ref != 'refs/heads/main' && !startsWith(github.head_ref, 'changeset-release/') && secrets.VERCEL_WORKFLOW_SERVER_URL || '' }} - name: Capture runtime logs on failure if: failure() @@ -600,12 +595,7 @@ jobs: WORKFLOW_VERCEL_PROJECT_SLUG: "example-nextjs-workflow-turbopack" # See the note on e2e-vercel-prod: PRs point at the protected # workflow-server preview; unset on main for production. - # TODO(sealed-log): DO NOT MERGE as-is — pinned to the sealed-log - # sequencer server preview (workflow-server PR) so this PR's e2e - # suite exercises spec-7 sequencing + noop sealing end-to-end. - # Revert to the secrets.VERCEL_WORKFLOW_SERVER_URL expression - # before landing. - VERCEL_WORKFLOW_SERVER_URL: https://workflow-server-git-pgp-sealed-log-sequencer.vercel.sh + VERCEL_WORKFLOW_SERVER_URL: ${{ github.ref != 'refs/heads/main' && !startsWith(github.head_ref, 'changeset-release/') && secrets.VERCEL_WORKFLOW_SERVER_URL || '' }} - name: Capture runtime logs on failure if: failure() @@ -779,9 +769,7 @@ jobs: WORKFLOW_VERCEL_TEAM: ${{ env.WS_TEAM_ID }} WORKFLOW_VERCEL_PROJECT: ${{ matrix.app.project-id }} WORKFLOW_VERCEL_PROJECT_SLUG: ${{ matrix.app.project-slug }} - # TODO(sealed-log): DO NOT MERGE as-is — see the note on the - # e2e-vercel-prod job; pinned to the sealed-log server preview. - VERCEL_WORKFLOW_SERVER_URL: https://workflow-server-git-pgp-sealed-log-sequencer.vercel.sh + VERCEL_WORKFLOW_SERVER_URL: ${{ github.ref != 'refs/heads/main' && secrets.VERCEL_WORKFLOW_SERVER_URL || '' }} - name: Capture runtime logs on failure if: failure() From 779c1ee949ac2a2f2beef0919fcaf3a5dad27b93 Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Thu, 20 Aug 2026 12:24:43 -0700 Subject: [PATCH 06/10] fix(core): stop a sealed noop from moving the QuickJS replay clock A `noop` occupies a position whose writer allocated it and died; the run never observed it, and its `createdAt` is the SEALER's wall clock, which can postdate every real event around it. The node:vm engine skips it in `EventsConsumer` before `onConsumedEvent` feeds the clock. The QuickJS engine's own event loop did not: `processEvents` advanced the clock from every row before it looked at the event type, so a sealed log replayed with a different `Date.now()` than the same log whose hole its own writer filled -- and than itself on the other engine. Monotonic clock, so every later read in the run inherited the skew. - `isSealedNoopEvent` moves to `@workflow/world`, the one place both engines and the trace viewer now test through. Three independent copies of "is this a seal" is what let the engines drift in the first place. - Sealed positions stop counting toward the event ceiling: the limit exists to stop a runaway workflow, and charging the user's budget for the backend's bookkeeping made the ceiling a run hits depend on how much write contention it happened to see. - A seal no longer disqualifies a run from TTFS telemetry, which had been dropping exactly the contended runs worth measuring. Regression test sits with the existing deterministic-replay-clock suite, with the hole between wait_created and wait_completed -- where a fanout actually leaves one. Co-Authored-By: Claude Opus 5 (1M context) --- .changeset/sealed-log-noop-clock-quickjs.md | 7 ++ packages/core/src/events-consumer.ts | 9 ++- packages/core/src/runtime.ts | 34 ++++++--- .../core/src/runtime/quickjs-runtime.test.ts | 69 +++++++++++++++++++ packages/core/src/runtime/quickjs-runtime.ts | 23 +++++-- packages/web-shared/src/lib/sealed-events.ts | 13 ++-- packages/world/src/events.ts | 17 +++++ packages/world/src/index.ts | 1 + packages/world/src/spec-version.ts | 7 ++ 9 files changed, 161 insertions(+), 19 deletions(-) create mode 100644 .changeset/sealed-log-noop-clock-quickjs.md diff --git a/.changeset/sealed-log-noop-clock-quickjs.md b/.changeset/sealed-log-noop-clock-quickjs.md new file mode 100644 index 0000000000..bbe33cde90 --- /dev/null +++ b/.changeset/sealed-log-noop-clock-quickjs.md @@ -0,0 +1,7 @@ +--- +'@workflow/web-shared': patch +'@workflow/world': patch +'@workflow/core': patch +--- + +Fix the QuickJS replay engine advancing the deterministic clock to a sealed position's timestamp. A `noop` is written by the backend when it seals a position whose writer died, so its timestamp is the sealer's wall clock and can postdate the events around it. The `node:vm` engine already skipped it; QuickJS did not, so the same log replayed with different `Date.now()` values on the two engines. `isSealedNoopEvent` is now exported from `@workflow/world` as the single test both engines and the trace viewer use. Sealed positions also no longer count toward a run's event ceiling or disqualify it from latency telemetry. diff --git a/packages/core/src/events-consumer.ts b/packages/core/src/events-consumer.ts index 427694041f..b8d11622c2 100644 --- a/packages/core/src/events-consumer.ts +++ b/packages/core/src/events-consumer.ts @@ -1,4 +1,9 @@ -import { type Event, entityEventClass, envNumber } from '@workflow/world'; +import { + type Event, + entityEventClass, + envNumber, + isSealedNoopEvent, +} from '@workflow/world'; import { eventsLogger } from './logger.js'; /** @@ -331,7 +336,7 @@ export class EventsConsumer { // event's by the index it holds. this.drainParked(); const currentEvent = this.events[this.eventIndex] ?? null; - if (currentEvent !== null && currentEvent.eventType === 'noop') { + if (currentEvent !== null && isSealedNoopEvent(currentEvent)) { this.skipSealedNoop(currentEvent); continue; } diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 11667984f4..5fa19d63ea 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -26,6 +26,7 @@ import { type EventResult, getQueueTopicPrefix, isLegacySpecVersion, + isSealedNoopEvent, isTerminalRunEventType, ROOT_RUN_ID_ATTRIBUTE, type RunInput, @@ -2906,14 +2907,24 @@ export function workflowEntrypoint( // reaches the server-supplied ceiling (undefined ⇒ no // enforcement). The throw is caught below and written as // run_failed / MAX_EVENTS_EXCEEDED. - if ( - maxEventsLimit !== undefined && - eventLog.events.length >= maxEventsLimit - ) { - throw new MaxEventsExceededError( - eventLog.events.length, - maxEventsLimit + // Sealed-log noops are excluded: the ceiling exists to + // stop a runaway WORKFLOW, and a noop is written by the + // backend to seal a position whose writer died. Counting + // them would spend the user's event budget on the + // backend's bookkeeping, and would make the limit a run + // hits depend on how much write contention it happened + // to see. + if (maxEventsLimit !== undefined) { + const workflowEventCount = eventLog.events.reduce( + (n, e) => (isSealedNoopEvent(e) ? n : n + 1), + 0 ); + if (workflowEventCount >= maxEventsLimit) { + throw new MaxEventsExceededError( + workflowEventCount, + maxEventsLimit + ); + } } // Latency telemetry: judge TTFS eligibility against the @@ -2924,11 +2935,18 @@ export function workflowEntrypoint( // committed pre-step attr_set, and the detour it marks // is subtracted via preStepAttrStartMs regardless of // which invocation wrote it (see runtime/step-latency.ts). + // noop is permitted for the same reason attr_set is: it + // is not evidence the run had already made progress. A + // seal says a concurrent writer died, which says nothing + // about this invocation, and excluding it would silently + // drop every contended run out of the TTFS dataset -- + // exactly the runs worth measuring. invocationStartedClean ??= eventLog.events.every( (e) => e.eventType === 'run_created' || e.eventType === 'run_started' || - e.eventType === 'attr_set' + e.eventType === 'attr_set' || + isSealedNoopEvent(e) ); runtimeLogger.debug('Starting workflow execution', { diff --git a/packages/core/src/runtime/quickjs-runtime.test.ts b/packages/core/src/runtime/quickjs-runtime.test.ts index a5232d1e0f..958cdf42ca 100644 --- a/packages/core/src/runtime/quickjs-runtime.test.ts +++ b/packages/core/src/runtime/quickjs-runtime.test.ts @@ -670,6 +670,75 @@ describe('deterministic replay clock', () => { }); expect(unwrapResult(r3.completed!.result)).toEqual(result); }); + + it('does not let a sealed-log noop move the clock', async () => { + // A noop's createdAt is the SEALER's wall clock, and a seal can happen + // long after the events at higher positions committed. Feeding it to the + // clock would make a log whose hole was sealed replay differently from + // the same log whose hole its own writer filled — and, because the clock + // is monotonic, would poison every later Date.now() in the run. The + // node:vm engine gets this from EventsConsumer's noop skip; this pins the + // same rule for the QuickJS event loop, which advances the clock in its + // own pass over the log. + const run = makeRun(); + + const probe = await runQuickJSWorkflow({ + workflowCode: sleepTimingWorkflow, + workflowId: 'workflow//test//workflow', + workflowRun: run, + events: [], + }); + const waitCid = probe.suspended!.pendingOperations[0].correlationId; + + const waitCreatedAt = new Date('2025-01-01T00:00:01Z'); + const waitCompletedAt = new Date('2025-01-01T00:00:11Z'); + const sealedAt = new Date('2025-01-01T01:00:00Z'); + + // The hole sits BETWEEN wait_created and wait_completed, which is where a + // fanout leaves one: the position was handed out, its writer died, and the + // events above it committed on their own clocks. + const withSeal = [ + runCreatedEvent(run), + { + eventId: 'evnt_002', + runId: run.runId, + eventType: 'wait_created' as const, + correlationId: waitCid, + eventData: { resumeAt: waitCompletedAt }, + createdAt: waitCreatedAt, + }, + { + eventId: 'evnt_003', + runId: run.runId, + eventType: 'noop' as const, + correlationId: 'noop_00000000000000000000000003', + eventData: { sealed: true }, + createdAt: sealedAt, + }, + { + eventId: 'evnt_004', + runId: run.runId, + eventType: 'wait_completed' as const, + correlationId: waitCid, + createdAt: waitCompletedAt, + }, + ]; + + const sealed = await runQuickJSWorkflow({ + workflowCode: sleepTimingWorkflow, + workflowId: 'workflow//test//workflow', + workflowRun: run, + events: withSeal as never, + }); + const result = unwrapResult(sealed.completed!.result) as { + startTime: number; + endTime: number; + }; + + // The wait completed at its own timestamp, not the seal's. + expect(result.endTime).toBe(+waitCompletedAt); + expect(result.endTime).toBeLessThan(+sealedAt); + }); }); describe('AbortController (hook-backed)', () => { diff --git a/packages/core/src/runtime/quickjs-runtime.ts b/packages/core/src/runtime/quickjs-runtime.ts index fdc46157f5..bc1cf5f80e 100644 --- a/packages/core/src/runtime/quickjs-runtime.ts +++ b/packages/core/src/runtime/quickjs-runtime.ts @@ -30,11 +30,12 @@ */ import { SerializationError } from '@workflow/errors'; -import type { - Event, - RunInput, - WorkflowRun, - WorldCapabilities, +import { + type Event, + isSealedNoopEvent, + type RunInput, + type WorkflowRun, + type WorldCapabilities, } from '@workflow/world'; import * as nanoid from 'nanoid'; import { @@ -1817,6 +1818,18 @@ async function processEvents( ): Promise { let resolved = false; for (const event of events) { + // A sealed-log noop occupies a slot whose writer died; the run never + // observed it. Step over it BEFORE the clock line below, not at the + // switch: its `createdAt` is the sealer's wall clock and can postdate + // every real event around it, so advancing to it would leak the sealer's + // schedule into replay — and because the clock is monotonic, every later + // Date.now() in the run with it. That would make a log whose hole was + // sealed replay differently from the same log whose hole its own writer + // filled, and differently from this log on the node:vm engine, which + // skips noops in `EventsConsumer` before `onConsumedEvent` feeds the + // clock. Same rule, both engines, one predicate. + if (isSealedNoopEvent(event)) continue; + // Advance the VM's deterministic clock to this event's creation time // BEFORE resolving anything, so workflow code unblocked by this event // observes Date.now() at (or after — the clock is monotonic) the time diff --git a/packages/web-shared/src/lib/sealed-events.ts b/packages/web-shared/src/lib/sealed-events.ts index 0eb777f455..d54b42ef9d 100644 --- a/packages/web-shared/src/lib/sealed-events.ts +++ b/packages/web-shared/src/lib/sealed-events.ts @@ -1,4 +1,4 @@ -import type { Event } from '@workflow/world'; +import { type Event, isSealedNoopEvent as isSealedNoop } from '@workflow/world'; /** * Sealed-position `noop` events (specVersion 7). @@ -19,9 +19,14 @@ import type { Event } from '@workflow/world'; * the run's. */ export const SEALED_EVENT_MESSAGE = - 'No-op events may be added by Workflow SDK to ensure correctness'; + 'No-op events are written by the backend to seal an abandoned log position'; -/** Whether `event` is a backend-written seal for an abandoned position. */ +/** + * Whether `event` is a backend-written seal for an abandoned position. + * + * Delegates to `@workflow/world` so the UI, the `node:vm` engine and the + * QuickJS engine cannot drift on what a seal is. + */ export function isSealedNoopEvent(event: Pick): boolean { - return event.eventType === 'noop'; + return isSealedNoop(event); } diff --git a/packages/world/src/events.ts b/packages/world/src/events.ts index 57085e034b..8d1cfd0ee2 100644 --- a/packages/world/src/events.ts +++ b/packages/world/src/events.ts @@ -193,6 +193,23 @@ export function isWaitEventType(eventType: string): eventType is WaitEventType { return WAIT_EVENT_TYPES.includes(eventType as WaitEventType); } +/** + * Whether an event is a sealed-log filler occupying an abandoned slot. + * + * The single home for this test, deliberately: a noop is invisible to the run + * but it is a real row of the log, so *every* pass over a log has to decide + * whether it is walking positions (count it) or reconstructing what happened + * (skip it). The two replay engines and the observability trace builder each + * make that decision independently, and the one thing they must agree on is + * that a noop's `createdAt` — the sealer's wall clock, which can postdate + * every real event around it — never becomes a time the run observed. + */ +export function isSealedNoopEvent( + event: Pick | { eventType: string } +): boolean { + return event.eventType === 'noop'; +} + const ChildEntityCreationEventTypeSchema = EventTypeSchema.extract([ 'step_created', 'hook_created', diff --git a/packages/world/src/index.ts b/packages/world/src/index.ts index 2d54b7c7e1..429bdc1d40 100644 --- a/packages/world/src/index.ts +++ b/packages/world/src/index.ts @@ -51,6 +51,7 @@ export { isHookEventRequiringExistence, isHookLifecycleEventType, isRunEventType, + isSealedNoopEvent, isStepEventType, isTerminalRunEventType, isTerminalStepEventType, diff --git a/packages/world/src/spec-version.ts b/packages/world/src/spec-version.ts index 2909c6f38d..6f884882e3 100644 --- a/packages/world/src/spec-version.ts +++ b/packages/world/src/spec-version.ts @@ -61,6 +61,13 @@ export const SPEC_VERSION_SUPPORTS_SLOT_IDENTITY = 6 as SpecVersion; * deterministic clock (see `EventsConsumer`). A reader below this version * would fail to parse the unknown event type, which is exactly what * `requiresNewerWorld` exists to catch. + * + * Note this is the READER contract only, so a World is spec-7 compliant by + * construction if it allocates each position at the commit that occupies it: + * no write can then leave a position empty, so it has no holes to seal and + * will never emit a `noop`. Pre-assigning positions ahead of the commit is + * what creates the obligation (see `building-a-world.mdx`), and only a World + * that does so needs the sealing half. */ export const SPEC_VERSION_SUPPORTS_SEALED_LOG = 7 as SpecVersion; From aa41ee5387e4713c8a6826967a731b055faf22c0 Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Thu, 20 Aug 2026 12:52:55 -0700 Subject: [PATCH 07/10] feat(world): gate the sealed-log spec bump behind WORKFLOW_SEALED_LOG Moving SPEC_VERSION_CURRENT to 7 stamped spec 7 on every run every World created, with no way back short of publishing another SDK. That took the Python runtime down outright -- it pins its own accepted range and rejects 7, so every event 500s and every run times out (8,444 validation errors in one E2E Python Conformance run, which is what held E2E Required Check red). The fix is the shape slot identity itself shipped behind before going unconditional: SPEC_VERSION_CURRENT stays the floor, `mintedSpecVersion()` answers what a World actually stamps, and the three Worlds declare that instead of the constant so they keep moving together. Opt-IN rather than opt-out, because stamping a version is not a local decision -- it changes what every other reader of those runs has to understand, including readers that do not ship on this release train. A spec-7 log may hold `noop` rows and a reader that does not skip them cannot replay it at all. Default-off makes the rollout a deployment setting instead of a publish. SPEC_VERSION_MAX_SUPPORTED stays at 7: every build READS sealed logs and skips noops regardless of the flag, which is what keeps the two dials honest -- and puts them back in the staged relationship they document, ceiling ahead of floor. That also restores the start.test.ts case the bump had to weaken when the two collapsed onto the same number. Runs are unaffected either way: a run's version is stamped once at creation and read from the run for life, so flipping this changes only what new runs get. Docs: WORKFLOW_SEALED_LOG in runtime-tuning, plus the gate noted in event-sourcing and building-a-world, where spec 7 read as unconditional. Co-Authored-By: Claude Opus 5 (1M context) --- .changeset/sealed-log-noop-clock-quickjs.md | 7 -- .../sealed-log-noop-shared-predicate.md | 5 ++ .changeset/sealed-log-spec-seven.md | 4 +- .../docs/v5/configuration/runtime-tuning.mdx | 10 +++ .../docs/v5/how-it-works/event-sourcing.mdx | 2 +- docs/content/worlds/v5/building-a-world.mdx | 4 +- packages/core/src/runtime/start.test.ts | 12 ++-- packages/world-local/src/index.ts | 4 +- packages/world-postgres/src/index.ts | 4 +- packages/world-vercel/src/index.ts | 4 +- packages/world/src/index.ts | 2 + packages/world/src/spec-version.test.ts | 39 +++++++++- packages/world/src/spec-version.ts | 72 +++++++++++++++---- 13 files changed, 131 insertions(+), 38 deletions(-) delete mode 100644 .changeset/sealed-log-noop-clock-quickjs.md create mode 100644 .changeset/sealed-log-noop-shared-predicate.md diff --git a/.changeset/sealed-log-noop-clock-quickjs.md b/.changeset/sealed-log-noop-clock-quickjs.md deleted file mode 100644 index bbe33cde90..0000000000 --- a/.changeset/sealed-log-noop-clock-quickjs.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -'@workflow/web-shared': patch -'@workflow/world': patch -'@workflow/core': patch ---- - -Fix the QuickJS replay engine advancing the deterministic clock to a sealed position's timestamp. A `noop` is written by the backend when it seals a position whose writer died, so its timestamp is the sealer's wall clock and can postdate the events around it. The `node:vm` engine already skipped it; QuickJS did not, so the same log replayed with different `Date.now()` values on the two engines. `isSealedNoopEvent` is now exported from `@workflow/world` as the single test both engines and the trace viewer use. Sealed positions also no longer count toward a run's event ceiling or disqualify it from latency telemetry. diff --git a/.changeset/sealed-log-noop-shared-predicate.md b/.changeset/sealed-log-noop-shared-predicate.md new file mode 100644 index 0000000000..396816dce5 --- /dev/null +++ b/.changeset/sealed-log-noop-shared-predicate.md @@ -0,0 +1,5 @@ +--- +'@workflow/web-shared': patch +--- + +Render sealed log positions (`noop` events) as the log rows they are: shown in event lists, excluded from span geometry and trace duration, since a seal's timestamp belongs to whichever reader wrote it rather than to the run. diff --git a/.changeset/sealed-log-spec-seven.md b/.changeset/sealed-log-spec-seven.md index cd52698930..8414e55f44 100644 --- a/.changeset/sealed-log-spec-seven.md +++ b/.changeset/sealed-log-spec-seven.md @@ -1,7 +1,9 @@ --- '@workflow/world': minor '@workflow/world-vercel': minor +'@workflow/world-local': minor +'@workflow/world-postgres': minor '@workflow/core': minor --- -Sealed-log event identity (specVersion 7): runs are stamped at spec 7, whose slot positions come from a per-run sequencer on the backend instead of writers racing conditional creates. The backend may fill a position whose writer died with a server-written `noop` event ("sealing"); the runtime skips noops during replay without advancing the deterministic clock, and the read union accepts the new event type. `SPEC_VERSION_SUPPORTS_SEALED_LOG` is exported from `@workflow/world`. +Add the sealed-log event identity (specVersion 7), opt-in via `WORKFLOW_SEALED_LOG=1`. A sealed-log run's event positions are assigned by the backend before each write commits, so concurrent writers never contend for a position. A position whose writer dies is closed by the backend with a `noop` event; replay steps over those without delivering them or advancing the deterministic clock. Every runtime reads sealed logs regardless of the setting, and a run's version is fixed at creation, so enabling it affects only new runs. diff --git a/docs/content/docs/v5/configuration/runtime-tuning.mdx b/docs/content/docs/v5/configuration/runtime-tuning.mdx index a2a23ad0ff..9cca7defba 100644 --- a/docs/content/docs/v5/configuration/runtime-tuning.mdx +++ b/docs/content/docs/v5/configuration/runtime-tuning.mdx @@ -97,6 +97,16 @@ For example, a workflow can run a 10-minute inline step even with `WORKFLOW_REPL - The check trades one failure for another. Most holes stand for an event that never happened, and replaying past those is correct. A hole standing for an event that did happen looks identical, and replaying past that one produces a run whose result is silently wrong. Failing is the recoverable side of that trade. - Set `0` to replay across holes instead. +### `WORKFLOW_SEALED_LOG` + +- Default: disabled +- Creates new runs at the sealed-log spec version, in which the World's backend assigns each event its position *before* the write commits rather than letting concurrent writers race for one. Concurrent writes then never contend for a position, which is what makes a wide fan-out cheap. +- The price of assigning positions in advance is that a writer which claims one and then dies leaves a position no writer will ever fill. The backend closes such a position by writing a `noop` event into it once it can prove the position was abandoned, so a reader still sees the dense log it needs. Replay steps over a `noop` without delivering it to the workflow and without advancing the [deterministic clock](/docs/how-it-works/replay) — its timestamp belongs to whichever reader sealed it, not to the run. +- Set `1` to create sealed-log runs. +- This is opt-in because stamping a spec version is not a local decision: it changes what every *other* reader of those runs has to understand. A run created here may be read back by a different runtime — the Python runtime, for one, pins its own accepted range — and a reader that does not know to skip `noop` events cannot replay the log at all. Turn it on only where every reader of those runs understands them. +- Existing runs are unaffected either way. A run's spec version is stamped once, at creation, and read from the run for the rest of its life, so flipping this changes only what *new* runs get. Every build reads sealed logs regardless of the setting. +- Only the Vercel World seals. The Local and Postgres Worlds allocate each position at the commit that occupies it, so they cannot leave a hole and never write a `noop`; the flag still moves the version they stamp, so the fleet stays on one spec. + ### `WORKFLOW_PRECONDITION_MAX_INPROCESS_RESTARTS` - Default: `3` diff --git a/docs/content/docs/v5/how-it-works/event-sourcing.mdx b/docs/content/docs/v5/how-it-works/event-sourcing.mdx index 0aa5411f84..7b97c3efac 100644 --- a/docs/content/docs/v5/how-it-works/event-sourcing.mdx +++ b/docs/content/docs/v5/how-it-works/event-sourcing.mdx @@ -271,7 +271,7 @@ The observability UI greys out the events it can identify this way, with the rea ## Sealed Positions (noop events) -Runs at specVersion 7 and above live in a *sealed log*: the backend hands each write its position from a per-run sequencer **before** the write commits, so concurrent writers hold distinct positions and never race one another for a slot. The trade is that a writer can claim a position and then die — a crashed process, a cancelled transaction — leaving a hole that no writer will ever fill, and a hole reads exactly like an event the reader failed to load. +Runs at specVersion 7 and above live in a *sealed log*: the backend hands each write its position from a per-run sequencer **before** the write commits, so concurrent writers hold distinct positions and never race one another for a slot. New runs are created this way only where [`WORKFLOW_SEALED_LOG`](/docs/configuration/runtime-tuning#workflow_sealed_log) is enabled; every runtime *reads* a sealed log regardless, and a run's version is fixed at creation, so turning the setting on or off never changes a run already in flight. The trade is that a writer can claim a position and then die — a crashed process, a cancelled transaction — leaving a hole that no writer will ever fill, and a hole reads exactly like an event the reader failed to load. The backend restores the dense log at read time by **sealing** such positions: once a hole is provably abandoned (bounded by the commit time of later positions — positions are handed out in order, so a committed later position proves how long the hole has been open), the backend writes a `noop` event into it. A `noop` occupies its position — length-based completeness checks, cursors, and pagination all count it — and means nothing: diff --git a/docs/content/worlds/v5/building-a-world.mdx b/docs/content/worlds/v5/building-a-world.mdx index 4fde24ae98..e822ea3e38 100644 --- a/docs/content/worlds/v5/building-a-world.mdx +++ b/docs/content/worlds/v5/building-a-world.mdx @@ -148,7 +148,9 @@ Allocate the position **at the commit**, in the same operation that appends the Spec version 7 legitimizes one alternative to allocate-at-commit, for Worlds whose store makes commit-time allocation a contention bottleneck: hand positions out from a per-run atomic counter **before** the commit, and restore density at read time. Pre-assignment means concurrent writers hold distinct positions and never race for one — but a writer that claims a position and dies leaves a permanent hole. A World that allocates this way MUST therefore **seal** provably abandoned positions by writing a `noop` event into them (racing the original writer at the same uniqueness fence — losing that race means the real event landed, which is success), and MUST NOT return a page with an interior hole: return the dense prefix below the hole and let the caller's next page pick up past it once the position resolves to an event or a seal. -The runtime's side of the contract: it skips `noop` events during replay — never delivered to a consumer, never advancing the deterministic clock — so a sealed log replays identically to one whose holes were filled by their writers. `noop` is not user-creatable and is never sent to `events.create()`; only your own read path may write one. Worlds that allocate at the commit (a synchronous counter, a unique-constraint append) keep perfect density by construction and never need any of this — `world-local` and `world-postgres` never seal. +The runtime's side of the contract: it skips `noop` events during replay — never delivered to a consumer, never advancing the deterministic clock — so a sealed log replays identically to one whose holes were filled by their writers. `noop` is not user-creatable and is never sent to `events.create()`; only your own read path may write one. Worlds that allocate at the commit (a synchronous counter, a unique-constraint append) keep perfect density by construction and never need any of this — `world-local` and `world-postgres` never seal, and a World that allocates at the commit is spec-7 compliant with no work. + +Note the version a World stamps is gated: `mintedSpecVersion()` answers 7 only where [`WORKFLOW_SEALED_LOG`](/docs/configuration/runtime-tuning#workflow_sealed_log) is enabled, and the slot-identity floor otherwise. Declare `mintedSpecVersion()` rather than a literal so your World moves with the fleet, and note that a run created at spec 7 may be read by a runtime other than the one that created it — the reader has to understand `noop` before anything stamps 7 in that environment. `events.create()` params carry `eventCount`: how many events the writer held in the log it replayed from, which is the position it expects to land on minus one. Attempt `eventCount + 1`. When that position is taken, **do not reject the write**. Advance to the next free position, commit there, and return the events occupying the positions you skipped over on the success response, in `events` with a matching `cursor` and `hasMore`. The writer merges them into its own log and replays once, rather than paying a second round trip to discover it was behind. A caller with a stale count is the normal case for a fan-out, and rejecting it would serialize writes the runtime issues in parallel. diff --git a/packages/core/src/runtime/start.test.ts b/packages/core/src/runtime/start.test.ts index f810c26168..40cad7473f 100644 --- a/packages/core/src/runtime/start.test.ts +++ b/packages/core/src/runtime/start.test.ts @@ -200,13 +200,11 @@ describe('start', () => { expect(mockQueue).not.toHaveBeenCalled(); }); - it('accepts a world that declares the ceiling version', async () => { - // With the sealed-log bump the default and the ceiling coincide at 7, - // so "above the default" is momentarily unoccupiable — what this pins - // instead is that a World declaring the ceiling is admitted and its - // declaration is what gets stamped, not this runtime's default. (When - // the ceiling next moves ahead of the default, point the declaration - // between them again.) + it('accepts a world that opts into a spec version above the default', async () => { + // `world-vercel` declares the sealed-log version when the mint flag is + // on, so its new runs are created above the floor. An equality check + // against the default would make the runtime refuse the adapter shipped + // alongside it, and the failure surfaces only in e2e against that World. const validWorkflow = Object.assign(() => Promise.resolve('result'), { workflowId: 'test-workflow', }); diff --git a/packages/world-local/src/index.ts b/packages/world-local/src/index.ts index dfa56584df..764b43afc5 100644 --- a/packages/world-local/src/index.ts +++ b/packages/world-local/src/index.ts @@ -2,7 +2,7 @@ import { promises as fs } from 'node:fs'; import { rm } from 'node:fs/promises'; import path from 'node:path'; import type { QueuePrefix, World } from '@workflow/world'; -import { reenqueueActiveRuns, SPEC_VERSION_CURRENT } from '@workflow/world'; +import { mintedSpecVersion, reenqueueActiveRuns } from '@workflow/world'; import { warnIfRunningInVercelDeployment } from './build-target-mismatch.js'; import type { Config } from './config.js'; import { config, resolveRecoverActiveRuns } from './config.js'; @@ -72,7 +72,7 @@ export function createWorld(args?: Partial): LocalWorld { ); const recoverActiveRuns = resolveRecoverActiveRuns(mergedConfig); return { - specVersion: SPEC_VERSION_CURRENT, + specVersion: mintedSpecVersion(), capabilities: { hookRetention: { active: true }, // world-local deduplicates concurrent `hook_received` writes sharing a diff --git a/packages/world-postgres/src/index.ts b/packages/world-postgres/src/index.ts index 84618fdbc2..4aeaa3ce58 100644 --- a/packages/world-postgres/src/index.ts +++ b/packages/world-postgres/src/index.ts @@ -1,5 +1,5 @@ import type { Storage, World } from '@workflow/world'; -import { reenqueueActiveRuns, SPEC_VERSION_CURRENT } from '@workflow/world'; +import { mintedSpecVersion, reenqueueActiveRuns } from '@workflow/world'; import { Pool } from 'pg'; import type { PostgresWorldConfig } from './config.js'; import { createClient, type Drizzle } from './drizzle/index.js'; @@ -63,7 +63,7 @@ export function createWorld( const streamer = createStreamer(pool, drizzle); return { - specVersion: SPEC_VERSION_CURRENT, + specVersion: mintedSpecVersion(), capabilities: { hookRetention: { active: true }, }, diff --git a/packages/world-vercel/src/index.ts b/packages/world-vercel/src/index.ts index 224f3091be..7ac12ec34e 100644 --- a/packages/world-vercel/src/index.ts +++ b/packages/world-vercel/src/index.ts @@ -1,5 +1,5 @@ import type { World } from '@workflow/world'; -import { SPEC_VERSION_CURRENT } from '@workflow/world'; +import { mintedSpecVersion } from '@workflow/world'; import { createAnalytics } from './analytics.js'; import { createRunId, describeRun } from './create-run-id.js'; import { createGetEncryptionKeyForRun } from './encryption.js'; @@ -39,7 +39,7 @@ export function createWorld(config?: APIConfig): World { // version that introduced slots: a bump has to move this declaration with // it, or the runtime's compatibility floor rises past the adapter shipped // alongside it and rejects it (see `assertWorldSupportsRuntimeProtocol`). - specVersion: SPEC_VERSION_CURRENT, + specVersion: mintedSpecVersion(), capabilities: { hookRetention: { active: true }, // Vercel Queues supports maxConcurrency-limited consumers, which diff --git a/packages/world/src/index.ts b/packages/world/src/index.ts index 429bdc1d40..d6efe39fac 100644 --- a/packages/world/src/index.ts +++ b/packages/world/src/index.ts @@ -137,7 +137,9 @@ export { export type { SpecVersion } from './spec-version.js'; export { isLegacySpecVersion, + mintedSpecVersion, requiresNewerWorld, + SEALED_LOG_ENV_VAR, SPEC_VERSION_CURRENT, SPEC_VERSION_LEGACY, SPEC_VERSION_MAX_SUPPORTED, diff --git a/packages/world/src/spec-version.test.ts b/packages/world/src/spec-version.test.ts index b348714955..87604633ae 100644 --- a/packages/world/src/spec-version.test.ts +++ b/packages/world/src/spec-version.test.ts @@ -1,7 +1,9 @@ import { describe, expect, it } from 'vitest'; import { isLegacySpecVersion, + mintedSpecVersion, requiresNewerWorld, + SEALED_LOG_ENV_VAR, SPEC_VERSION_CURRENT, SPEC_VERSION_LEGACY, SPEC_VERSION_MAX_SUPPORTED, @@ -12,10 +14,43 @@ import { } from './spec-version.js'; describe('spec version constants', () => { - it('current spec version is the sealed-log version', () => { + it('the floor a World stamps is still the slot-identity version', () => { expect(SPEC_VERSION_SUPPORTS_SLOT_IDENTITY).toBe(6); expect(SPEC_VERSION_SUPPORTS_SEALED_LOG).toBe(7); - expect(SPEC_VERSION_CURRENT).toBe(SPEC_VERSION_SUPPORTS_SEALED_LOG); + // Sealed-log runs are opt-in, so the floor has NOT moved to 7: the version + // a World actually stamps comes from `mintedSpecVersion`. + expect(SPEC_VERSION_CURRENT).toBe(SPEC_VERSION_SUPPORTS_SLOT_IDENTITY); + }); + + describe('mintedSpecVersion', () => { + it('stamps the slot-identity floor by default', () => { + // Default-off matters beyond this package: a spec-7 log may hold `noop` + // rows, and every reader of those runs has to know to skip them -- + // including readers that do not ship on this release train. + expect(mintedSpecVersion({})).toBe(SPEC_VERSION_CURRENT); + }); + + it('stamps the sealed-log version when opted in', () => { + for (const on of ['1', 'true', 'TRUE']) { + expect(mintedSpecVersion({ [SEALED_LOG_ENV_VAR]: on })).toBe( + SPEC_VERSION_SUPPORTS_SEALED_LOG + ); + } + }); + + it('treats an explicit off and a malformed value as off', () => { + for (const off of ['0', 'false', '', 'yes-please']) { + expect(mintedSpecVersion({ [SEALED_LOG_ENV_VAR]: off })).toBe( + SPEC_VERSION_CURRENT + ); + } + }); + + it('never stamps a version this build cannot read back', () => { + expect( + mintedSpecVersion({ [SEALED_LOG_ENV_VAR]: '1' }) + ).toBeLessThanOrEqual(SPEC_VERSION_MAX_SUPPORTED); + }); }); it('the readable ceiling moves with the version we stamp', () => { diff --git a/packages/world/src/spec-version.ts b/packages/world/src/spec-version.ts index 6f884882e3..d4076837fa 100644 --- a/packages/world/src/spec-version.ts +++ b/packages/world/src/spec-version.ts @@ -5,6 +5,8 @@ * from @workflow/world rather than using arbitrary numbers. */ +import { envFlag } from './env-config.js'; + declare const SpecVersionBrand: unique symbol; /** @@ -77,11 +79,15 @@ export const SPEC_VERSION_SUPPORTS_SEALED_LOG = 7 as SpecVersion; * * This is both the version a World stamps on the runs it creates and the * *lowest* one this runtime accepts from a World (see - * `assertWorldSupportsRuntimeProtocol`). The two coincide because slot - * numbering is a requirement of the World contract rather than a capability to - * opt into: a World declaring anything below this allocates event ids the - * runtime cannot read positions out of, so admitting it would only move the - * failure from startup to the middle of a run. + * `assertWorldSupportsRuntimeProtocol`). Slot numbering is a requirement of + * the World contract rather than a capability to opt into: a World declaring + * anything below this allocates event ids the runtime cannot read positions + * out of, so admitting it would only move the failure from startup to the + * middle of a run. + * + * This is the FLOOR, not necessarily what gets stamped. Sealed-log runs sit + * one version above it and are opt-in, so what a World actually stamps comes + * from {@link mintedSpecVersion}; this is what that falls back to. * * A World therefore declares this constant rather than a literal, so a bump * moves the declaration and the floor together. Pinning a literal would leave @@ -93,18 +99,58 @@ export const SPEC_VERSION_SUPPORTS_SEALED_LOG = 7 as SpecVersion; * run's identity scheme from what is stored rather than from this constant. */ export const SPEC_VERSION_CURRENT = - SPEC_VERSION_SUPPORTS_SEALED_LOG as SpecVersion; + SPEC_VERSION_SUPPORTS_SLOT_IDENTITY as SpecVersion; + +/** + * Environment variable that opts new runs INTO the sealed log. + * + * Read per `createWorld()` call rather than at module load, so a test or a + * single process can create worlds in both modes. + */ +export const SEALED_LOG_ENV_VAR = 'WORKFLOW_SEALED_LOG'; + +/** + * The spec version a World should stamp on the runs it creates. + * + * Sealed-log runs are opt-in for now, so this answers + * {@link SPEC_VERSION_CURRENT} unless {@link SEALED_LOG_ENV_VAR} turns it on. + * Same shape, and the same reasoning, as the flag slot identity itself shipped + * behind before going unconditional. + * + * Opt-in rather than opt-out because stamping a version is not a local + * decision: it changes what every OTHER reader of the run has to understand. + * A spec-7 log may contain `noop` rows, and a reader that does not know to + * skip them cannot replay it — which includes readers that are not this + * package and do not ship on its release train. The Python runtime pins its + * own accepted range and rejects 7 outright today, so a default-on bump takes + * every Python workflow down the moment this is published, with no way back + * except another release. Default-off makes the rollout a deployment setting: + * turn it on where the backend seals and every reader of those runs + * understands noops, leave it off everywhere else. + * + * Every World reads runs up to {@link SPEC_VERSION_MAX_SUPPORTED} whatever + * this returns, so turning the flag off here does not make runs another + * process created unreadable. + */ +export function mintedSpecVersion( + env: Record = process.env +): SpecVersion { + return envFlag(SEALED_LOG_ENV_VAR, false, env) + ? SPEC_VERSION_SUPPORTS_SEALED_LOG + : SPEC_VERSION_CURRENT; +} /** * The highest spec version this SDK can read. * - * Kept distinct from `SPEC_VERSION_CURRENT` even though the two are equal - * today. They answer different questions, "what do we write?" versus "what can - * we still read?", and they come apart in the release order a spec bump - * follows: a reader that can already handle the next version raises this - * ceiling first, and `SPEC_VERSION_CURRENT` follows only once the version is - * safe to stamp. Collapsing them into one constant would make that staging - * impossible to express. + * Kept distinct from `SPEC_VERSION_CURRENT`, and right now they genuinely + * differ. They answer different questions, "what do we write?" versus "what + * can we still read?", and they come apart in exactly the release order a spec + * bump follows: a reader that can already handle the next version raises this + * ceiling first, and stamping follows only once the version is safe to mint + * everywhere. Sealed-log support is at that first stage — every build reads + * spec 7 and skips `noop`, while {@link mintedSpecVersion} still has to be + * turned on before anything creates a spec-7 run. */ export const SPEC_VERSION_MAX_SUPPORTED = SPEC_VERSION_SUPPORTS_SEALED_LOG as SpecVersion; From d6194ccaf571c3242d3f00777f57ca6516e76e3d Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Thu, 20 Aug 2026 13:02:18 -0700 Subject: [PATCH 08/10] docs: drop a link to a page that does not exist The WORKFLOW_SEALED_LOG entry linked /docs/how-it-works/replay for the deterministic clock; there is no such page, and Docs Links caught it. The sentence stands on its own without it. Co-Authored-By: Claude Opus 5 (1M context) --- docs/content/docs/v5/configuration/runtime-tuning.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/docs/v5/configuration/runtime-tuning.mdx b/docs/content/docs/v5/configuration/runtime-tuning.mdx index 9cca7defba..c495500a3e 100644 --- a/docs/content/docs/v5/configuration/runtime-tuning.mdx +++ b/docs/content/docs/v5/configuration/runtime-tuning.mdx @@ -101,7 +101,7 @@ For example, a workflow can run a 10-minute inline step even with `WORKFLOW_REPL - Default: disabled - Creates new runs at the sealed-log spec version, in which the World's backend assigns each event its position *before* the write commits rather than letting concurrent writers race for one. Concurrent writes then never contend for a position, which is what makes a wide fan-out cheap. -- The price of assigning positions in advance is that a writer which claims one and then dies leaves a position no writer will ever fill. The backend closes such a position by writing a `noop` event into it once it can prove the position was abandoned, so a reader still sees the dense log it needs. Replay steps over a `noop` without delivering it to the workflow and without advancing the [deterministic clock](/docs/how-it-works/replay) — its timestamp belongs to whichever reader sealed it, not to the run. +- The price of assigning positions in advance is that a writer which claims one and then dies leaves a position no writer will ever fill. The backend closes such a position by writing a `noop` event into it once it can prove the position was abandoned, so a reader still sees the dense log it needs. Replay steps over a `noop` without delivering it to the workflow and without advancing the deterministic clock — its timestamp belongs to whichever reader sealed it, not to the run. - Set `1` to create sealed-log runs. - This is opt-in because stamping a spec version is not a local decision: it changes what every *other* reader of those runs has to understand. A run created here may be read back by a different runtime — the Python runtime, for one, pins its own accepted range — and a reader that does not know to skip `noop` events cannot replay the log at all. Turn it on only where every reader of those runs understands them. - Existing runs are unaffected either way. A run's spec version is stamped once, at creation, and read from the run for the rest of its life, so flipping this changes only what *new* runs get. Every build reads sealed logs regardless of the setting. From b832f7dbd393be16d39851c1952f46d9d276ffd5 Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Thu, 20 Aug 2026 13:44:52 -0700 Subject: [PATCH 09/10] feat(world): make the sealed log the default, keep a kill switch Flips `WORKFLOW_SEALED_LOG` from opt-in to opt-out and moves SPEC_VERSION_CURRENT to the sealed-log version, so new runs get spec 7 by default and the flag exists only to put a deployment back on the previous scheme without a release. The floor `assertWorldSupportsRuntimeProtocol` admits moves down to the slot-identity version, and that part is load-bearing rather than cosmetic. The accepted range was `[SPEC_VERSION_CURRENT, MAX_SUPPORTED]`, so with the default at 7 a World switched back by the kill switch declares 6 and the runtime would reject it -- the rollback would surface as a startup failure, which is the opposite of a kill switch. Flooring at slot identity also admits a World package one version behind the runtime it ships with, the normal state mid-bump: slot identity is what the runtime actually requires, and sealing is a backend capability on top of it. The range narrows again when spec 7 becomes mandatory, exactly as slot identity's own floor did. That also restores the staged floor-below-ceiling relationship the two constants document, so start.test.ts can pin both ends again: a World declaring the ceiling is admitted and stamped, and a World declaring the kill switch's fallback is admitted too. Known consequence, stated rather than discovered: the Python runtime pins its own accepted spec range and rejects 7, so its e2e lanes fail until a `vercel-workflow` release accepts spec 7 and skips `noop`. WORKFLOW_SEALED_LOG=0 is the per-environment escape until then. Co-Authored-By: Claude Opus 5 (1M context) --- .changeset/sealed-log-spec-seven.md | 2 +- .../docs/v5/configuration/runtime-tuning.mdx | 12 +++--- .../docs/v5/how-it-works/event-sourcing.mdx | 2 +- docs/content/worlds/v5/building-a-world.mdx | 2 +- packages/core/src/runtime/start.test.ts | 39 +++++++++++++++--- .../core/src/runtime/world-compatibility.ts | 35 ++++++++++------ packages/world/src/spec-version.test.ts | 34 ++++++++-------- packages/world/src/spec-version.ts | 40 +++++++++---------- 8 files changed, 100 insertions(+), 66 deletions(-) diff --git a/.changeset/sealed-log-spec-seven.md b/.changeset/sealed-log-spec-seven.md index 8414e55f44..8c67fb6325 100644 --- a/.changeset/sealed-log-spec-seven.md +++ b/.changeset/sealed-log-spec-seven.md @@ -6,4 +6,4 @@ '@workflow/core': minor --- -Add the sealed-log event identity (specVersion 7), opt-in via `WORKFLOW_SEALED_LOG=1`. A sealed-log run's event positions are assigned by the backend before each write commits, so concurrent writers never contend for a position. A position whose writer dies is closed by the backend with a `noop` event; replay steps over those without delivering them or advancing the deterministic clock. Every runtime reads sealed logs regardless of the setting, and a run's version is fixed at creation, so enabling it affects only new runs. +New runs are created with the sealed-log event identity (specVersion 7). A sealed-log run's event positions are assigned by the backend before each write commits, so concurrent writers never contend for a position. A position whose writer dies is closed by the backend with a `noop` event; replay steps over those without delivering them or advancing the deterministic clock. Set `WORKFLOW_SEALED_LOG=0` to put a deployment back on the previous scheme. Every runtime reads sealed logs either way, and a run's version is fixed at creation, so the setting only affects new runs. diff --git a/docs/content/docs/v5/configuration/runtime-tuning.mdx b/docs/content/docs/v5/configuration/runtime-tuning.mdx index c495500a3e..086645c400 100644 --- a/docs/content/docs/v5/configuration/runtime-tuning.mdx +++ b/docs/content/docs/v5/configuration/runtime-tuning.mdx @@ -99,13 +99,13 @@ For example, a workflow can run a 10-minute inline step even with `WORKFLOW_REPL ### `WORKFLOW_SEALED_LOG` -- Default: disabled -- Creates new runs at the sealed-log spec version, in which the World's backend assigns each event its position *before* the write commits rather than letting concurrent writers race for one. Concurrent writes then never contend for a position, which is what makes a wide fan-out cheap. +- Default: enabled +- New runs are created at the sealed-log spec version, in which the World's backend assigns each event its position *before* the write commits rather than letting concurrent writers race for one. Concurrent writes then never contend for a position, which is what makes a wide fan-out cheap. - The price of assigning positions in advance is that a writer which claims one and then dies leaves a position no writer will ever fill. The backend closes such a position by writing a `noop` event into it once it can prove the position was abandoned, so a reader still sees the dense log it needs. Replay steps over a `noop` without delivering it to the workflow and without advancing the deterministic clock — its timestamp belongs to whichever reader sealed it, not to the run. -- Set `1` to create sealed-log runs. -- This is opt-in because stamping a spec version is not a local decision: it changes what every *other* reader of those runs has to understand. A run created here may be read back by a different runtime — the Python runtime, for one, pins its own accepted range — and a reader that does not know to skip `noop` events cannot replay the log at all. Turn it on only where every reader of those runs understands them. -- Existing runs are unaffected either way. A run's spec version is stamped once, at creation, and read from the run for the rest of its life, so flipping this changes only what *new* runs get. Every build reads sealed logs regardless of the setting. -- Only the Vercel World seals. The Local and Postgres Worlds allocate each position at the commit that occupies it, so they cannot leave a hole and never write a `noop`; the flag still moves the version they stamp, so the fleet stays on one spec. +- Set `0` to put a deployment back on the previous scheme, where each position is allocated by the write that occupies it. Use this as the kill switch if position assignment turns out to be at fault for event-log problems. +- Existing runs are unaffected either way. A run's spec version is stamped once, at creation, and read from the run for the rest of its life, so flipping this changes only what *new* runs get, and a run in flight keeps the scheme it started on. Every build reads sealed logs regardless of the setting. +- A run created at the sealed-log version can only be replayed by a reader that knows to skip `noop` events. That is every runtime on this release train, but a runtime that pins its own accepted spec range separately — the Python runtime, for one — has to have caught up before it can read these runs. Switch this off in an environment where it has not. +- Only the Vercel World seals. The Local and Postgres Worlds allocate each position at the commit that occupies it, so they cannot leave a hole and never write a `noop`; the setting still moves the version they stamp, so the fleet stays on one spec. ### `WORKFLOW_PRECONDITION_MAX_INPROCESS_RESTARTS` diff --git a/docs/content/docs/v5/how-it-works/event-sourcing.mdx b/docs/content/docs/v5/how-it-works/event-sourcing.mdx index 7b97c3efac..0a7b870ce9 100644 --- a/docs/content/docs/v5/how-it-works/event-sourcing.mdx +++ b/docs/content/docs/v5/how-it-works/event-sourcing.mdx @@ -271,7 +271,7 @@ The observability UI greys out the events it can identify this way, with the rea ## Sealed Positions (noop events) -Runs at specVersion 7 and above live in a *sealed log*: the backend hands each write its position from a per-run sequencer **before** the write commits, so concurrent writers hold distinct positions and never race one another for a slot. New runs are created this way only where [`WORKFLOW_SEALED_LOG`](/docs/configuration/runtime-tuning#workflow_sealed_log) is enabled; every runtime *reads* a sealed log regardless, and a run's version is fixed at creation, so turning the setting on or off never changes a run already in flight. The trade is that a writer can claim a position and then die — a crashed process, a cancelled transaction — leaving a hole that no writer will ever fill, and a hole reads exactly like an event the reader failed to load. +Runs at specVersion 7 and above live in a *sealed log*: the backend hands each write its position from a per-run sequencer **before** the write commits, so concurrent writers hold distinct positions and never race one another for a slot. This is how new runs are created by default; [`WORKFLOW_SEALED_LOG=0`](/docs/configuration/runtime-tuning#workflow_sealed_log) puts a deployment back on the previous scheme. Every runtime *reads* a sealed log regardless, and a run's version is fixed at creation, so changing the setting never affects a run already in flight. The trade is that a writer can claim a position and then die — a crashed process, a cancelled transaction — leaving a hole that no writer will ever fill, and a hole reads exactly like an event the reader failed to load. The backend restores the dense log at read time by **sealing** such positions: once a hole is provably abandoned (bounded by the commit time of later positions — positions are handed out in order, so a committed later position proves how long the hole has been open), the backend writes a `noop` event into it. A `noop` occupies its position — length-based completeness checks, cursors, and pagination all count it — and means nothing: diff --git a/docs/content/worlds/v5/building-a-world.mdx b/docs/content/worlds/v5/building-a-world.mdx index e822ea3e38..4d28f94c4f 100644 --- a/docs/content/worlds/v5/building-a-world.mdx +++ b/docs/content/worlds/v5/building-a-world.mdx @@ -150,7 +150,7 @@ Spec version 7 legitimizes one alternative to allocate-at-commit, for Worlds who The runtime's side of the contract: it skips `noop` events during replay — never delivered to a consumer, never advancing the deterministic clock — so a sealed log replays identically to one whose holes were filled by their writers. `noop` is not user-creatable and is never sent to `events.create()`; only your own read path may write one. Worlds that allocate at the commit (a synchronous counter, a unique-constraint append) keep perfect density by construction and never need any of this — `world-local` and `world-postgres` never seal, and a World that allocates at the commit is spec-7 compliant with no work. -Note the version a World stamps is gated: `mintedSpecVersion()` answers 7 only where [`WORKFLOW_SEALED_LOG`](/docs/configuration/runtime-tuning#workflow_sealed_log) is enabled, and the slot-identity floor otherwise. Declare `mintedSpecVersion()` rather than a literal so your World moves with the fleet, and note that a run created at spec 7 may be read by a runtime other than the one that created it — the reader has to understand `noop` before anything stamps 7 in that environment. +Note the version a World stamps comes from `mintedSpecVersion()`: 7 by default, and the slot-identity version when [`WORKFLOW_SEALED_LOG=0`](/docs/configuration/runtime-tuning#workflow_sealed_log) switches it off. Declare `mintedSpecVersion()` rather than a literal so your World moves with the fleet, and note that a run created at spec 7 may be read by a runtime other than the one that created it — the reader has to understand `noop` before anything stamps 7 in that environment. `events.create()` params carry `eventCount`: how many events the writer held in the log it replayed from, which is the position it expects to land on minus one. Attempt `eventCount + 1`. When that position is taken, **do not reject the write**. Advance to the next free position, commit there, and return the events occupying the positions you skipped over on the success response, in `events` with a matching `cursor` and `hasMore`. The writer merges them into its own log and replays once, rather than paying a second round trip to discover it was behind. A caller with a stale count is the normal case for a fan-out, and rejecting it would serialize writes the runtime issues in parallel. diff --git a/packages/core/src/runtime/start.test.ts b/packages/core/src/runtime/start.test.ts index 40cad7473f..048606d40c 100644 --- a/packages/core/src/runtime/start.test.ts +++ b/packages/core/src/runtime/start.test.ts @@ -5,6 +5,7 @@ import { SPEC_VERSION_MAX_SUPPORTED, SPEC_VERSION_SUPPORTS_ATTRIBUTES, SPEC_VERSION_SUPPORTS_CBOR_QUEUE_TRANSPORT, + SPEC_VERSION_SUPPORTS_SLOT_IDENTITY, } from '@workflow/world'; import { afterEach, @@ -200,11 +201,39 @@ describe('start', () => { expect(mockQueue).not.toHaveBeenCalled(); }); - it('accepts a world that opts into a spec version above the default', async () => { - // `world-vercel` declares the sealed-log version when the mint flag is - // on, so its new runs are created above the floor. An equality check - // against the default would make the runtime refuse the adapter shipped - // alongside it, and the failure surfaces only in e2e against that World. + it('accepts a world switched back to the pre-sealed-log version', async () => { + // What `WORKFLOW_SEALED_LOG=0` produces: `mintedSpecVersion()` answers + // the slot-identity version, so the World declares one BELOW the version + // this runtime stamps by default. The runtime has to admit it, or the + // kill switch would reject the very World it selects and a rollback + // would surface as a startup failure instead. + const rolledBack = Object.assign(() => Promise.resolve('result'), { + workflowId: 'test-workflow', + }); + + setWorld({ + specVersion: SPEC_VERSION_SUPPORTS_SLOT_IDENTITY, + getDeploymentId: vi.fn().mockResolvedValue('deploy_123'), + events: { create: mockEventsCreate }, + queue: mockQueue, + } as any); + + await start(rolledBack, []); + + expect(mockEventsCreate).toHaveBeenCalledWith( + expect.stringMatching(/^wrun_/), + expect.objectContaining({ + eventType: 'run_created', + specVersion: SPEC_VERSION_SUPPORTS_SLOT_IDENTITY, + }), + expect.anything() + ); + }); + + it('accepts a world that declares the ceiling version', async () => { + // The default and the ceiling coincide at the sealed log, so what this + // pins is that a World declaring the ceiling is admitted and its own + // declaration is what gets stamped, not this runtime's default. const validWorkflow = Object.assign(() => Promise.resolve('result'), { workflowId: 'test-workflow', }); diff --git a/packages/core/src/runtime/world-compatibility.ts b/packages/core/src/runtime/world-compatibility.ts index 6409a64c6b..f1afce9a29 100644 --- a/packages/core/src/runtime/world-compatibility.ts +++ b/packages/core/src/runtime/world-compatibility.ts @@ -1,8 +1,8 @@ import { WorkflowRuntimeError } from '@workflow/errors'; import type { World } from '@workflow/world'; import { - SPEC_VERSION_CURRENT, SPEC_VERSION_MAX_SUPPORTED, + SPEC_VERSION_SUPPORTS_SLOT_IDENTITY, } from '@workflow/world'; type WorldSpecVersionMetadata = Pick; @@ -10,18 +10,29 @@ type WorldSpecVersionMetadata = Pick; /** * Rejects a World this runtime cannot speak to. * - * The accepted range is `[SPEC_VERSION_CURRENT, SPEC_VERSION_MAX_SUPPORTED]`. - * Below the current version means an old World package paired with a new - * runtime, which cannot serve the protocol this runtime speaks. Above the + * The accepted range is + * `[SPEC_VERSION_SUPPORTS_SLOT_IDENTITY, SPEC_VERSION_MAX_SUPPORTED]`. Below + * the floor means an old World package paired with a new runtime, which cannot + * serve the protocol this runtime speaks — a World that does not number events + * by position allocates ids the runtime cannot read positions out of. Above the * ceiling means a World built against a newer spec than this runtime knows how * to read. * - * Both bounds are the same version today, so this currently admits exactly one. - * It stays written as a range because the two constants answer different - * questions and come apart while a spec bump is staged: the ceiling rises when - * this runtime learns to read the next version, the floor when that version - * becomes the one Worlds stamp. An equality check against either constant alone - * would reject a World during that window. + * The floor is deliberately the slot-identity version rather than + * `SPEC_VERSION_CURRENT`, which now sits one above it at the sealed log. Two + * reasons, and both are about the window a spec bump is staged over: + * + * - `WORKFLOW_SEALED_LOG=0` puts a deployment back on slot identity, so its + * World declares the lower version. Flooring at the version we stamp by + * default would make that kill switch reject the very World it selects, + * turning a rollback into a startup failure. + * - A World package one version behind the runtime it ships alongside is the + * normal state mid-bump, and it can still serve the protocol: slot identity + * is what the runtime actually requires, and sealed logs are a capability on + * top of it that only the backend implements. + * + * The range narrows again when the sealed log becomes mandatory and the flag + * goes away, exactly as slot identity's own floor did. */ export function assertWorldSupportsRuntimeProtocol( world: WorldSpecVersionMetadata @@ -30,7 +41,7 @@ export function assertWorldSupportsRuntimeProtocol( if ( declared !== undefined && declared !== null && - declared >= SPEC_VERSION_CURRENT && + declared >= SPEC_VERSION_SUPPORTS_SLOT_IDENTITY && declared <= SPEC_VERSION_MAX_SUPPORTED ) { return; @@ -38,7 +49,7 @@ export function assertWorldSupportsRuntimeProtocol( const supportedVersion = declared ?? 'none'; throw new WorkflowRuntimeError( - `This Workflow runtime supports Worlds with spec version ${SPEC_VERSION_CURRENT} ` + + `This Workflow runtime supports Worlds with spec version ${SPEC_VERSION_SUPPORTS_SLOT_IDENTITY} ` + `through ${SPEC_VERSION_MAX_SUPPORTED}, ` + `but the configured World declares spec version ${supportedVersion}. ` + 'Install a World package version compatible with the current Workflow runtime.' diff --git a/packages/world/src/spec-version.test.ts b/packages/world/src/spec-version.test.ts index 87604633ae..e03e735b01 100644 --- a/packages/world/src/spec-version.test.ts +++ b/packages/world/src/spec-version.test.ts @@ -14,42 +14,40 @@ import { } from './spec-version.js'; describe('spec version constants', () => { - it('the floor a World stamps is still the slot-identity version', () => { + it('current spec version is the sealed-log version', () => { expect(SPEC_VERSION_SUPPORTS_SLOT_IDENTITY).toBe(6); expect(SPEC_VERSION_SUPPORTS_SEALED_LOG).toBe(7); - // Sealed-log runs are opt-in, so the floor has NOT moved to 7: the version - // a World actually stamps comes from `mintedSpecVersion`. - expect(SPEC_VERSION_CURRENT).toBe(SPEC_VERSION_SUPPORTS_SLOT_IDENTITY); + expect(SPEC_VERSION_CURRENT).toBe(SPEC_VERSION_SUPPORTS_SEALED_LOG); }); describe('mintedSpecVersion', () => { - it('stamps the slot-identity floor by default', () => { - // Default-off matters beyond this package: a spec-7 log may hold `noop` - // rows, and every reader of those runs has to know to skip them -- - // including readers that do not ship on this release train. + it('stamps the sealed-log version by default', () => { expect(mintedSpecVersion({})).toBe(SPEC_VERSION_CURRENT); + expect(mintedSpecVersion({})).toBe(SPEC_VERSION_SUPPORTS_SEALED_LOG); }); - it('stamps the sealed-log version when opted in', () => { - for (const on of ['1', 'true', 'TRUE']) { - expect(mintedSpecVersion({ [SEALED_LOG_ENV_VAR]: on })).toBe( - SPEC_VERSION_SUPPORTS_SEALED_LOG + it('falls back to slot identity when switched off', () => { + for (const off of ['0', 'false']) { + expect(mintedSpecVersion({ [SEALED_LOG_ENV_VAR]: off })).toBe( + SPEC_VERSION_SUPPORTS_SLOT_IDENTITY ); } }); - it('treats an explicit off and a malformed value as off', () => { - for (const off of ['0', 'false', '', 'yes-please']) { - expect(mintedSpecVersion({ [SEALED_LOG_ENV_VAR]: off })).toBe( + it('stays on by default for an unset or malformed value', () => { + // A flag is an escape hatch, not a hard requirement: a typo must not + // silently move a deployment onto the older identity scheme. + for (const raw of ['', '1', 'true', 'yes-please']) { + expect(mintedSpecVersion({ [SEALED_LOG_ENV_VAR]: raw })).toBe( SPEC_VERSION_CURRENT ); } }); it('never stamps a version this build cannot read back', () => { - expect( - mintedSpecVersion({ [SEALED_LOG_ENV_VAR]: '1' }) - ).toBeLessThanOrEqual(SPEC_VERSION_MAX_SUPPORTED); + expect(mintedSpecVersion({})).toBeLessThanOrEqual( + SPEC_VERSION_MAX_SUPPORTED + ); }); }); diff --git a/packages/world/src/spec-version.ts b/packages/world/src/spec-version.ts index d4076837fa..a4263a9a31 100644 --- a/packages/world/src/spec-version.ts +++ b/packages/world/src/spec-version.ts @@ -99,10 +99,10 @@ export const SPEC_VERSION_SUPPORTS_SEALED_LOG = 7 as SpecVersion; * run's identity scheme from what is stored rather than from this constant. */ export const SPEC_VERSION_CURRENT = - SPEC_VERSION_SUPPORTS_SLOT_IDENTITY as SpecVersion; + SPEC_VERSION_SUPPORTS_SEALED_LOG as SpecVersion; /** - * Environment variable that opts new runs INTO the sealed log. + * Environment variable that opts new runs OUT of the sealed log. * * Read per `createWorld()` call rather than at module load, so a test or a * single process can create worlds in both modes. @@ -110,34 +110,30 @@ export const SPEC_VERSION_CURRENT = export const SEALED_LOG_ENV_VAR = 'WORKFLOW_SEALED_LOG'; /** - * The spec version a World should stamp on the runs it creates. + * The spec version a World should stamp on the runs it creates: the sealed log + * unless {@link SEALED_LOG_ENV_VAR} switches it off, in which case the + * slot-identity version it supersedes. * - * Sealed-log runs are opt-in for now, so this answers - * {@link SPEC_VERSION_CURRENT} unless {@link SEALED_LOG_ENV_VAR} turns it on. * Same shape, and the same reasoning, as the flag slot identity itself shipped - * behind before going unconditional. - * - * Opt-in rather than opt-out because stamping a version is not a local - * decision: it changes what every OTHER reader of the run has to understand. - * A spec-7 log may contain `noop` rows, and a reader that does not know to - * skip them cannot replay it — which includes readers that are not this - * package and do not ship on its release train. The Python runtime pins its - * own accepted range and rejects 7 outright today, so a default-on bump takes - * every Python workflow down the moment this is published, with no way back - * except another release. Default-off makes the rollout a deployment setting: - * turn it on where the backend seals and every reader of those runs - * understands noops, leave it off everywhere else. + * behind before going unconditional: default on, with one env var to put a + * deployment back on the previous scheme without a release. + * + * The fallback is a real fallback, not a formality. Turning this off has to + * leave a World the runtime still admits, which is why + * `assertWorldSupportsRuntimeProtocol` floors at the slot-identity version + * rather than at {@link SPEC_VERSION_CURRENT} — a kill switch that made the + * runtime reject its own World would be no kill switch at all. * * Every World reads runs up to {@link SPEC_VERSION_MAX_SUPPORTED} whatever - * this returns, so turning the flag off here does not make runs another - * process created unreadable. + * this returns, so switching it off here does not make runs another process + * created unreadable. */ export function mintedSpecVersion( env: Record = process.env ): SpecVersion { - return envFlag(SEALED_LOG_ENV_VAR, false, env) - ? SPEC_VERSION_SUPPORTS_SEALED_LOG - : SPEC_VERSION_CURRENT; + return envFlag(SEALED_LOG_ENV_VAR, true, env) + ? SPEC_VERSION_CURRENT + : SPEC_VERSION_SUPPORTS_SLOT_IDENTITY; } /** From 8ef570b186b6959c8b129009128a4129ded17a82 Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Thu, 20 Aug 2026 13:45:08 -0700 Subject: [PATCH 10/10] test(world-vercel): point e2e at the sealed-log server branch DO NOT MERGE THIS COMMIT. Revert it before landing -- the Lint job's "Check WORKFLOW_SERVER_URL_OVERRIDE is empty" step exists to catch it and is expected to fail while it is here. Without this, the e2e suite runs the spec-7 client against a server that does not implement position assignment or sealing. That server is not broken by spec 7 -- its slot-identity checks are open-ended `>=`, so it just treats the run as plain slot identity -- which is worse than a failure for review purposes: the suite goes green having exercised none of the mechanism this PR is about. The inline constant rather than VERCEL_WORKFLOW_SERVER_URL because it wins over the env var, so one value points the deployed workbench apps AND the e2e harness at the same server. That matters here: the harness reads run state from the server the app wrote it to, and splitting them across two servers forks the run. Two world-vercel suites drive a loopback origin by setting VERCEL_WORKFLOW_SERVER_URL, which this constant overrides by design, so they cannot reach their own server while it is pinned. They now skipIf a non-empty override instead of failing with an off-machine 302, and come back automatically when this is reverted. Co-Authored-By: Claude Opus 5 (1M context) --- .../src/trace-propagation.test.ts | 149 ++++++++------- packages/world-vercel/src/utils.test.ts | 172 ++++++++++-------- packages/world-vercel/src/utils.ts | 12 +- 3 files changed, 183 insertions(+), 150 deletions(-) diff --git a/packages/world-vercel/src/trace-propagation.test.ts b/packages/world-vercel/src/trace-propagation.test.ts index b22b2d0ec6..9966e3c641 100644 --- a/packages/world-vercel/src/trace-propagation.test.ts +++ b/packages/world-vercel/src/trace-propagation.test.ts @@ -393,84 +393,95 @@ describe('ws events transport upgrade trace propagation', () => { // MockAgent, so none of them would notice if the node:http client dropped the // injection. This one puts a real origin on loopback and reads the header off // the wire. -describe('node:http mode trace propagation', () => { - let server: Server | undefined; +// These run against a loopback origin, which they select through +// `VERCEL_WORKFLOW_SERVER_URL`. The inline `WORKFLOW_SERVER_URL_OVERRIDE` +// constant WINS over that env var by design, so while a branch-testing +// override is pinned there is no way for these to reach their own server and +// every request leaves the machine. Skipped in that case rather than left to +// fail confusingly; they run again the moment the override goes back to ''. +describe.skipIf(WORKFLOW_SERVER_URL_OVERRIDE !== '')( + 'node:http mode trace propagation', + () => { + let server: Server | undefined; + + beforeEach(() => { + vi.stubEnv(NODE_HTTP_ENV_VAR, '1'); + }); - beforeEach(() => { - vi.stubEnv(NODE_HTTP_ENV_VAR, '1'); - }); + afterEach(async () => { + const toClose = server; + server = undefined; + if (toClose) { + toClose.closeAllConnections(); + await new Promise((resolve) => toClose.close(resolve)); + } + }); - afterEach(async () => { - const toClose = server; - server = undefined; - if (toClose) { - toClose.closeAllConnections(); - await new Promise((resolve) => toClose.close(resolve)); - } - }); + it('sends traceparent on a request that never touches undici, parented to the client span', async () => { + const schema = z.object({ value: z.string() }); + let sentTraceparent: string | undefined; - it('sends traceparent on a request that never touches undici, parented to the client span', async () => { - const schema = z.object({ value: z.string() }); - let sentTraceparent: string | undefined; + server = createServer((request, response) => { + sentTraceparent = request.headers.traceparent as string | undefined; + request.resume(); + response.setHeader('content-type', 'application/cbor'); + response.end(encode({ value: 'ok' })); + }); + await new Promise((resolve) => + server?.listen(0, '127.0.0.1', resolve) + ); + const { port } = server.address() as AddressInfo; + vi.stubEnv('VERCEL_WORKFLOW_SERVER_URL', `http://127.0.0.1:${port}`); + + const tracer = otelTrace.getTracer('test'); + let traceId = ''; + let spanId = ''; + await tracer.startActiveSpan('flow-invocation', async (span) => { + traceId = span.spanContext().traceId; + spanId = span.spanContext().spanId; + const result = await makeRequest({ + endpoint: '/v3/runs/wrun_test/events', + options: { method: 'GET' }, + schema, + }); + expect(result).toEqual({ value: 'ok' }); + span.end(); + }); - server = createServer((request, response) => { - sentTraceparent = request.headers.traceparent as string | undefined; - request.resume(); - response.setHeader('content-type', 'application/cbor'); - response.end(encode({ value: 'ok' })); + expect(sentTraceparent).toMatch(/^00-[0-9a-f]{32}-[0-9a-f]{16}-0[01]$/); + const clientSpan = exporter + .getFinishedSpans() + .find((s) => s.name === 'http GET'); + expect(clientSpan?.spanContext().traceId).toBe(traceId); + expect(clientSpan?.parentSpanId).toBe(spanId); + expect(sentTraceparent).toBe( + `00-${traceId}-${clientSpan?.spanContext().spanId}-01` + ); + // Both transports emit `http GET` against the same `url.full`, so this + // attribute is the only thing in a trace that names which one ran. + expect(clientSpan?.attributes['workflow.http.transport']).toBe( + 'node-http' + ); }); - await new Promise((resolve) => - server?.listen(0, '127.0.0.1', resolve) - ); - const { port } = server.address() as AddressInfo; - vi.stubEnv('VERCEL_WORKFLOW_SERVER_URL', `http://127.0.0.1:${port}`); - const tracer = otelTrace.getTracer('test'); - let traceId = ''; - let spanId = ''; - await tracer.startActiveSpan('flow-invocation', async (span) => { - traceId = span.spanContext().traceId; - spanId = span.spanContext().spanId; - const result = await makeRequest({ + it('marks the undici path with the same attribute', async () => { + vi.stubEnv(NODE_HTTP_ENV_VAR, '0'); + const schema = z.object({ value: z.string() }); + vi.stubGlobal( + 'fetch', + vi.fn(async () => cborResponse({ value: 'ok' })) + ); + + await makeRequest({ endpoint: '/v3/runs/wrun_test/events', options: { method: 'GET' }, schema, }); - expect(result).toEqual({ value: 'ok' }); - span.end(); - }); - expect(sentTraceparent).toMatch(/^00-[0-9a-f]{32}-[0-9a-f]{16}-0[01]$/); - const clientSpan = exporter - .getFinishedSpans() - .find((s) => s.name === 'http GET'); - expect(clientSpan?.spanContext().traceId).toBe(traceId); - expect(clientSpan?.parentSpanId).toBe(spanId); - expect(sentTraceparent).toBe( - `00-${traceId}-${clientSpan?.spanContext().spanId}-01` - ); - // Both transports emit `http GET` against the same `url.full`, so this - // attribute is the only thing in a trace that names which one ran. - expect(clientSpan?.attributes['workflow.http.transport']).toBe('node-http'); - }); - - it('marks the undici path with the same attribute', async () => { - vi.stubEnv(NODE_HTTP_ENV_VAR, '0'); - const schema = z.object({ value: z.string() }); - vi.stubGlobal( - 'fetch', - vi.fn(async () => cborResponse({ value: 'ok' })) - ); - - await makeRequest({ - endpoint: '/v3/runs/wrun_test/events', - options: { method: 'GET' }, - schema, + const clientSpan = exporter + .getFinishedSpans() + .find((s) => s.name === 'http GET'); + expect(clientSpan?.attributes['workflow.http.transport']).toBe('undici'); }); - - const clientSpan = exporter - .getFinishedSpans() - .find((s) => s.name === 'http GET'); - expect(clientSpan?.attributes['workflow.http.transport']).toBe('undici'); - }); -}); + } +); diff --git a/packages/world-vercel/src/utils.test.ts b/packages/world-vercel/src/utils.test.ts index 18273e3deb..4dec341201 100644 --- a/packages/world-vercel/src/utils.test.ts +++ b/packages/world-vercel/src/utils.test.ts @@ -722,98 +722,110 @@ describe('makeRequest transport errors', () => { // origin instead, covering the two contracts the runtime branches on: a // failed request has to stay retryable, and a typed error status has to keep // producing the same typed error whichever transport carried it. -describe('makeRequest over node:http', () => { - const schema = z.object({ value: z.string() }); - const originalEnv = process.env; - let server: Server | undefined; +// These run against a loopback origin, which they select through +// `VERCEL_WORKFLOW_SERVER_URL`. The inline `WORKFLOW_SERVER_URL_OVERRIDE` +// constant WINS over that env var by design, so while a branch-testing +// override is pinned there is no way for these to reach their own server and +// every request leaves the machine. Skipped in that case rather than left to +// fail confusingly; they run again the moment the override goes back to ''. +describe.skipIf(WORKFLOW_SERVER_URL_OVERRIDE !== '')( + 'makeRequest over node:http', + () => { + const schema = z.object({ value: z.string() }); + const originalEnv = process.env; + let server: Server | undefined; + + beforeEach(() => { + process.env = { ...originalEnv }; + delete process.env.VERCEL_OIDC_TOKEN; + process.env[NODE_HTTP_ENV_VAR] = '1'; + }); - beforeEach(() => { - process.env = { ...originalEnv }; - delete process.env.VERCEL_OIDC_TOKEN; - process.env[NODE_HTTP_ENV_VAR] = '1'; - }); + afterEach(async () => { + process.env = originalEnv; + const toClose = server; + server = undefined; + if (toClose) { + toClose.closeAllConnections(); + await new Promise((resolve) => toClose.close(resolve)); + } + }); - afterEach(async () => { - process.env = originalEnv; - const toClose = server; - server = undefined; - if (toClose) { - toClose.closeAllConnections(); - await new Promise((resolve) => toClose.close(resolve)); + /** Start a loopback origin and point the client at it. */ + async function listen(handler: RequestListener): Promise { + server = createServer(handler); + await new Promise((resolve) => + server?.listen(0, '127.0.0.1', resolve) + ); + const { port } = server.address() as AddressInfo; + process.env.VERCEL_WORKFLOW_SERVER_URL = `http://127.0.0.1:${port}`; } - }); - /** Start a loopback origin and point the client at it. */ - async function listen(handler: RequestListener): Promise { - server = createServer(handler); - await new Promise((resolve) => - server?.listen(0, '127.0.0.1', resolve) - ); - const { port } = server.address() as AddressInfo; - process.env.VERCEL_WORKFLOW_SERVER_URL = `http://127.0.0.1:${port}`; - } - - it('maps a dropped socket to a retryable TRANSPORT error', async () => { - await listen((request) => request.socket.destroy()); + it('maps a dropped socket to a retryable TRANSPORT error', async () => { + await listen((request) => request.socket.destroy()); - // Node raises ECONNRESET on the error itself rather than on a `cause`, so - // this only passes if getTransientTransportCode reads the top-level code. - await expect( - makeRequest({ - endpoint: '/v3/runs/wrun_test/events', - options: { method: 'GET' }, - schema, - }) - ).rejects.toMatchObject({ name: 'WorkflowWorldError', code: 'TRANSPORT' }); - }); + // Node raises ECONNRESET on the error itself rather than on a `cause`, so + // this only passes if getTransientTransportCode reads the top-level code. + await expect( + makeRequest({ + endpoint: '/v3/runs/wrun_test/events', + options: { method: 'GET' }, + schema, + }) + ).rejects.toMatchObject({ + name: 'WorkflowWorldError', + code: 'TRANSPORT', + }); + }); - it('maps a 412 response to PreconditionFailedError', async () => { - await listen((request, response) => { - request.resume(); - response.statusCode = 412; - response.setHeader('content-type', 'application/cbor'); - response.end( - encode({ - success: false, - error: 'precondition-failed', - code: 'precondition-failed', - message: 'precondition-failed', + it('maps a 412 response to PreconditionFailedError', async () => { + await listen((request, response) => { + request.resume(); + response.statusCode = 412; + response.setHeader('content-type', 'application/cbor'); + response.end( + encode({ + success: false, + error: 'precondition-failed', + code: 'precondition-failed', + message: 'precondition-failed', + }) + ); + }); + + await expect( + makeRequest({ + endpoint: '/v3/runs/wrun_test/events', + options: { method: 'POST' }, + data: { eventType: 'run_completed' }, + schema, }) - ); + ).rejects.toBeInstanceOf(PreconditionFailedError); }); - await expect( - makeRequest({ + it('round-trips a CBOR POST body to the origin', async () => { + let seen: { method?: string; length?: string } = {}; + await listen((request, response) => { + seen = { + method: request.method, + length: request.headers['content-length'], + }; + request.resume(); + response.setHeader('content-type', 'application/cbor'); + response.end(encode({ value: 'ok' })); + }); + + const result = await makeRequest({ endpoint: '/v3/runs/wrun_test/events', options: { method: 'POST' }, data: { eventType: 'run_completed' }, schema, - }) - ).rejects.toBeInstanceOf(PreconditionFailedError); - }); + }); - it('round-trips a CBOR POST body to the origin', async () => { - let seen: { method?: string; length?: string } = {}; - await listen((request, response) => { - seen = { - method: request.method, - length: request.headers['content-length'], - }; - request.resume(); - response.setHeader('content-type', 'application/cbor'); - response.end(encode({ value: 'ok' })); + expect(result).toEqual({ value: 'ok' }); + expect(seen.method).toBe('POST'); + // A declared length, not a chunked body: some origins reject the latter. + expect(Number(seen.length)).toBeGreaterThan(0); }); - - const result = await makeRequest({ - endpoint: '/v3/runs/wrun_test/events', - options: { method: 'POST' }, - data: { eventType: 'run_completed' }, - schema, - }); - - expect(result).toEqual({ value: 'ok' }); - expect(seen.method).toBe('POST'); - // A declared length, not a chunked body: some origins reject the latter. - expect(Number(seen.length)).toBeGreaterThan(0); - }); -}); + } +); diff --git a/packages/world-vercel/src/utils.ts b/packages/world-vercel/src/utils.ts index 33450c6fee..5b7767d665 100644 --- a/packages/world-vercel/src/utils.ts +++ b/packages/world-vercel/src/utils.ts @@ -38,8 +38,18 @@ import { version } from './version.js'; * Inline workflow-server URL override. Must remain an empty string on * `main` — rewritten by external CI for branch-deployment testing. * Prefer `VERCEL_WORKFLOW_SERVER_URL` for deployment-time configuration. + * + * TODO(sealed-log): REVERT TO '' BEFORE MERGE. Pinned to the sealed-log + * sequencer's server branch (vercel/workflow-server#805) so this PR's e2e + * suite exercises spec-7 position assignment and `noop` sealing against the + * server that implements them, rather than against a server that treats spec 7 + * as plain slot identity. This constant wins over + * `VERCEL_WORKFLOW_SERVER_URL`, so it points both the deployed workbench apps + * and the e2e harness at the same server — which is the point: the harness has + * to read run state from the server the app wrote it to. */ -export const WORKFLOW_SERVER_URL_OVERRIDE = ''; +export const WORKFLOW_SERVER_URL_OVERRIDE = + 'https://workflow-server-git-pgp-sealed-log-sequencer.vercel.sh'; /** * HTTP methods that are safe to transparently re-issue inside the adapter.