From ecc0d3024497b3708d52eebbbcfb33a7490d1c57 Mon Sep 17 00:00:00 2001 From: shalabhc Date: Fri, 17 Jul 2026 08:53:21 -0700 Subject: [PATCH 01/15] Implement MAX_EVENTS_PER_RUN limit --- packages/core/src/describe-error.test.ts | 19 ++++ packages/core/src/describe-error.ts | 13 +++ packages/core/src/runtime.ts | 19 ++++ packages/core/src/runtime/constants.test.ts | 36 +++++++ packages/core/src/runtime/constants.ts | 18 ++++ packages/core/src/runtime/event-limit.test.ts | 101 ++++++++++++++++++ packages/core/src/runtime/event-limit.ts | 58 ++++++++++ packages/errors/src/error-codes.ts | 2 + 8 files changed, 266 insertions(+) create mode 100644 packages/core/src/runtime/event-limit.test.ts create mode 100644 packages/core/src/runtime/event-limit.ts diff --git a/packages/core/src/describe-error.test.ts b/packages/core/src/describe-error.test.ts index 6dac492d01..44efd6740f 100644 --- a/packages/core/src/describe-error.test.ts +++ b/packages/core/src/describe-error.test.ts @@ -108,6 +108,16 @@ describe('describeError', () => { expect(result.hint).toContain('SDK contract'); }); + test('MAX_EVENTS_EXCEEDED via precomputed errorCode is attributed to the user', () => { + const result = describeError( + undefined, + RUN_ERROR_CODES.MAX_EVENTS_EXCEEDED + ); + expect(result.attribution).toBe('user'); + expect(result.errorCode).toBe(RUN_ERROR_CODES.MAX_EVENTS_EXCEEDED); + expect(result.hint).toContain('maximum number of events'); + }); + test('precomputed errorCode wins over classifyRunError when both are provided', () => { // A plain Error would classify as USER_ERROR, but passing REPLAY_TIMEOUT // explicitly overrides that — useful for callers that know the failure @@ -193,6 +203,15 @@ describe('describeRunError', () => { expect(result.hint).toContain('max-delivery budget'); }); + test('MAX_EVENTS_EXCEEDED errorCode is attributed to the user', () => { + const result = describeRunError({ + errorCode: RUN_ERROR_CODES.MAX_EVENTS_EXCEEDED, + }); + expect(result.attribution).toBe('user'); + expect(result.errorCode).toBe(RUN_ERROR_CODES.MAX_EVENTS_EXCEEDED); + expect(result.hint).toContain('maximum number of events'); + }); + test('WORLD_CONTRACT_ERROR errorCode is attributed to the SDK', () => { const result = describeRunError({ errorCode: RUN_ERROR_CODES.WORLD_CONTRACT_ERROR, diff --git a/packages/core/src/describe-error.ts b/packages/core/src/describe-error.ts index 74a4b5a9f1..b939e62a7e 100644 --- a/packages/core/src/describe-error.ts +++ b/packages/core/src/describe-error.ts @@ -83,6 +83,8 @@ const REPLAY_TIMEOUT_HINT = 'The workflow replay between step boundaries took too long. This bounds workflow-VM and event-log replay time only — step bodies (`"use step"` functions) are excluded. This usually means the event log is unusually large or the workflow function is doing heavy synchronous work in workflow code outside of step bodies. Override the default budget via the WORKFLOW_REPLAY_TIMEOUT_MS env var if needed.'; const MAX_DELIVERIES_HINT = 'The workflow queue exceeded its max-delivery budget. This usually indicates a persistent runtime failure — check the most recent stack traces for the underlying cause.'; +const MAX_EVENTS_HINT = + 'The workflow exceeded the maximum number of events per run. This usually means unbounded work in the workflow function — e.g. a loop that keeps creating steps without terminating. Break long-running workflows into child workflows to stay under the limit.'; const WORLD_CONTRACT_HINT = 'The workflow backend returned data that violated the SDK contract. This is not retryable; please report it with the stack trace and runId.'; @@ -130,6 +132,9 @@ export function describeRunError( if (errorCode === RUN_ERROR_CODES.MAX_DELIVERIES_EXCEEDED) { return { attribution: 'sdk', errorCode, hint: MAX_DELIVERIES_HINT }; } + if (errorCode === RUN_ERROR_CODES.MAX_EVENTS_EXCEEDED) { + return { attribution: 'user', errorCode, hint: MAX_EVENTS_HINT }; + } if (errorCode === RUN_ERROR_CODES.CORRUPTED_EVENT_LOG) { return { attribution: 'sdk', @@ -228,6 +233,14 @@ export function describeError( }; } + if (effectiveCode === RUN_ERROR_CODES.MAX_EVENTS_EXCEEDED) { + return { + attribution: 'user', + errorCode: effectiveCode, + hint: MAX_EVENTS_HINT, + }; + } + if (effectiveCode === RUN_ERROR_CODES.CORRUPTED_EVENT_LOG) { return { attribution: 'sdk', diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 0c455a50a3..8c7168887a 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -37,11 +37,14 @@ import { type StepInvocationQueueItem, WorkflowSuspension } from './global.js'; import { runtimeLogger } from './logger.js'; import { ReplayPayloadCache } from './replay-payload-cache.js'; import { + getMaxEventsPerRun, getMaxQueueDeliveries, getReplayDivergenceMaxRetries, isInlineOwnershipEnabled, isTurboEnabled, + MAX_EVENTS_BUFFER, } from './runtime/constants.js'; +import { handleEventLimitExceeded } from './runtime/event-limit.js'; import { getQueueOverhead, getWorkflowQueueName, @@ -1422,6 +1425,22 @@ export function workflowEntrypoint( return; } + // Event-limit guard: fail a runaway run once its log + // grows past the quota (+ buffer). Checked after the log + // is loaded and before replay schedules more work. + const maxEvents = + getMaxEventsPerRun() + MAX_EVENTS_BUFFER; + if (events.length >= maxEvents) { + await handleEventLimitExceeded({ + runId, + workflowName, + requestId, + eventCount: events.length, + limit: getMaxEventsPerRun(), + }); + return; + } + // Update cache reference (may have been set for first time) cachedEvents = events; diff --git a/packages/core/src/runtime/constants.test.ts b/packages/core/src/runtime/constants.test.ts index 208250e8e6..f665fbb925 100644 --- a/packages/core/src/runtime/constants.test.ts +++ b/packages/core/src/runtime/constants.test.ts @@ -3,6 +3,7 @@ import { runtimeLogger } from '../logger.js'; import { _resetReplayTimeoutWarnCacheForTests, getInlineOwnershipLeaseSeconds, + getMaxEventsPerRun, getMaxInlineSteps, getMaxQueueDeliveries, getReplayTimeoutMs, @@ -11,6 +12,7 @@ import { isOptimisticInlineStartEnabled, isOptimisticInlineStartExplicitlyDisabled, isTurboEnabled, + MAX_EVENTS_PER_RUN, MAX_INLINE_OWNERSHIP_LEASE_SECONDS, MAX_INLINE_STEPS, MAX_MAX_INLINE_STEPS, @@ -393,3 +395,37 @@ describe('getInlineOwnershipLeaseSeconds', () => { expect(getInlineOwnershipLeaseSeconds()).toBe(1); }); }); + +describe('getMaxEventsPerRun', () => { + const ENV = 'WORKFLOW_MAX_EVENTS'; + const original = process.env[ENV]; + + beforeEach(() => { + delete process.env[ENV]; + }); + + afterEach(() => { + if (original === undefined) delete process.env[ENV]; + else process.env[ENV] = original; + }); + + it('defaults to the published per-run event limit (25,000)', () => { + expect(getMaxEventsPerRun()).toBe(25_000); + expect(MAX_EVENTS_PER_RUN).toBe(25_000); + }); + + it('returns the default when the env var is empty', () => { + process.env[ENV] = ''; + expect(getMaxEventsPerRun()).toBe(MAX_EVENTS_PER_RUN); + }); + + it('allows a lower override (used by tests to exercise the limit)', () => { + process.env[ENV] = '5'; + expect(getMaxEventsPerRun()).toBe(5); + }); + + it('falls back to the default on a non-numeric override', () => { + process.env[ENV] = 'not-a-number'; + expect(getMaxEventsPerRun()).toBe(MAX_EVENTS_PER_RUN); + }); +}); diff --git a/packages/core/src/runtime/constants.ts b/packages/core/src/runtime/constants.ts index 8db7a3f723..e0b7d85369 100644 --- a/packages/core/src/runtime/constants.ts +++ b/packages/core/src/runtime/constants.ts @@ -37,6 +37,24 @@ export function getMaxQueueDeliveries(): number { }); } +/** + * Published "Events per run" limit (https://vercel.com/docs/workflows/pricing). + * Past this (+ MAX_EVENTS_BUFFER) the runtime fails the run with + * RUN_ERROR_CODES.MAX_EVENTS_EXCEEDED — a client-side guardrail for runaway + * workflow code, not a security boundary. + */ +export const MAX_EVENTS_PER_RUN = 25_000; + +export const MAX_EVENTS_BUFFER = 100; + +/** Effective max events per run. Override via `WORKFLOW_MAX_EVENTS` (tests). */ +export function getMaxEventsPerRun(): number { + return envNumber('WORKFLOW_MAX_EVENTS', MAX_EVENTS_PER_RUN, { + integer: true, + min: 1, + }); +} + /** * Default maximum time allowed for the *replay* portion of a single workflow * handler invocation (in ms). This budget only covers deterministic-replay diff --git a/packages/core/src/runtime/event-limit.test.ts b/packages/core/src/runtime/event-limit.test.ts new file mode 100644 index 0000000000..23234dcc7c --- /dev/null +++ b/packages/core/src/runtime/event-limit.test.ts @@ -0,0 +1,101 @@ +import type { World } from '@workflow/world'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { runtimeLogger } from '../logger.js'; +import { handleEventLimitExceeded } from './event-limit.js'; +import { getWorld } from './world.js'; + +vi.mock('./world.js', () => ({ + getWorld: vi.fn(), +})); + +vi.mock('../serialization.js', () => ({ + dehydrateRunError: vi.fn(async () => new Uint8Array([1, 2, 3])), +})); + +vi.mock('./helpers.js', () => ({ + memoizeEncryptionKey: () => async () => undefined, +})); + +describe('handleEventLimitExceeded', () => { + let mockEventsCreate: ReturnType; + + function makeMockWorld(): World { + return { + events: { create: mockEventsCreate }, + } as unknown as World; + } + + beforeEach(() => { + mockEventsCreate = vi.fn().mockResolvedValue({}); + + // Silence the run-scoped logger; tests don't introspect its calls. + const noopLogger = { + warn: vi.fn(), + error: vi.fn(), + info: vi.fn(), + debug: vi.fn(), + forRun: vi.fn(), + child: vi.fn(), + }; + noopLogger.forRun.mockReturnValue(noopLogger); + noopLogger.child.mockReturnValue(noopLogger); + vi.spyOn(runtimeLogger, 'forRun').mockReturnValue(noopLogger as never); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('writes a terminal run_failed with MAX_EVENTS_EXCEEDED', async () => { + vi.mocked(getWorld).mockResolvedValue(makeMockWorld()); + + await handleEventLimitExceeded({ + runId: 'wrun_test', + workflowName: 'wf', + requestId: 'req_test', + eventCount: 25_101, + limit: 25_000, + }); + + expect(mockEventsCreate).toHaveBeenCalledTimes(1); + const [runId, event, opts] = mockEventsCreate.mock.calls[0]; + expect(runId).toBe('wrun_test'); + expect(event.eventType).toBe('run_failed'); + expect(event.eventData.errorCode).toBe('MAX_EVENTS_EXCEEDED'); + expect(opts).toEqual({ requestId: 'req_test' }); + }); + + it('never calls process.exit (deterministic failure, no redelivery)', async () => { + vi.mocked(getWorld).mockResolvedValue(makeMockWorld()); + const exitSpy = vi.spyOn(process, 'exit').mockImplementation((( + code?: number + ) => { + throw new Error(`__test_process_exit__:${code}`); + }) as never); + + await handleEventLimitExceeded({ + runId: 'wrun_test', + workflowName: 'wf', + requestId: undefined, + eventCount: 25_101, + limit: 25_000, + }); + + expect(exitSpy).not.toHaveBeenCalled(); + }); + + it('does not throw when the run_failed write fails (best-effort)', async () => { + mockEventsCreate.mockRejectedValueOnce(new Error('backend down')); + vi.mocked(getWorld).mockResolvedValue(makeMockWorld()); + + await expect( + handleEventLimitExceeded({ + runId: 'wrun_test', + workflowName: 'wf', + requestId: undefined, + eventCount: 25_101, + limit: 25_000, + }) + ).resolves.toBeUndefined(); + }); +}); diff --git a/packages/core/src/runtime/event-limit.ts b/packages/core/src/runtime/event-limit.ts new file mode 100644 index 0000000000..4a7774b119 --- /dev/null +++ b/packages/core/src/runtime/event-limit.ts @@ -0,0 +1,58 @@ +import { FatalError, RUN_ERROR_CODES } from '@workflow/errors'; +import { SPEC_VERSION_CURRENT } from '@workflow/world'; +import { runtimeLogger } from '../logger.js'; +import { dehydrateRunError } from '../serialization.js'; +import { memoizeEncryptionKey } from './helpers.js'; +import { getWorld } from './world.js'; + +/** + * Fail a run whose event log has exceeded the max event count. Deterministic + * (a redelivery would replay the same over-limit log), so unlike + * `handleReplayBudgetExhausted` there's no retry/`process.exit` — just write a + * best-effort terminal `run_failed` with MAX_EVENTS_EXCEEDED and return. + */ +export async function handleEventLimitExceeded(args: { + runId: string; + workflowName: string; + requestId: string | undefined; + eventCount: number; + limit: number; +}): Promise { + const { runId, workflowName, requestId, eventCount, limit } = args; + const runLogger = runtimeLogger.forRun(runId, workflowName); + + runLogger.error( + 'Workflow exceeded the maximum number of events; failing the run', + { eventCount, limit } + ); + + try { + const world = await getWorld(); + const getEncryptionKey = memoizeEncryptionKey(world, runId); + const limitErr = new FatalError( + `Workflow exceeded the maximum of ${limit} events per run` + ); + await world.events.create( + runId, + { + eventType: 'run_failed', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + error: await dehydrateRunError( + limitErr, + runId, + await getEncryptionKey() + ), + errorCode: RUN_ERROR_CODES.MAX_EVENTS_EXCEEDED, + }, + }, + { requestId } + ); + } catch (err) { + // Best-effort: the run stops anyway when the loop returns; log why. + runLogger.warn('Unable to mark run as failed after exceeding event limit', { + errorName: err instanceof Error ? err.name : 'UnknownError', + errorMessage: err instanceof Error ? err.message : String(err), + }); + } +} diff --git a/packages/errors/src/error-codes.ts b/packages/errors/src/error-codes.ts index 656f479496..3a68d79590 100644 --- a/packages/errors/src/error-codes.ts +++ b/packages/errors/src/error-codes.ts @@ -14,6 +14,8 @@ export const RUN_ERROR_CODES = { REPLAY_DIVERGENCE: 'REPLAY_DIVERGENCE', /** Run exceeded the maximum number of queue deliveries */ MAX_DELIVERIES_EXCEEDED: 'MAX_DELIVERIES_EXCEEDED', + /** Run exceeded the maximum number of events per run */ + MAX_EVENTS_EXCEEDED: 'MAX_EVENTS_EXCEEDED', /** Workflow replay exceeded the maximum allowed duration */ REPLAY_TIMEOUT: 'REPLAY_TIMEOUT', /** World response violated the SDK contract and cannot be retried safely */ From 536ee94011dd27a1641497bfa1990aa417c43bcd Mon Sep 17 00:00:00 2001 From: shalabhc Date: Fri, 17 Jul 2026 11:54:09 -0700 Subject: [PATCH 02/15] add changeset --- .changeset/event-limit-enforcement.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changeset/event-limit-enforcement.md diff --git a/.changeset/event-limit-enforcement.md b/.changeset/event-limit-enforcement.md new file mode 100644 index 0000000000..808dd7f682 --- /dev/null +++ b/.changeset/event-limit-enforcement.md @@ -0,0 +1,6 @@ +--- +"@workflow/errors": patch +"@workflow/core": patch +--- + +Enforce 25K events per run limit From d60ad96b2cd67db038949b18987ea99464204497 Mon Sep 17 00:00:00 2001 From: shalabhc Date: Fri, 17 Jul 2026 15:47:23 -0700 Subject: [PATCH 03/15] Use server side value for limit --- packages/core/src/runtime.ts | 23 +++++++----- packages/core/src/runtime/constants.test.ts | 36 ------------------- packages/core/src/runtime/constants.ts | 18 ---------- packages/world-vercel/src/events-v4.ts | 5 +++ packages/world-vercel/src/events.ts | 5 +++ packages/world/src/events.ts | 2 ++ .../workflows/test_blob_violation.ts | 6 ++++ .../workflows/test_multiple_violations.ts | 10 ++++++ 8 files changed, 43 insertions(+), 62 deletions(-) create mode 100644 workbench/nextjs-turbopack/workflows/test_blob_violation.ts create mode 100644 workbench/nextjs-turbopack/workflows/test_multiple_violations.ts diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 8c7168887a..d654bd2d9c 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -37,12 +37,10 @@ import { type StepInvocationQueueItem, WorkflowSuspension } from './global.js'; import { runtimeLogger } from './logger.js'; import { ReplayPayloadCache } from './replay-payload-cache.js'; import { - getMaxEventsPerRun, getMaxQueueDeliveries, getReplayDivergenceMaxRetries, isInlineOwnershipEnabled, isTurboEnabled, - MAX_EVENTS_BUFFER, } from './runtime/constants.js'; import { handleEventLimitExceeded } from './runtime/event-limit.js'; import { @@ -554,6 +552,10 @@ export function workflowEntrypoint( // Shared state: set by either the background step path // or the run_started setup below. let workflowRun: WorkflowRun | undefined; + // Server-owned per-run event ceiling, read off the + // run_started response. Undefined ⇒ no enforcement (older + // servers, or the turbo path which backgrounds run_started). + let maxEventsLimit: number | undefined; let workflowStartedAt = -1; let preloadedEvents: Event[] | undefined; let preloadedEventsCursor: string | null | undefined; @@ -1039,6 +1041,7 @@ export function workflowEntrypoint( ); } workflowRun = result.run; + maxEventsLimit = result.maxEvents; // Anchors RSFS — see the declaration above. runStartedReceivedAtMs = Date.now(); @@ -1426,17 +1429,21 @@ export function workflowEntrypoint( } // Event-limit guard: fail a runaway run once its log - // grows past the quota (+ buffer). Checked after the log - // is loaded and before replay schedules more work. - const maxEvents = - getMaxEventsPerRun() + MAX_EVENTS_BUFFER; - if (events.length >= maxEvents) { + // reaches the server-owned ceiling returned on the + // run_started response. Undefined ⇒ no enforcement + // (older servers, or the turbo path which backgrounds + // run_started). Checked after the log is loaded and + // before replay schedules more work. + if ( + maxEventsLimit !== undefined && + events.length >= maxEventsLimit + ) { await handleEventLimitExceeded({ runId, workflowName, requestId, eventCount: events.length, - limit: getMaxEventsPerRun(), + limit: maxEventsLimit, }); return; } diff --git a/packages/core/src/runtime/constants.test.ts b/packages/core/src/runtime/constants.test.ts index f665fbb925..208250e8e6 100644 --- a/packages/core/src/runtime/constants.test.ts +++ b/packages/core/src/runtime/constants.test.ts @@ -3,7 +3,6 @@ import { runtimeLogger } from '../logger.js'; import { _resetReplayTimeoutWarnCacheForTests, getInlineOwnershipLeaseSeconds, - getMaxEventsPerRun, getMaxInlineSteps, getMaxQueueDeliveries, getReplayTimeoutMs, @@ -12,7 +11,6 @@ import { isOptimisticInlineStartEnabled, isOptimisticInlineStartExplicitlyDisabled, isTurboEnabled, - MAX_EVENTS_PER_RUN, MAX_INLINE_OWNERSHIP_LEASE_SECONDS, MAX_INLINE_STEPS, MAX_MAX_INLINE_STEPS, @@ -395,37 +393,3 @@ describe('getInlineOwnershipLeaseSeconds', () => { expect(getInlineOwnershipLeaseSeconds()).toBe(1); }); }); - -describe('getMaxEventsPerRun', () => { - const ENV = 'WORKFLOW_MAX_EVENTS'; - const original = process.env[ENV]; - - beforeEach(() => { - delete process.env[ENV]; - }); - - afterEach(() => { - if (original === undefined) delete process.env[ENV]; - else process.env[ENV] = original; - }); - - it('defaults to the published per-run event limit (25,000)', () => { - expect(getMaxEventsPerRun()).toBe(25_000); - expect(MAX_EVENTS_PER_RUN).toBe(25_000); - }); - - it('returns the default when the env var is empty', () => { - process.env[ENV] = ''; - expect(getMaxEventsPerRun()).toBe(MAX_EVENTS_PER_RUN); - }); - - it('allows a lower override (used by tests to exercise the limit)', () => { - process.env[ENV] = '5'; - expect(getMaxEventsPerRun()).toBe(5); - }); - - it('falls back to the default on a non-numeric override', () => { - process.env[ENV] = 'not-a-number'; - expect(getMaxEventsPerRun()).toBe(MAX_EVENTS_PER_RUN); - }); -}); diff --git a/packages/core/src/runtime/constants.ts b/packages/core/src/runtime/constants.ts index e0b7d85369..8db7a3f723 100644 --- a/packages/core/src/runtime/constants.ts +++ b/packages/core/src/runtime/constants.ts @@ -37,24 +37,6 @@ export function getMaxQueueDeliveries(): number { }); } -/** - * Published "Events per run" limit (https://vercel.com/docs/workflows/pricing). - * Past this (+ MAX_EVENTS_BUFFER) the runtime fails the run with - * RUN_ERROR_CODES.MAX_EVENTS_EXCEEDED — a client-side guardrail for runaway - * workflow code, not a security boundary. - */ -export const MAX_EVENTS_PER_RUN = 25_000; - -export const MAX_EVENTS_BUFFER = 100; - -/** Effective max events per run. Override via `WORKFLOW_MAX_EVENTS` (tests). */ -export function getMaxEventsPerRun(): number { - return envNumber('WORKFLOW_MAX_EVENTS', MAX_EVENTS_PER_RUN, { - integer: true, - min: 1, - }); -} - /** * Default maximum time allowed for the *replay* portion of a single workflow * handler invocation (in ms). This budget only covers deterministic-replay diff --git a/packages/world-vercel/src/events-v4.ts b/packages/world-vercel/src/events-v4.ts index e8e251d333..f5422c81d9 100644 --- a/packages/world-vercel/src/events-v4.ts +++ b/packages/world-vercel/src/events-v4.ts @@ -230,6 +230,11 @@ export interface CreateEventV4Result { * creator). Threaded into EventResult.stepCreated by the events adapter. */ stepCreated?: boolean; + /** + * Server-owned per-run event ceiling, returned on run-lifecycle responses. + * Absent from older servers. Threaded into EventResult.maxEvents. + */ + maxEvents?: number; }; } diff --git a/packages/world-vercel/src/events.ts b/packages/world-vercel/src/events.ts index a25f835fa5..1548a853b6 100644 --- a/packages/world-vercel/src/events.ts +++ b/packages/world-vercel/src/events.ts @@ -761,5 +761,10 @@ async function createWorkflowRunEventInner( // signal through so the owned-inline runtime path can gate body execution // on it. Absent from older servers → undefined → safe default. ...(body.stepCreated ? { stepCreated: true } : {}), + // Server-owned per-run event ceiling (run_started/run_created). Absent + // from older servers → undefined → no client-side enforcement. + ...(typeof body.maxEvents === 'number' + ? { maxEvents: body.maxEvents } + : {}), }; } diff --git a/packages/world/src/events.ts b/packages/world/src/events.ts index 4b06d2b4a8..75d5caa009 100644 --- a/packages/world/src/events.ts +++ b/packages/world/src/events.ts @@ -778,6 +778,8 @@ export interface EventResult { hook?: import('./hooks.js').Hook; /** The wait entity (for wait_created/wait_completed events) */ wait?: import('./waits.js').Wait; + /** Server-owned max event count for the run (run-lifecycle responses); the runtime enforces it. */ + maxEvents?: number; /** * Events with data resolved. Two producers populate this: * diff --git a/workbench/nextjs-turbopack/workflows/test_blob_violation.ts b/workbench/nextjs-turbopack/workflows/test_blob_violation.ts new file mode 100644 index 0000000000..9298a068b2 --- /dev/null +++ b/workbench/nextjs-turbopack/workflows/test_blob_violation.ts @@ -0,0 +1,6 @@ +import { getPlatform } from 'workflow-test-dual-entry-package'; + +export async function blobViolationWorkflow() { + 'use workflow'; + return getPlatform(); +} diff --git a/workbench/nextjs-turbopack/workflows/test_multiple_violations.ts b/workbench/nextjs-turbopack/workflows/test_multiple_violations.ts new file mode 100644 index 0000000000..50d8af4cf4 --- /dev/null +++ b/workbench/nextjs-turbopack/workflows/test_multiple_violations.ts @@ -0,0 +1,10 @@ +import { readFileSync } from 'fs'; +import { join } from 'path'; +import { createHash } from 'crypto'; + +export async function multipleViolationsWorkflow() { + 'use workflow'; + const content = readFileSync(join('/', 'package.json'), 'utf8'); + const hash = createHash('sha256').update(content).digest('hex'); + return hash; +} From f56341ccf4aab2f9f50bfae0b5d66e667ad2b2db Mon Sep 17 00:00:00 2001 From: shalabhc Date: Fri, 17 Jul 2026 16:04:20 -0700 Subject: [PATCH 04/15] remove accidental checking --- .../nextjs-turbopack/workflows/test_blob_violation.ts | 6 ------ .../workflows/test_multiple_violations.ts | 10 ---------- 2 files changed, 16 deletions(-) delete mode 100644 workbench/nextjs-turbopack/workflows/test_blob_violation.ts delete mode 100644 workbench/nextjs-turbopack/workflows/test_multiple_violations.ts diff --git a/workbench/nextjs-turbopack/workflows/test_blob_violation.ts b/workbench/nextjs-turbopack/workflows/test_blob_violation.ts deleted file mode 100644 index 9298a068b2..0000000000 --- a/workbench/nextjs-turbopack/workflows/test_blob_violation.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { getPlatform } from 'workflow-test-dual-entry-package'; - -export async function blobViolationWorkflow() { - 'use workflow'; - return getPlatform(); -} diff --git a/workbench/nextjs-turbopack/workflows/test_multiple_violations.ts b/workbench/nextjs-turbopack/workflows/test_multiple_violations.ts deleted file mode 100644 index 50d8af4cf4..0000000000 --- a/workbench/nextjs-turbopack/workflows/test_multiple_violations.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { readFileSync } from 'fs'; -import { join } from 'path'; -import { createHash } from 'crypto'; - -export async function multipleViolationsWorkflow() { - 'use workflow'; - const content = readFileSync(join('/', 'package.json'), 'utf8'); - const hash = createHash('sha256').update(content).digest('hex'); - return hash; -} From f063f9c87d77b903ccb72ef7721aa4efbd96d0ca Mon Sep 17 00:00:00 2001 From: shalabhc Date: Fri, 17 Jul 2026 16:09:19 -0700 Subject: [PATCH 05/15] update changeset --- .changeset/event-limit-enforcement.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.changeset/event-limit-enforcement.md b/.changeset/event-limit-enforcement.md index 808dd7f682..8a99bcbb98 100644 --- a/.changeset/event-limit-enforcement.md +++ b/.changeset/event-limit-enforcement.md @@ -1,6 +1,8 @@ --- "@workflow/errors": patch "@workflow/core": patch +"@workflow/world": patch +"@workflow/world-vercel": patch --- -Enforce 25K events per run limit +Enforce a server-supplied per-run event limit (default 25K) From ec9fd74195d4d4ed3776f8691b4773e4c0cff1d7 Mon Sep 17 00:00:00 2001 From: shalabhc Date: Mon, 20 Jul 2026 10:26:15 -0700 Subject: [PATCH 06/15] Add limit check to world-local --- .../world-local/src/storage/events-storage.ts | 19 ++++++++++ packages/world-testing/src/event-limit.mts | 38 +++++++++++++++++++ packages/world-testing/src/index.mts | 2 + .../world-testing/workflows/event-limit.ts | 20 ++++++++++ 4 files changed, 79 insertions(+) create mode 100644 packages/world-testing/src/event-limit.mts create mode 100644 packages/world-testing/workflows/event-limit.ts diff --git a/packages/world-local/src/storage/events-storage.ts b/packages/world-local/src/storage/events-storage.ts index 7b3a8ddcc2..dee6cc7f76 100644 --- a/packages/world-local/src/storage/events-storage.ts +++ b/packages/world-local/src/storage/events-storage.ts @@ -80,6 +80,21 @@ import { import { handleLegacyEvent } from './legacy.js'; import { withRunFileLock } from './runs-storage.js'; +/** + * Server-owned per-run event ceiling the Local World reports on run responses, + * mirroring the Vercel World's `maxEvents` so the SDK runtime can enforce the + * limit in-process (no server involved). Overridable via `WORKFLOW_MAX_EVENTS` + * (used by tests to exercise the limit); defaults to the published 25,000. + */ +const DEFAULT_MAX_EVENTS_PER_RUN = 25_000; +function getMaxEventsPerRun(): number { + const raw = process.env.WORKFLOW_MAX_EVENTS; + const parsed = raw !== undefined ? Number(raw) : Number.NaN; + return Number.isInteger(parsed) && parsed > 0 + ? parsed + : DEFAULT_MAX_EVENTS_PER_RUN; +} + /** * Per-step in-process async mutex. Serializes concurrent `events.create` calls * that target the same step, so that the "check terminal state, then write step @@ -883,6 +898,7 @@ export function createEventsStorage( return { event: stripEventDataRefs(event, resolveData), run: currentRun, + ...(currentRun ? { maxEvents: getMaxEventsPerRun() } : {}), }; } @@ -2292,6 +2308,9 @@ export function createEventsStorage( cursor, hasMore, ...(stepCreatedLazily ? { stepCreated: true } : {}), + // Server-owned per-run event ceiling (mirrors the Vercel World) so + // the runtime enforces the limit under the Local World too. + ...(run ? { maxEvents: getMaxEventsPerRun() } : {}), }; } // end createImpl }, diff --git a/packages/world-testing/src/event-limit.mts b/packages/world-testing/src/event-limit.mts new file mode 100644 index 0000000000..61be7c3f54 --- /dev/null +++ b/packages/world-testing/src/event-limit.mts @@ -0,0 +1,38 @@ +import { expect, test, vi } from 'vitest'; +import { createFetcher, startServer } from './util.mjs'; + +export function eventLimit(world: string) { + test( + 'fails a runaway run at the server-supplied event limit', + { timeout: 59_000 }, + async () => { + // Low ceiling so a short runaway trips it; turbo off because turbo + // backgrounds run_started and the guard reads `maxEvents` off the + // run_started response (so enforcement is on the non-turbo path). + const server = await startServer({ + world, + env: { WORKFLOW_MAX_EVENTS: '10', WORKFLOW_TURBO: '0' }, + }).then(createFetcher); + + const result = await server.invoke( + 'workflows/event-limit.ts', + 'runawayWorkflow', + [] + ); + expect(result.runId).toMatch(/^wrun_.+/); + + const run = await vi.waitFor( + async () => { + const run = await server.getRun(result.runId); + expect(run.status).toBe('failed'); + return run; + }, + { + interval: 200, + timeout: 50_000, + } + ); + expect(run.errorCode).toBe('MAX_EVENTS_EXCEEDED'); + } + ); +} diff --git a/packages/world-testing/src/index.mts b/packages/world-testing/src/index.mts index 04637f882f..67a7ff983b 100644 --- a/packages/world-testing/src/index.mts +++ b/packages/world-testing/src/index.mts @@ -1,5 +1,6 @@ import { addition } from './addition.mjs'; import { errors } from './errors.mjs'; +import { eventLimit } from './event-limit.mjs'; import { hooks } from './hooks.mjs'; import { idempotency } from './idempotency.mjs'; import { inlineExecution } from './inline-execution.mjs'; @@ -14,4 +15,5 @@ export function createTestSuite(pkgName: string) { errors(pkgName); inlineExecution(pkgName); lineage(pkgName); + eventLimit(pkgName); } diff --git a/packages/world-testing/workflows/event-limit.ts b/packages/world-testing/workflows/event-limit.ts new file mode 100644 index 0000000000..1796382df0 --- /dev/null +++ b/packages/world-testing/workflows/event-limit.ts @@ -0,0 +1,20 @@ +async function tick(i: number): Promise { + 'use step'; + return i; +} + +/** + * Runaway workflow: creates far more events than a low `WORKFLOW_MAX_EVENTS` + * ceiling allows. With the limit set low (and turbo disabled so the runtime + * reads `maxEvents` off the run_started response), the event-limit guard fails + * the run with MAX_EVENTS_EXCEEDED before the loop finishes — so the 100 + * iterations never all run. + */ +export async function runawayWorkflow(): Promise { + 'use workflow'; + let total = 0; + for (let i = 0; i < 100; i++) { + total += await tick(i); + } + return total; +} From 848bd36d17a6c2bed24bf082dfcf792f86f45713 Mon Sep 17 00:00:00 2001 From: shalabhc Date: Mon, 20 Jul 2026 10:48:33 -0700 Subject: [PATCH 07/15] address comments --- packages/core/src/classify-error.test.ts | 7 ++ packages/core/src/classify-error.ts | 5 + packages/core/src/runtime.ts | 19 ++-- packages/core/src/runtime/event-limit.test.ts | 101 ------------------ packages/core/src/runtime/event-limit.ts | 58 ---------- packages/errors/src/index.ts | 26 +++++ 6 files changed, 47 insertions(+), 169 deletions(-) delete mode 100644 packages/core/src/runtime/event-limit.test.ts delete mode 100644 packages/core/src/runtime/event-limit.ts diff --git a/packages/core/src/classify-error.test.ts b/packages/core/src/classify-error.test.ts index a2f202c845..2719063a6d 100644 --- a/packages/core/src/classify-error.test.ts +++ b/packages/core/src/classify-error.test.ts @@ -1,6 +1,7 @@ import { CorruptedEventLogError, HookConflictError, + MaxEventsExceededError, ReplayDivergenceError, RUN_ERROR_CODES, RuntimeDecryptionError, @@ -20,6 +21,12 @@ describe('classifyRunError', () => { ).toBe(RUN_ERROR_CODES.CORRUPTED_EVENT_LOG); }); + it('classifies MaxEventsExceededError as MAX_EVENTS_EXCEEDED', () => { + expect(classifyRunError(new MaxEventsExceededError(25_100, 25_000))).toBe( + RUN_ERROR_CODES.MAX_EVENTS_EXCEEDED + ); + }); + it('classifies ReplayDivergenceError as REPLAY_DIVERGENCE', () => { expect( classifyRunError( diff --git a/packages/core/src/classify-error.ts b/packages/core/src/classify-error.ts index cc21416d02..51d6868153 100644 --- a/packages/core/src/classify-error.ts +++ b/packages/core/src/classify-error.ts @@ -1,5 +1,6 @@ import { CorruptedEventLogError, + MaxEventsExceededError, ReplayDivergenceError, RUN_ERROR_CODES, type RunErrorCode, @@ -116,6 +117,10 @@ export function classifyRunError(err: unknown): RunErrorCode { return RUN_ERROR_CODES.CORRUPTED_EVENT_LOG; } + if (MaxEventsExceededError.is(err)) { + return RUN_ERROR_CODES.MAX_EVENTS_EXCEEDED; + } + // World-layer faults — both a malformed response (contract violation) and a // transient infrastructure failure (throttle / 5xx / transport / timeout, // e.g. a firewall challenge) — are the backend's fault, not the user's. diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index d654bd2d9c..c5c32fa125 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -3,6 +3,7 @@ import { CorruptedEventLogError, EntityConflictError, FatalError, + MaxEventsExceededError, PreconditionFailedError, ReplayDivergenceError, RUN_ERROR_CODES, @@ -42,7 +43,6 @@ import { isInlineOwnershipEnabled, isTurboEnabled, } from './runtime/constants.js'; -import { handleEventLimitExceeded } from './runtime/event-limit.js'; import { getQueueOverhead, getWorkflowQueueName, @@ -1433,19 +1433,18 @@ export function workflowEntrypoint( // run_started response. Undefined ⇒ no enforcement // (older servers, or the turbo path which backgrounds // run_started). Checked after the log is loaded and - // before replay schedules more work. + // before replay schedules more work. Throwing lets the + // loop's terminal catch write run_failed (classified as + // MAX_EVENTS_EXCEEDED); the failure is deterministic so + // it is never retried/redelivered. if ( maxEventsLimit !== undefined && events.length >= maxEventsLimit ) { - await handleEventLimitExceeded({ - runId, - workflowName, - requestId, - eventCount: events.length, - limit: maxEventsLimit, - }); - return; + throw new MaxEventsExceededError( + events.length, + maxEventsLimit + ); } // Update cache reference (may have been set for first time) diff --git a/packages/core/src/runtime/event-limit.test.ts b/packages/core/src/runtime/event-limit.test.ts deleted file mode 100644 index 23234dcc7c..0000000000 --- a/packages/core/src/runtime/event-limit.test.ts +++ /dev/null @@ -1,101 +0,0 @@ -import type { World } from '@workflow/world'; -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { runtimeLogger } from '../logger.js'; -import { handleEventLimitExceeded } from './event-limit.js'; -import { getWorld } from './world.js'; - -vi.mock('./world.js', () => ({ - getWorld: vi.fn(), -})); - -vi.mock('../serialization.js', () => ({ - dehydrateRunError: vi.fn(async () => new Uint8Array([1, 2, 3])), -})); - -vi.mock('./helpers.js', () => ({ - memoizeEncryptionKey: () => async () => undefined, -})); - -describe('handleEventLimitExceeded', () => { - let mockEventsCreate: ReturnType; - - function makeMockWorld(): World { - return { - events: { create: mockEventsCreate }, - } as unknown as World; - } - - beforeEach(() => { - mockEventsCreate = vi.fn().mockResolvedValue({}); - - // Silence the run-scoped logger; tests don't introspect its calls. - const noopLogger = { - warn: vi.fn(), - error: vi.fn(), - info: vi.fn(), - debug: vi.fn(), - forRun: vi.fn(), - child: vi.fn(), - }; - noopLogger.forRun.mockReturnValue(noopLogger); - noopLogger.child.mockReturnValue(noopLogger); - vi.spyOn(runtimeLogger, 'forRun').mockReturnValue(noopLogger as never); - }); - - afterEach(() => { - vi.restoreAllMocks(); - }); - - it('writes a terminal run_failed with MAX_EVENTS_EXCEEDED', async () => { - vi.mocked(getWorld).mockResolvedValue(makeMockWorld()); - - await handleEventLimitExceeded({ - runId: 'wrun_test', - workflowName: 'wf', - requestId: 'req_test', - eventCount: 25_101, - limit: 25_000, - }); - - expect(mockEventsCreate).toHaveBeenCalledTimes(1); - const [runId, event, opts] = mockEventsCreate.mock.calls[0]; - expect(runId).toBe('wrun_test'); - expect(event.eventType).toBe('run_failed'); - expect(event.eventData.errorCode).toBe('MAX_EVENTS_EXCEEDED'); - expect(opts).toEqual({ requestId: 'req_test' }); - }); - - it('never calls process.exit (deterministic failure, no redelivery)', async () => { - vi.mocked(getWorld).mockResolvedValue(makeMockWorld()); - const exitSpy = vi.spyOn(process, 'exit').mockImplementation((( - code?: number - ) => { - throw new Error(`__test_process_exit__:${code}`); - }) as never); - - await handleEventLimitExceeded({ - runId: 'wrun_test', - workflowName: 'wf', - requestId: undefined, - eventCount: 25_101, - limit: 25_000, - }); - - expect(exitSpy).not.toHaveBeenCalled(); - }); - - it('does not throw when the run_failed write fails (best-effort)', async () => { - mockEventsCreate.mockRejectedValueOnce(new Error('backend down')); - vi.mocked(getWorld).mockResolvedValue(makeMockWorld()); - - await expect( - handleEventLimitExceeded({ - runId: 'wrun_test', - workflowName: 'wf', - requestId: undefined, - eventCount: 25_101, - limit: 25_000, - }) - ).resolves.toBeUndefined(); - }); -}); diff --git a/packages/core/src/runtime/event-limit.ts b/packages/core/src/runtime/event-limit.ts deleted file mode 100644 index 4a7774b119..0000000000 --- a/packages/core/src/runtime/event-limit.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { FatalError, RUN_ERROR_CODES } from '@workflow/errors'; -import { SPEC_VERSION_CURRENT } from '@workflow/world'; -import { runtimeLogger } from '../logger.js'; -import { dehydrateRunError } from '../serialization.js'; -import { memoizeEncryptionKey } from './helpers.js'; -import { getWorld } from './world.js'; - -/** - * Fail a run whose event log has exceeded the max event count. Deterministic - * (a redelivery would replay the same over-limit log), so unlike - * `handleReplayBudgetExhausted` there's no retry/`process.exit` — just write a - * best-effort terminal `run_failed` with MAX_EVENTS_EXCEEDED and return. - */ -export async function handleEventLimitExceeded(args: { - runId: string; - workflowName: string; - requestId: string | undefined; - eventCount: number; - limit: number; -}): Promise { - const { runId, workflowName, requestId, eventCount, limit } = args; - const runLogger = runtimeLogger.forRun(runId, workflowName); - - runLogger.error( - 'Workflow exceeded the maximum number of events; failing the run', - { eventCount, limit } - ); - - try { - const world = await getWorld(); - const getEncryptionKey = memoizeEncryptionKey(world, runId); - const limitErr = new FatalError( - `Workflow exceeded the maximum of ${limit} events per run` - ); - await world.events.create( - runId, - { - eventType: 'run_failed', - specVersion: SPEC_VERSION_CURRENT, - eventData: { - error: await dehydrateRunError( - limitErr, - runId, - await getEncryptionKey() - ), - errorCode: RUN_ERROR_CODES.MAX_EVENTS_EXCEEDED, - }, - }, - { requestId } - ); - } catch (err) { - // Best-effort: the run stops anyway when the loop returns; log why. - runLogger.warn('Unable to mark run as failed after exceeding event limit', { - errorName: err instanceof Error ? err.name : 'UnknownError', - errorMessage: err instanceof Error ? err.message : String(err), - }); - } -} diff --git a/packages/errors/src/index.ts b/packages/errors/src/index.ts index 3d7ab5a49f..3f72d27d7a 100644 --- a/packages/errors/src/index.ts +++ b/packages/errors/src/index.ts @@ -352,6 +352,32 @@ export class ReplayDivergenceError extends WorkflowRuntimeError { } } +/** + * Thrown by the runtime when a run's event log reaches the server-supplied + * per-run event ceiling. Recorded with the distinct `MAX_EVENTS_EXCEEDED` code + * (see `classifyRunError`) and attributed to the user's runaway workflow, + * rather than falling through to a generic `USER_ERROR`. + */ +export class MaxEventsExceededError extends WorkflowError { + readonly eventCount: number; + readonly limit: number; + + constructor( + eventCount: number, + limit: number, + options?: WorkflowErrorOptions + ) { + super(`Workflow exceeded the maximum of ${limit} events per run`, options); + this.name = 'MaxEventsExceededError'; + this.eventCount = eventCount; + this.limit = limit; + } + + static is(value: unknown): value is MaxEventsExceededError { + return isError(value) && value.name === 'MaxEventsExceededError'; + } +} + /** * Optional structured context attached to a {@link RuntimeDecryptionError}, * carried over from the underlying decrypt call site to help diagnose the From 51962310335e97fc54c03e6eea5b3757d43ebacf Mon Sep 17 00:00:00 2001 From: shalabhc Date: Mon, 20 Jul 2026 10:53:16 -0700 Subject: [PATCH 08/15] shrink code comments --- packages/core/src/runtime.ts | 16 +++++----------- packages/errors/src/index.ts | 6 ++---- .../world-local/src/storage/events-storage.ts | 9 +++------ packages/world-testing/src/event-limit.mts | 5 ++--- packages/world-testing/workflows/event-limit.ts | 5 +---- packages/world-vercel/src/events.ts | 3 +-- 6 files changed, 14 insertions(+), 30 deletions(-) diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index c5c32fa125..7e8204c26c 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -552,9 +552,8 @@ export function workflowEntrypoint( // Shared state: set by either the background step path // or the run_started setup below. let workflowRun: WorkflowRun | undefined; - // Server-owned per-run event ceiling, read off the - // run_started response. Undefined ⇒ no enforcement (older - // servers, or the turbo path which backgrounds run_started). + // Server-supplied per-run event ceiling from the run_started + // response. Undefined ⇒ no enforcement (older servers, turbo). let maxEventsLimit: number | undefined; let workflowStartedAt = -1; let preloadedEvents: Event[] | undefined; @@ -1429,14 +1428,9 @@ export function workflowEntrypoint( } // Event-limit guard: fail a runaway run once its log - // reaches the server-owned ceiling returned on the - // run_started response. Undefined ⇒ no enforcement - // (older servers, or the turbo path which backgrounds - // run_started). Checked after the log is loaded and - // before replay schedules more work. Throwing lets the - // loop's terminal catch write run_failed (classified as - // MAX_EVENTS_EXCEEDED); the failure is deterministic so - // it is never retried/redelivered. + // reaches the server-supplied ceiling (undefined ⇒ no + // enforcement). The throw is caught below and written as + // run_failed / MAX_EVENTS_EXCEEDED. if ( maxEventsLimit !== undefined && events.length >= maxEventsLimit diff --git a/packages/errors/src/index.ts b/packages/errors/src/index.ts index 3f72d27d7a..a69dbf7602 100644 --- a/packages/errors/src/index.ts +++ b/packages/errors/src/index.ts @@ -353,10 +353,8 @@ export class ReplayDivergenceError extends WorkflowRuntimeError { } /** - * Thrown by the runtime when a run's event log reaches the server-supplied - * per-run event ceiling. Recorded with the distinct `MAX_EVENTS_EXCEEDED` code - * (see `classifyRunError`) and attributed to the user's runaway workflow, - * rather than falling through to a generic `USER_ERROR`. + * Thrown when a run's event log reaches the server-supplied per-run event + * ceiling. Classified as `MAX_EVENTS_EXCEEDED` (see `classifyRunError`). */ export class MaxEventsExceededError extends WorkflowError { readonly eventCount: number; diff --git a/packages/world-local/src/storage/events-storage.ts b/packages/world-local/src/storage/events-storage.ts index dee6cc7f76..d55c01933f 100644 --- a/packages/world-local/src/storage/events-storage.ts +++ b/packages/world-local/src/storage/events-storage.ts @@ -81,10 +81,8 @@ import { handleLegacyEvent } from './legacy.js'; import { withRunFileLock } from './runs-storage.js'; /** - * Server-owned per-run event ceiling the Local World reports on run responses, - * mirroring the Vercel World's `maxEvents` so the SDK runtime can enforce the - * limit in-process (no server involved). Overridable via `WORKFLOW_MAX_EVENTS` - * (used by tests to exercise the limit); defaults to the published 25,000. + * Per-run event ceiling the Local World reports on run responses (mirrors the + * Vercel World). Overridable via `WORKFLOW_MAX_EVENTS`; defaults to 25,000. */ const DEFAULT_MAX_EVENTS_PER_RUN = 25_000; function getMaxEventsPerRun(): number { @@ -2308,8 +2306,7 @@ export function createEventsStorage( cursor, hasMore, ...(stepCreatedLazily ? { stepCreated: true } : {}), - // Server-owned per-run event ceiling (mirrors the Vercel World) so - // the runtime enforces the limit under the Local World too. + // Per-run event ceiling (mirrors the Vercel World). ...(run ? { maxEvents: getMaxEventsPerRun() } : {}), }; } // end createImpl diff --git a/packages/world-testing/src/event-limit.mts b/packages/world-testing/src/event-limit.mts index 61be7c3f54..d91db356b5 100644 --- a/packages/world-testing/src/event-limit.mts +++ b/packages/world-testing/src/event-limit.mts @@ -6,9 +6,8 @@ export function eventLimit(world: string) { 'fails a runaway run at the server-supplied event limit', { timeout: 59_000 }, async () => { - // Low ceiling so a short runaway trips it; turbo off because turbo - // backgrounds run_started and the guard reads `maxEvents` off the - // run_started response (so enforcement is on the non-turbo path). + // Low ceiling to trip it fast; turbo off because turbo backgrounds + // run_started (the guard reads maxEvents off that response). const server = await startServer({ world, env: { WORKFLOW_MAX_EVENTS: '10', WORKFLOW_TURBO: '0' }, diff --git a/packages/world-testing/workflows/event-limit.ts b/packages/world-testing/workflows/event-limit.ts index 1796382df0..dc20f9877a 100644 --- a/packages/world-testing/workflows/event-limit.ts +++ b/packages/world-testing/workflows/event-limit.ts @@ -5,10 +5,7 @@ async function tick(i: number): Promise { /** * Runaway workflow: creates far more events than a low `WORKFLOW_MAX_EVENTS` - * ceiling allows. With the limit set low (and turbo disabled so the runtime - * reads `maxEvents` off the run_started response), the event-limit guard fails - * the run with MAX_EVENTS_EXCEEDED before the loop finishes — so the 100 - * iterations never all run. + * ceiling, so the event-limit guard fails it before the loop finishes. */ export async function runawayWorkflow(): Promise { 'use workflow'; diff --git a/packages/world-vercel/src/events.ts b/packages/world-vercel/src/events.ts index 1548a853b6..706daa742d 100644 --- a/packages/world-vercel/src/events.ts +++ b/packages/world-vercel/src/events.ts @@ -761,8 +761,7 @@ async function createWorkflowRunEventInner( // signal through so the owned-inline runtime path can gate body execution // on it. Absent from older servers → undefined → safe default. ...(body.stepCreated ? { stepCreated: true } : {}), - // Server-owned per-run event ceiling (run_started/run_created). Absent - // from older servers → undefined → no client-side enforcement. + // Server-supplied per-run event ceiling; absent from older servers. ...(typeof body.maxEvents === 'number' ? { maxEvents: body.maxEvents } : {}), From 13b66f0f78ef7654dca1def8067feba8a9d40548 Mon Sep 17 00:00:00 2001 From: shalabhc Date: Mon, 20 Jul 2026 11:03:20 -0700 Subject: [PATCH 09/15] remove event limit from world-testing --- packages/world-testing/src/index.mts | 2 -- packages/world-testing/test/event-limit.test.ts | 6 ++++++ 2 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 packages/world-testing/test/event-limit.test.ts diff --git a/packages/world-testing/src/index.mts b/packages/world-testing/src/index.mts index 67a7ff983b..04637f882f 100644 --- a/packages/world-testing/src/index.mts +++ b/packages/world-testing/src/index.mts @@ -1,6 +1,5 @@ import { addition } from './addition.mjs'; import { errors } from './errors.mjs'; -import { eventLimit } from './event-limit.mjs'; import { hooks } from './hooks.mjs'; import { idempotency } from './idempotency.mjs'; import { inlineExecution } from './inline-execution.mjs'; @@ -15,5 +14,4 @@ export function createTestSuite(pkgName: string) { errors(pkgName); inlineExecution(pkgName); lineage(pkgName); - eventLimit(pkgName); } diff --git a/packages/world-testing/test/event-limit.test.ts b/packages/world-testing/test/event-limit.test.ts new file mode 100644 index 0000000000..bc5c135554 --- /dev/null +++ b/packages/world-testing/test/event-limit.test.ts @@ -0,0 +1,6 @@ +import { eventLimit } from '../src/event-limit.mjs'; + +// Local-only: enforcement needs a world that returns `maxEvents` on the +// run_started response. The Local World does (via WORKFLOW_MAX_EVENTS); the +// Postgres world doesn't yet, so this is not part of the shared createTestSuite. +eventLimit('local'); From a9abf591cdc4396ac80c9dc56bf0fafde186468a Mon Sep 17 00:00:00 2001 From: shalabhc Date: Mon, 20 Jul 2026 12:15:23 -0700 Subject: [PATCH 10/15] address new comments --- packages/world-testing/workflows/event-limit.ts | 2 +- packages/world/src/events.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/world-testing/workflows/event-limit.ts b/packages/world-testing/workflows/event-limit.ts index dc20f9877a..e30c34a5c0 100644 --- a/packages/world-testing/workflows/event-limit.ts +++ b/packages/world-testing/workflows/event-limit.ts @@ -10,7 +10,7 @@ async function tick(i: number): Promise { export async function runawayWorkflow(): Promise { 'use workflow'; let total = 0; - for (let i = 0; i < 100; i++) { + for (let i = 0; i < 15; i++) { total += await tick(i); } return total; diff --git a/packages/world/src/events.ts b/packages/world/src/events.ts index 75d5caa009..36cc7f2a43 100644 --- a/packages/world/src/events.ts +++ b/packages/world/src/events.ts @@ -778,8 +778,6 @@ export interface EventResult { hook?: import('./hooks.js').Hook; /** The wait entity (for wait_created/wait_completed events) */ wait?: import('./waits.js').Wait; - /** Server-owned max event count for the run (run-lifecycle responses); the runtime enforces it. */ - maxEvents?: number; /** * Events with data resolved. Two producers populate this: * @@ -807,6 +805,8 @@ export interface EventResult { * the safe default (treated as "not the lazy creator"). */ stepCreated?: boolean; + /** Server-owned max event count for the run (run-lifecycle responses); the runtime enforces it. */ + maxEvents?: number; } export interface GetEventParams { From 70c005b495cbf3d1343fce6ca406782aef7412de Mon Sep 17 00:00:00 2001 From: shalabhc Date: Mon, 20 Jul 2026 15:28:50 -0700 Subject: [PATCH 11/15] fix events-storage --- packages/world-local/src/storage/events-storage.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/world-local/src/storage/events-storage.ts b/packages/world-local/src/storage/events-storage.ts index d55c01933f..d96aab2d07 100644 --- a/packages/world-local/src/storage/events-storage.ts +++ b/packages/world-local/src/storage/events-storage.ts @@ -1131,7 +1131,7 @@ export function createEventsStorage( // because this is a rare race-condition path — the runtime // falls back to loadWorkflowRunEvents(). if (currentRun.status === 'running') { - return { run: currentRun }; + return { run: currentRun, maxEvents: getMaxEventsPerRun() }; } run = await writeRunUnderLifecycleLock( From e38f404abace40b4e4573cf5e9cb0504743b8cbb Mon Sep 17 00:00:00 2001 From: shalabhc Date: Mon, 20 Jul 2026 15:39:00 -0700 Subject: [PATCH 12/15] blank From 4b10e0be76ce7cf4d072500c52823fd27b8b3f69 Mon Sep 17 00:00:00 2001 From: shalabhc Date: Mon, 20 Jul 2026 17:51:02 -0700 Subject: [PATCH 13/15] blank From da048ffd69f1d66927add9066da5710a644718fa Mon Sep 17 00:00:00 2001 From: shalabhc Date: Tue, 21 Jul 2026 11:17:18 -0700 Subject: [PATCH 14/15] fix turbo path --- packages/core/src/runtime.ts | 30 +++++++++++++++++++++++++- packages/core/src/runtime/constants.ts | 29 +++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 7e8204c26c..37160dd1a1 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -38,6 +38,7 @@ import { type StepInvocationQueueItem, WorkflowSuspension } from './global.js'; import { runtimeLogger } from './logger.js'; import { ReplayPayloadCache } from './replay-payload-cache.js'; import { + getMaxEventsOverride, getMaxQueueDeliveries, getReplayDivergenceMaxRetries, isInlineOwnershipEnabled, @@ -148,6 +149,20 @@ export { type WorldFactoryModule, } from './runtime/world.js'; +/** + * Apply the optional client-side event-limit override. + * `WORKFLOW_MAX_EVENTS_OVERRIDE`, when set to a positive integer, clamps the + * server-supplied per-run event ceiling to a smaller value so enforcement can + * be exercised without a server-side change. Clamp-down only: it never raises + * the server's limit, and it takes effect even when the server returns none. + * Unset ⇒ server value passes through unchanged. + */ +function clampMaxEvents(serverValue: number | undefined): number | undefined { + const override = getMaxEventsOverride(); + if (override === undefined) return serverValue; + return serverValue === undefined ? override : Math.min(serverValue, override); +} + function getWorkflowSetupErrorCode(err: unknown): RunErrorCode | null { if (WorkflowRuntimeError.is(err)) { return RUN_ERROR_CODES.RUNTIME_ERROR; @@ -977,6 +992,19 @@ export function workflowEntrypoint( { requestId, skipPreload: true } ); runReadyBarrier = startedPromise; + // Turbo backgrounds run_started, so the non-turbo assignment + // below never runs — thread the per-run event ceiling off the + // backgrounded response here instead. The guard re-checks + // maxEventsLimit every loop iteration, so a value that lands + // shortly after start still enforces well before a runaway + // log approaches the ceiling. + startedPromise.then( + (r) => { + const limit = clampMaxEvents(r?.maxEvents); + if (limit !== undefined) maxEventsLimit = limit; + }, + () => {} + ); // Attach a no-op rejection handler so an early failure // never surfaces as an unhandledRejection before a consumer // (await/then) is attached; consumers still observe it. @@ -1040,7 +1068,7 @@ export function workflowEntrypoint( ); } workflowRun = result.run; - maxEventsLimit = result.maxEvents; + maxEventsLimit = clampMaxEvents(result.maxEvents); // Anchors RSFS — see the declaration above. runStartedReceivedAtMs = Date.now(); diff --git a/packages/core/src/runtime/constants.ts b/packages/core/src/runtime/constants.ts index 8db7a3f723..619c50f890 100644 --- a/packages/core/src/runtime/constants.ts +++ b/packages/core/src/runtime/constants.ts @@ -215,6 +215,35 @@ export function getMaxInlineSteps(): number { return parsed; } +const warnedMaxEventsValues = new Set(); + +/** + * Optional client-side override for the server-supplied per-run event ceiling. + * When set to a positive integer, the runtime clamps the server's limit *down* + * to this value (never raises it) so enforcement can be exercised without a + * server-side change. `undefined` (unset) ⇒ use the server value as-is. + * + * Reads `process.env.WORKFLOW_MAX_EVENTS_OVERRIDE` lazily so tests and + * deployments can override per invocation. Invalid values fall back to unset + * (no throw — the env var is an escape hatch) and emit a one-time warning. + */ +export function getMaxEventsOverride(): number | undefined { + const raw = process.env.WORKFLOW_MAX_EVENTS_OVERRIDE; + if (!raw) return undefined; + const parsed = Number(raw); + if (!Number.isInteger(parsed) || parsed <= 0) { + if (!warnedMaxEventsValues.has(raw)) { + warnedMaxEventsValues.add(raw); + runtimeLogger.warn( + 'Ignoring WORKFLOW_MAX_EVENTS_OVERRIDE: not a positive integer; using server limit', + { raw } + ); + } + return undefined; + } + return parsed; +} + /** * Whether optimistic inline step start is enabled. When on, the owned-inline * path begins running a brand-new step's body *before* its lazy `step_started` From 81e9dcbec54064877a439563a42846807667e221 Mon Sep 17 00:00:00 2001 From: shalabhc Date: Tue, 21 Jul 2026 11:27:51 -0700 Subject: [PATCH 15/15] fix test --- packages/world-testing/src/event-limit.mts | 65 ++++++++++++---------- 1 file changed, 35 insertions(+), 30 deletions(-) diff --git a/packages/world-testing/src/event-limit.mts b/packages/world-testing/src/event-limit.mts index d91db356b5..dc15b65c81 100644 --- a/packages/world-testing/src/event-limit.mts +++ b/packages/world-testing/src/event-limit.mts @@ -2,36 +2,41 @@ import { expect, test, vi } from 'vitest'; import { createFetcher, startServer } from './util.mjs'; export function eventLimit(world: string) { - test( - 'fails a runaway run at the server-supplied event limit', - { timeout: 59_000 }, - async () => { - // Low ceiling to trip it fast; turbo off because turbo backgrounds - // run_started (the guard reads maxEvents off that response). - const server = await startServer({ - world, - env: { WORKFLOW_MAX_EVENTS: '10', WORKFLOW_TURBO: '0' }, - }).then(createFetcher); + // Cover enforcement end-to-end under both configs (turbo is the production + // default). Against the local world these don't isolate the turbo run_started + // threading — a non-turbo continuation backfills the ceiling regardless — but + // they guard that a runaway run fails under each. + for (const turbo of ['1', '0'] as const) { + test( + `fails a runaway run at the server-supplied event limit (turbo=${turbo})`, + { timeout: 59_000 }, + async () => { + // Low ceiling to trip it fast. + const server = await startServer({ + world, + env: { WORKFLOW_MAX_EVENTS: '10', WORKFLOW_TURBO: turbo }, + }).then(createFetcher); - const result = await server.invoke( - 'workflows/event-limit.ts', - 'runawayWorkflow', - [] - ); - expect(result.runId).toMatch(/^wrun_.+/); + const result = await server.invoke( + 'workflows/event-limit.ts', + 'runawayWorkflow', + [] + ); + expect(result.runId).toMatch(/^wrun_.+/); - const run = await vi.waitFor( - async () => { - const run = await server.getRun(result.runId); - expect(run.status).toBe('failed'); - return run; - }, - { - interval: 200, - timeout: 50_000, - } - ); - expect(run.errorCode).toBe('MAX_EVENTS_EXCEEDED'); - } - ); + const run = await vi.waitFor( + async () => { + const run = await server.getRun(result.runId); + expect(run.status).toBe('failed'); + return run; + }, + { + interval: 200, + timeout: 50_000, + } + ); + expect(run.errorCode).toBe('MAX_EVENTS_EXCEEDED'); + } + ); + } }