Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions .changeset/event-limit-enforcement.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
---
"@workflow/errors": patch
"@workflow/core": patch
"@workflow/world": patch
"@workflow/world-vercel": patch
---

Enforce a server-supplied per-run event limit (default 25K)
7 changes: 7 additions & 0 deletions packages/core/src/classify-error.test.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import {
CorruptedEventLogError,
HookConflictError,
MaxEventsExceededError,
ReplayDivergenceError,
RUN_ERROR_CODES,
RuntimeDecryptionError,
Expand All @@ -20,6 +21,12 @@ describe('classifyRunError', () => {
).toBe(RUN_ERROR_CODES.CORRUPTED_EVENT_LOG);
});

it('classifies MaxEventsExceededError as MAX_EVENTS_EXCEEDED', () => {
expect(classifyRunError(new MaxEventsExceededError(25_100, 25_000))).toBe(
RUN_ERROR_CODES.MAX_EVENTS_EXCEEDED
);
});

it('classifies ReplayDivergenceError as REPLAY_DIVERGENCE', () => {
expect(
classifyRunError(
Expand Down
5 changes: 5 additions & 0 deletions packages/core/src/classify-error.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import {
CorruptedEventLogError,
MaxEventsExceededError,
ReplayDivergenceError,
RUN_ERROR_CODES,
type RunErrorCode,
Expand Down Expand Up @@ -116,6 +117,10 @@ export function classifyRunError(err: unknown): RunErrorCode {
return RUN_ERROR_CODES.CORRUPTED_EVENT_LOG;
}

if (MaxEventsExceededError.is(err)) {
return RUN_ERROR_CODES.MAX_EVENTS_EXCEEDED;
}

// World-layer faults — both a malformed response (contract violation) and a
// transient infrastructure failure (throttle / 5xx / transport / timeout,
// e.g. a firewall challenge) — are the backend's fault, not the user's.
Expand Down
19 changes: 19 additions & 0 deletions packages/core/src/describe-error.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,16 @@ describe('describeError', () => {
expect(result.hint).toContain('SDK contract');
});

test('MAX_EVENTS_EXCEEDED via precomputed errorCode is attributed to the user', () => {
const result = describeError(
undefined,
RUN_ERROR_CODES.MAX_EVENTS_EXCEEDED
);
expect(result.attribution).toBe('user');
expect(result.errorCode).toBe(RUN_ERROR_CODES.MAX_EVENTS_EXCEEDED);
expect(result.hint).toContain('maximum number of events');
});

test('precomputed errorCode wins over classifyRunError when both are provided', () => {
// A plain Error would classify as USER_ERROR, but passing REPLAY_TIMEOUT
// explicitly overrides that — useful for callers that know the failure
Expand Down Expand Up @@ -193,6 +203,15 @@ describe('describeRunError', () => {
expect(result.hint).toContain('max-delivery budget');
});

test('MAX_EVENTS_EXCEEDED errorCode is attributed to the user', () => {
const result = describeRunError({
errorCode: RUN_ERROR_CODES.MAX_EVENTS_EXCEEDED,
});
expect(result.attribution).toBe('user');
expect(result.errorCode).toBe(RUN_ERROR_CODES.MAX_EVENTS_EXCEEDED);
expect(result.hint).toContain('maximum number of events');
});

test('WORLD_CONTRACT_ERROR errorCode is attributed to the SDK', () => {
const result = describeRunError({
errorCode: RUN_ERROR_CODES.WORLD_CONTRACT_ERROR,
Expand Down
13 changes: 13 additions & 0 deletions packages/core/src/describe-error.ts
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,8 @@ const REPLAY_TIMEOUT_HINT =
'The workflow replay between step boundaries took too long. This bounds workflow-VM and event-log replay time only — step bodies (`"use step"` functions) are excluded. This usually means the event log is unusually large or the workflow function is doing heavy synchronous work in workflow code outside of step bodies. Override the default budget via the WORKFLOW_REPLAY_TIMEOUT_MS env var if needed.';
const MAX_DELIVERIES_HINT =
'The workflow queue exceeded its max-delivery budget. This usually indicates a persistent runtime failure — check the most recent stack traces for the underlying cause.';
const MAX_EVENTS_HINT =
'The workflow exceeded the maximum number of events per run. This usually means unbounded work in the workflow function — e.g. a loop that keeps creating steps without terminating. Break long-running workflows into child workflows to stay under the limit.';
const WORLD_CONTRACT_HINT =
'The workflow backend returned data that violated the SDK contract. This is not retryable; please report it with the stack trace and runId.';

Expand Down Expand Up @@ -130,6 +132,9 @@ export function describeRunError(
if (errorCode === RUN_ERROR_CODES.MAX_DELIVERIES_EXCEEDED) {
return { attribution: 'sdk', errorCode, hint: MAX_DELIVERIES_HINT };
}
if (errorCode === RUN_ERROR_CODES.MAX_EVENTS_EXCEEDED) {
return { attribution: 'user', errorCode, hint: MAX_EVENTS_HINT };
}
if (errorCode === RUN_ERROR_CODES.CORRUPTED_EVENT_LOG) {
return {
attribution: 'sdk',
Expand Down Expand Up @@ -228,6 +233,14 @@ export function describeError(
};
}

if (effectiveCode === RUN_ERROR_CODES.MAX_EVENTS_EXCEEDED) {
return {
attribution: 'user',
errorCode: effectiveCode,
hint: MAX_EVENTS_HINT,
};
}

if (effectiveCode === RUN_ERROR_CODES.CORRUPTED_EVENT_LOG) {
return {
attribution: 'sdk',
Expand Down
19 changes: 19 additions & 0 deletions packages/core/src/runtime.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import {
CorruptedEventLogError,
EntityConflictError,
FatalError,
MaxEventsExceededError,
PreconditionFailedError,
ReplayDivergenceError,
RUN_ERROR_CODES,
Expand Down Expand Up @@ -551,6 +552,9 @@ export function workflowEntrypoint(
// Shared state: set by either the background step path
// or the run_started setup below.
let workflowRun: WorkflowRun | undefined;
// Server-supplied per-run event ceiling from the run_started
// response. Undefined ⇒ no enforcement (older servers, turbo).
let maxEventsLimit: number | undefined;
let workflowStartedAt = -1;
let preloadedEvents: Event[] | undefined;
let preloadedEventsCursor: string | null | undefined;
Expand Down Expand Up @@ -1036,6 +1040,7 @@ export function workflowEntrypoint(
);
}
workflowRun = result.run;
maxEventsLimit = result.maxEvents;
// Anchors RSFS — see the declaration above.
runStartedReceivedAtMs = Date.now();

Expand Down Expand Up @@ -1422,6 +1427,20 @@ export function workflowEntrypoint(
return;
}

// Event-limit guard: fail a runaway run once its log
// reaches the server-supplied ceiling (undefined ⇒ no
// enforcement). The throw is caught below and written as
// run_failed / MAX_EVENTS_EXCEEDED.
if (
maxEventsLimit !== undefined &&
events.length >= maxEventsLimit
) {
throw new MaxEventsExceededError(
events.length,
maxEventsLimit
);
}

// Update cache reference (may have been set for first time)
cachedEvents = events;

Expand Down
2 changes: 2 additions & 0 deletions packages/errors/src/error-codes.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ export const RUN_ERROR_CODES = {
REPLAY_DIVERGENCE: 'REPLAY_DIVERGENCE',
/** Run exceeded the maximum number of queue deliveries */
MAX_DELIVERIES_EXCEEDED: 'MAX_DELIVERIES_EXCEEDED',
/** Run exceeded the maximum number of events per run */
MAX_EVENTS_EXCEEDED: 'MAX_EVENTS_EXCEEDED',
/** Workflow replay exceeded the maximum allowed duration */
REPLAY_TIMEOUT: 'REPLAY_TIMEOUT',
/** World response violated the SDK contract and cannot be retried safely */
Expand Down
24 changes: 24 additions & 0 deletions packages/errors/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -352,6 +352,30 @@ export class ReplayDivergenceError extends WorkflowRuntimeError {
}
}

/**
* Thrown when a run's event log reaches the server-supplied per-run event
* ceiling. Classified as `MAX_EVENTS_EXCEEDED` (see `classifyRunError`).
*/
export class MaxEventsExceededError extends WorkflowError {
readonly eventCount: number;
readonly limit: number;

constructor(
eventCount: number,
limit: number,
options?: WorkflowErrorOptions
) {
super(`Workflow exceeded the maximum of ${limit} events per run`, options);
this.name = 'MaxEventsExceededError';
this.eventCount = eventCount;
this.limit = limit;
}

static is(value: unknown): value is MaxEventsExceededError {
return isError(value) && value.name === 'MaxEventsExceededError';
}
}

/**
* Optional structured context attached to a {@link RuntimeDecryptionError},
* carried over from the underlying decrypt call site to help diagnose the
Expand Down
16 changes: 16 additions & 0 deletions packages/world-local/src/storage/events-storage.ts
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,19 @@ import {
import { handleLegacyEvent } from './legacy.js';
import { withRunFileLock } from './runs-storage.js';

/**
* Per-run event ceiling the Local World reports on run responses (mirrors the
* Vercel World). Overridable via `WORKFLOW_MAX_EVENTS`; defaults to 25,000.
*/
const DEFAULT_MAX_EVENTS_PER_RUN = 25_000;
function getMaxEventsPerRun(): number {
const raw = process.env.WORKFLOW_MAX_EVENTS;
const parsed = raw !== undefined ? Number(raw) : Number.NaN;
return Number.isInteger(parsed) && parsed > 0
? parsed
: DEFAULT_MAX_EVENTS_PER_RUN;
}

/**
* Per-step in-process async mutex. Serializes concurrent `events.create` calls
* that target the same step, so that the "check terminal state, then write step
Expand Down Expand Up @@ -883,6 +896,7 @@ export function createEventsStorage(
return {
event: stripEventDataRefs(event, resolveData),
run: currentRun,
...(currentRun ? { maxEvents: getMaxEventsPerRun() } : {}),
};
}

Expand Down Expand Up @@ -2292,6 +2306,8 @@ export function createEventsStorage(
cursor,
hasMore,
...(stepCreatedLazily ? { stepCreated: true } : {}),
// Per-run event ceiling (mirrors the Vercel World).
...(run ? { maxEvents: getMaxEventsPerRun() } : {}),
};
} // end createImpl
},
Expand Down
37 changes: 37 additions & 0 deletions packages/world-testing/src/event-limit.mts
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
import { expect, test, vi } from 'vitest';
import { createFetcher, startServer } from './util.mjs';

export function eventLimit(world: string) {
test(
'fails a runaway run at the server-supplied event limit',
{ timeout: 59_000 },
async () => {
// Low ceiling to trip it fast; turbo off because turbo backgrounds
// run_started (the guard reads maxEvents off that response).
const server = await startServer({
world,
env: { WORKFLOW_MAX_EVENTS: '10', WORKFLOW_TURBO: '0' },
}).then(createFetcher);

const result = await server.invoke(
'workflows/event-limit.ts',
'runawayWorkflow',
[]
);
expect(result.runId).toMatch(/^wrun_.+/);

const run = await vi.waitFor(
async () => {
const run = await server.getRun(result.runId);
expect(run.status).toBe('failed');
return run;
},
{
interval: 200,
timeout: 50_000,
}
);
expect(run.errorCode).toBe('MAX_EVENTS_EXCEEDED');
}
);
}
6 changes: 6 additions & 0 deletions packages/world-testing/test/event-limit.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
import { eventLimit } from '../src/event-limit.mjs';

// Local-only: enforcement needs a world that returns `maxEvents` on the
// run_started response. The Local World does (via WORKFLOW_MAX_EVENTS); the
// Postgres world doesn't yet, so this is not part of the shared createTestSuite.
eventLimit('local');
17 changes: 17 additions & 0 deletions packages/world-testing/workflows/event-limit.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
async function tick(i: number): Promise<number> {
'use step';
return i;
}

/**
* Runaway workflow: creates far more events than a low `WORKFLOW_MAX_EVENTS`
* ceiling, so the event-limit guard fails it before the loop finishes.
*/
export async function runawayWorkflow(): Promise<number> {
'use workflow';
let total = 0;
for (let i = 0; i < 15; i++) {
total += await tick(i);
}
return total;
}
5 changes: 5 additions & 0 deletions packages/world-vercel/src/events-v4.ts
Original file line number Diff line number Diff line change
Expand Up @@ -225,6 +225,11 @@ export interface CreateEventV4Result {
* creator). Threaded into EventResult.stepCreated by the events adapter.
*/
stepCreated?: boolean;
/**
* Server-owned per-run event ceiling, returned on run-lifecycle responses.
* Absent from older servers. Threaded into EventResult.maxEvents.
*/
maxEvents?: number;
};
}

Expand Down
4 changes: 4 additions & 0 deletions packages/world-vercel/src/events.ts
Original file line number Diff line number Diff line change
Expand Up @@ -754,5 +754,9 @@ async function createWorkflowRunEventInner(
// signal through so the owned-inline runtime path can gate body execution
// on it. Absent from older servers → undefined → safe default.
...(body.stepCreated ? { stepCreated: true } : {}),
// Server-supplied per-run event ceiling; absent from older servers.
...(typeof body.maxEvents === 'number'
? { maxEvents: body.maxEvents }
: {}),
};
}
2 changes: 2 additions & 0 deletions packages/world/src/events.ts
Original file line number Diff line number Diff line change
Expand Up @@ -794,6 +794,8 @@ export interface EventResult {
* the safe default (treated as "not the lazy creator").
*/
stepCreated?: boolean;
/** Server-owned max event count for the run (run-lifecycle responses); the runtime enforces it. */
maxEvents?: number;
}

export interface GetEventParams {
Expand Down
Loading