diff --git a/docs/config/retry.md b/docs/config/retry.md index 48c0e8f1026e..66d50ba068d0 100644 --- a/docs/config/retry.md +++ b/docs/config/retry.md @@ -5,8 +5,141 @@ outline: deep # retry -- **Type:** `number` +Retry the test specific number of times if it fails. + +- **Type:** `number | { count?: number, delay?: number, condition?: RegExp }` - **Default:** `0` -- **CLI:** `--retry=` +- **CLI:** `--retry `, `--retry.count `, `--retry.delay `, `--retry.condition ` -Retry the test specific number of times if it fails. +## Basic Usage + +Specify a number to retry failed tests: + +```ts +export default defineConfig({ + test: { + retry: 3, + }, +}) +``` + +## CLI Usage + +You can also configure retry options from the command line: + +```bash +# Simple retry count +vitest --retry 3 + +# Advanced options using dot notation +vitest --retry.count 3 --retry.delay 500 --retry.condition 'ECONNREFUSED|timeout' +``` + +## Advanced Options 4.1.0 {#advanced-options} + +Use an object to configure retry behavior: + +```ts +export default defineConfig({ + test: { + retry: { + count: 3, // Number of times to retry + delay: 1000, // Delay in milliseconds between retries + condition: /ECONNREFUSED|timeout/i, // RegExp to match errors that should trigger retry + }, + }, +}) +``` + +### count + +Number of times to retry a test if it fails. Default is `0`. + +```ts +export default defineConfig({ + test: { + retry: { + count: 2, + }, + }, +}) +``` + +### delay + +Delay in milliseconds between retry attempts. Useful for tests that interact with rate-limited APIs or need time to recover. Default is `0`. + +```ts +export default defineConfig({ + test: { + retry: { + count: 3, + delay: 500, // Wait 500ms between retries + }, + }, +}) +``` + +### condition + +A RegExp pattern or a function to determine if a test should be retried based on the error. + +- When a **RegExp**, it's tested against the error message +- When a **function**, it receives the error and returns a boolean + +::: warning +When defining `condition` as a function, it must be done in a test file directly, not in a configuration file (configurations are serialized for worker threads). +::: + +#### RegExp condition (in config file): + +```ts +export default defineConfig({ + test: { + retry: { + count: 2, + condition: /ECONNREFUSED|ETIMEDOUT/i, // Retry on connection/timeout errors + }, + }, +}) +``` + +#### Function condition (in test file): + +```ts +import { describe, test } from 'vitest' + +describe('tests with advanced retry condition', () => { + test('with function condition', { retry: { count: 2, condition: error => error.message.includes('Network') } }, () => { + // test code + }) +}) +``` + +## Test File Override + +You can also define retry options per test or suite in test files: + +```ts +import { describe, test } from 'vitest' + +describe('flaky tests', { + retry: { + count: 2, + delay: 100, + }, +}, () => { + test('network request', () => { + // test code + }) +}) + +test('another test', { + retry: { + count: 3, + condition: error => error.message.includes('timeout'), + }, +}, () => { + // test code +}) +``` diff --git a/packages/runner/src/run.ts b/packages/runner/src/run.ts index 64cb2fec323c..61f62f77e97f 100644 --- a/packages/runner/src/run.ts +++ b/packages/runner/src/run.ts @@ -1,4 +1,4 @@ -import type { Awaitable } from '@vitest/utils' +import type { Awaitable, TestError } from '@vitest/utils' import type { DiffOptions } from '@vitest/utils/diff' import type { FileSpecification, VitestRunner } from './types/runner' import type { @@ -34,6 +34,42 @@ const now = globalThis.performance ? globalThis.performance.now.bind(globalThis. const unixNow = Date.now const { clearTimeout, setTimeout } = getSafeTimers() +/** + * Normalizes retry configuration to extract individual values. + * Handles both number and object forms. + */ +function getRetryCount(retry: number | { count?: number } | undefined): number { + if (retry === undefined) { + return 0 + } + if (typeof retry === 'number') { + return retry + } + return retry.count ?? 0 +} + +function getRetryDelay(retry: number | { delay?: number } | undefined): number { + if (retry === undefined) { + return 0 + } + if (typeof retry === 'number') { + return 0 + } + return retry.delay ?? 0 +} + +function getRetryCondition( + retry: number | { condition?: RegExp | ((error: TestError) => boolean) } | undefined, +): RegExp | ((error: TestError) => boolean) | undefined { + if (retry === undefined) { + return undefined + } + if (typeof retry === 'number') { + return undefined + } + return retry.condition +} + function updateSuiteHookState( task: Task, name: keyof SuiteHooks, @@ -266,6 +302,32 @@ async function callCleanupHooks(runner: VitestRunner, cleanups: unknown[]) { } } +/** + * Determines if a test should be retried based on its retryCondition configuration + */ +function passesRetryCondition(test: Test, errors: TestError[] | undefined): boolean { + const condition = getRetryCondition(test.retry) + + if (!errors || errors.length === 0) { + return false + } + + if (!condition) { + return true + } + + const error = errors[errors.length - 1] + + if (condition instanceof RegExp) { + return condition.test(error.message || '') + } + else if (typeof condition === 'function') { + return condition(error) + } + + return false +} + export async function runTest(test: Test, runner: VitestRunner): Promise { await runner.onBeforeRunTask?.(test) @@ -300,7 +362,7 @@ export async function runTest(test: Test, runner: VitestRunner): Promise { const repeats = test.repeats ?? 0 for (let repeatCount = 0; repeatCount <= repeats; repeatCount++) { - const retry = test.retry ?? 0 + const retry = getRetryCount(test.retry) for (let retryCount = 0; retryCount <= retry; retryCount++) { let beforeEachCleanups: unknown[] = [] try { @@ -412,9 +474,19 @@ export async function runTest(test: Test, runner: VitestRunner): Promise { } if (retryCount < retry) { - // reset state when retry test + const shouldRetry = passesRetryCondition(test, test.result.errors) + + if (!shouldRetry) { + break + } + test.result.state = 'run' test.result.retryCount = (test.result.retryCount ?? 0) + 1 + + const delay = getRetryDelay(test.retry) + if (delay > 0) { + await new Promise(resolve => setTimeout(resolve, delay)) + } } // update retry info diff --git a/packages/runner/src/types.ts b/packages/runner/src/types.ts index 84e2379eef8f..c15b3b2af6df 100644 --- a/packages/runner/src/types.ts +++ b/packages/runner/src/types.ts @@ -20,10 +20,12 @@ export type { InferFixturesTypes, OnTestFailedHandler, OnTestFinishedHandler, + Retry, RunMode, RuntimeContext, SequenceHooks, SequenceSetupFiles, + SerializableRetry, Suite, SuiteAPI, SuiteCollector, diff --git a/packages/runner/src/types/runner.ts b/packages/runner/src/types/runner.ts index 542a2da7ff44..09959375e4fc 100644 --- a/packages/runner/src/types/runner.ts +++ b/packages/runner/src/types/runner.ts @@ -4,6 +4,7 @@ import type { ImportDuration, SequenceHooks, SequenceSetupFiles, + SerializableRetry, Suite, TaskEventPack, TaskResultPack, @@ -36,7 +37,7 @@ export interface VitestRunnerConfig { maxConcurrency: number testTimeout: number hookTimeout: number - retry: number + retry: SerializableRetry includeTaskLocation?: boolean diffOptions?: DiffOptions } diff --git a/packages/runner/src/types/tasks.ts b/packages/runner/src/types/tasks.ts index 214aae3923df..779ce7a25137 100644 --- a/packages/runner/src/types/tasks.ts +++ b/packages/runner/src/types/tasks.ts @@ -87,10 +87,12 @@ export interface TaskBase { */ result?: TaskResult /** - * The amount of times the task should be retried if it fails. + * Retry configuration for the task. + * - If a number, specifies how many times to retry + * - If an object, allows fine-grained retry control * @default 0 */ - retry?: number + retry?: Retry /** * The amount of times the task should be repeated after the successful run. * If the task fails, it will not be retried unless `retry` is specified. @@ -461,18 +463,70 @@ type ChainableTestAPI = ChainableFunction< type TestCollectorOptions = Omit +/** + * Retry configuration for tests. + * Can be a number for simple retry count, or an object for advanced retry control. + */ +export type Retry = number | { + /** + * The number of times to retry the test if it fails. + * @default 0 + */ + count?: number + /** + * Delay in milliseconds between retry attempts. + * @default 0 + */ + delay?: number + /** + * Condition to determine if a test should be retried based on the error. + * - If a RegExp, it is tested against the error message + * - If a function, called with the TestError object; return true to retry + * + * NOTE: Functions can only be used in test files, not in vitest.config.ts, + * because the configuration is serialized when passed to worker threads. + * + * @default undefined (retry on all errors) + */ + condition?: RegExp | ((error: TestError) => boolean) +} + +/** + * Serializable retry configuration (used in config files). + * Functions cannot be serialized, so only string conditions are allowed. + */ +export type SerializableRetry = number | { + /** + * The number of times to retry the test if it fails. + * @default 0 + */ + count?: number + /** + * Delay in milliseconds between retry attempts. + * @default 0 + */ + delay?: number + /** + * Condition to determine if a test should be retried based on the error. + * Must be a RegExp tested against the error message. + * + * @default undefined (retry on all errors) + */ + condition?: RegExp +} + export interface TestOptions { /** * Test timeout. */ timeout?: number /** - * Times to retry the test if fails. Useful for making flaky tests more stable. - * When retries is up, the last test error will be thrown. - * + * Retry configuration for the test. + * - If a number, specifies how many times to retry + * - If an object, allows fine-grained retry control * @default 0 */ - retry?: number + retry?: Retry /** * How many times the test will run again. * Only inner tests will repeat if set on `describe()`, nested `describe()` will inherit parent's repeat by default. diff --git a/packages/vitest/src/node/cli/cli-config.ts b/packages/vitest/src/node/cli/cli-config.ts index a3e078e5b387..0b5db5a7fad9 100644 --- a/packages/vitest/src/node/cli/cli-config.ts +++ b/packages/vitest/src/node/cli/cli-config.ts @@ -530,6 +530,29 @@ export const cliOptionsConfig: VitestCLIOptions = { description: 'Retry the test specific number of times if it fails (default: `0`)', argument: '', + subcommands: { + count: { + description: + 'Number of times to retry a test if it fails (default: `0`)', + argument: '', + }, + delay: { + description: + 'Delay in milliseconds between retry attempts (default: `0`)', + argument: '', + }, + condition: { + description: + 'Regex pattern to match error messages that should trigger a retry. Only errors matching this pattern will cause a retry (default: retry on all errors)', + argument: '', + transform: (value) => { + if (typeof value === 'string') { + return new RegExp(value, 'i') + } + return value + }, + }, + }, }, diff: { description: diff --git a/packages/vitest/src/node/config/resolveConfig.ts b/packages/vitest/src/node/config/resolveConfig.ts index 45e99c651dcb..fed4e7b913e2 100644 --- a/packages/vitest/src/node/config/resolveConfig.ts +++ b/packages/vitest/src/node/config/resolveConfig.ts @@ -142,6 +142,18 @@ export function resolveConfig( mode, } as any as ResolvedConfig + if (resolved.retry && typeof resolved.retry === 'object' && typeof resolved.retry.condition === 'function') { + logger.console.warn( + c.yellow('Warning: retry.condition function cannot be used inside a config file. ' + + 'Use a RegExp pattern instead, or define the function in your test file.'), + ) + + resolved.retry = { + ...resolved.retry, + condition: undefined, + } + } + if (options.pool && typeof options.pool !== 'string') { resolved.pool = options.pool.name resolved.poolRunner = options.pool diff --git a/packages/vitest/src/node/reporters/reported-tasks.ts b/packages/vitest/src/node/reporters/reported-tasks.ts index 3ba098c51c8d..f5d0c3822e40 100644 --- a/packages/vitest/src/node/reporters/reported-tasks.ts +++ b/packages/vitest/src/node/reporters/reported-tasks.ts @@ -4,6 +4,7 @@ import type { Test as RunnerTestCase, File as RunnerTestFile, Suite as RunnerTestSuite, + SerializableRetry, TaskMeta, TestAnnotation, TestArtifact, @@ -524,7 +525,7 @@ export interface TaskOptions { readonly fails: boolean | undefined readonly concurrent: boolean | undefined readonly shuffle: boolean | undefined - readonly retry: number | undefined + readonly retry: SerializableRetry | undefined readonly repeats: number | undefined readonly mode: 'run' | 'only' | 'skip' | 'todo' } @@ -537,7 +538,7 @@ function buildOptions( fails: task.type === 'test' && task.fails, concurrent: task.concurrent, shuffle: task.shuffle, - retry: task.retry, + retry: task.retry as SerializableRetry | undefined, repeats: task.repeats, // runner types are too broad, but the public API should be more strict // the queued state exists only on Files and this method is called diff --git a/packages/vitest/src/node/types/config.ts b/packages/vitest/src/node/types/config.ts index fcd1f881083e..cb792a5e7ea1 100644 --- a/packages/vitest/src/node/types/config.ts +++ b/packages/vitest/src/node/types/config.ts @@ -1,6 +1,6 @@ import type { FakeTimerInstallOpts } from '@sinonjs/fake-timers' import type { PrettyFormatOptions } from '@vitest/pretty-format' -import type { SequenceHooks, SequenceSetupFiles } from '@vitest/runner' +import type { SequenceHooks, SequenceSetupFiles, SerializableRetry } from '@vitest/runner' import type { SnapshotStateOptions } from '@vitest/snapshot' import type { Arrayable } from '@vitest/utils' import type { SerializedDiffOptions } from '@vitest/utils/diff' @@ -783,11 +783,17 @@ export interface InlineConfig { bail?: number /** - * Retry the test specific number of times if it fails. + * Retry configuration for tests. + * - If a number, specifies how many times to retry failed tests + * - If an object, allows fine-grained retry control * - * @default 0 + * ⚠️ WARNING: Function form is NOT supported in a config file + * because configurations are serialized when passed to worker threads. + * Use the function form only in test files directly. + * + * @default 0 // Don't retry */ - retry?: number + retry?: SerializableRetry /** * Show full diff when snapshot fails instead of a patch. diff --git a/packages/vitest/src/runtime/config.ts b/packages/vitest/src/runtime/config.ts index 1e7cf9561eb9..10ff10ce6bd4 100644 --- a/packages/vitest/src/runtime/config.ts +++ b/packages/vitest/src/runtime/config.ts @@ -1,6 +1,6 @@ import type { FakeTimerInstallOpts } from '@sinonjs/fake-timers' import type { PrettyFormatOptions } from '@vitest/pretty-format' -import type { SequenceHooks, SequenceSetupFiles } from '@vitest/runner' +import type { SequenceHooks, SequenceSetupFiles, SerializableRetry } from '@vitest/runner' import type { SnapshotEnvironment, SnapshotUpdateState } from '@vitest/snapshot' import type { SerializedDiffOptions } from '@vitest/utils/diff' @@ -77,7 +77,7 @@ export interface SerializedConfig { truncateThreshold?: number } | undefined diff: string | SerializedDiffOptions | undefined - retry: number + retry: SerializableRetry includeTaskLocation: boolean | undefined inspect: boolean | string | undefined inspectBrk: boolean | string | undefined diff --git a/packages/vitest/src/runtime/runners/index.ts b/packages/vitest/src/runtime/runners/index.ts index e3f5aca59e52..2defd1e12219 100644 --- a/packages/vitest/src/runtime/runners/index.ts +++ b/packages/vitest/src/runtime/runners/index.ts @@ -103,6 +103,20 @@ export async function resolveTestRunner( state.durations.prepare = 0 state.durations.environment = 0 }) + + // Strip function conditions from retry config before sending via RPC + // Functions cannot be cloned by structured clone algorithm + const sanitizeRetryConditions = (task: any) => { + if (task.retry && typeof task.retry === 'object' && typeof task.retry.condition === 'function') { + // Remove function condition - it can't be serialized + task.retry = { ...task.retry, condition: undefined } + } + if (task.tasks) { + task.tasks.forEach(sanitizeRetryConditions) + } + } + files.forEach(sanitizeRetryConditions) + rpc().onCollected(files) await originalOnCollected?.call(testRunner, files) } diff --git a/test/cli/fixtures/retry-config/vitest.config.ts b/test/cli/fixtures/retry-config/vitest.config.ts new file mode 100644 index 000000000000..c97b47894e15 --- /dev/null +++ b/test/cli/fixtures/retry-config/vitest.config.ts @@ -0,0 +1,11 @@ +import { defineConfig } from 'vitest/config' + +export default defineConfig({ + test: { + retry: { + count: 3, + // @ts-expect-error + condition: () => true + } + } +}) diff --git a/test/cli/test/fails.test.ts b/test/cli/test/fails.test.ts index d966cdfaa9ea..b36cf19362ef 100644 --- a/test/cli/test/fails.test.ts +++ b/test/cli/test/fails.test.ts @@ -161,3 +161,11 @@ it('reports test file if it failed to load', async () => { ] `) }) + +it('should warn if retry.condition is a function in config', async () => { + const { stderr } = await runVitest({ + root: 'fixtures/retry-config', + }) + + expect(stderr).toContain('Warning: retry.condition function cannot be used inside a config file.') +}) diff --git a/test/core/test/retry-condition.test.ts b/test/core/test/retry-condition.test.ts new file mode 100644 index 000000000000..0269c8f771fd --- /dev/null +++ b/test/core/test/retry-condition.test.ts @@ -0,0 +1,71 @@ +import type { TestError } from 'vitest' +import { describe, expect, it } from 'vitest' + +// Test with RegExp condition that eventually passes +let matchingCount = 0 +it('retry with matching condition', { + retry: { + count: 5, + condition: /retry/i, + }, +}, () => { + matchingCount += 1 + if (matchingCount < 3) { + throw new Error('Please retry this test') + } + // Third attempt succeeds +}) + +it('verify matching condition retried', () => { + expect(matchingCount).toBe(3) +}) + +// Test with no condition (should retry all errors) +let noConditionCount = 0 +it('retry without condition', { retry: 2 }, () => { + noConditionCount += 1 + expect(noConditionCount).toBe(3) +}) + +it('verify no condition retried all attempts', () => { + expect(noConditionCount).toBe(3) +}) + +// Test with function condition +let functionCount = 0 +const condition = (error: TestError) => error.name === 'TimeoutError' + +it('retry with function condition', { + retry: { + count: 5, + condition, + }, +}, () => { + functionCount += 1 + const err: any = new Error('Test failed') + err.name = 'TimeoutError' + if (functionCount < 3) { + throw err + } + // Third attempt succeeds +}) + +it('verify function condition worked', () => { + expect(functionCount).toBe(3) +}) + +describe('retry condition with describe', { + retry: { + count: 2, + condition: /flaky/i, + }, +}, () => { + let describeCount = 0 + it('test should inherit retryCondition from describe block', () => { + describeCount += 1 + if (describeCount === 1) { + throw new Error('Flaky test error') + } + // Second attempt succeeds + }) +}) diff --git a/test/core/test/retry-delay.test.ts b/test/core/test/retry-delay.test.ts new file mode 100644 index 000000000000..103d224ce8fa --- /dev/null +++ b/test/core/test/retry-delay.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it } from 'vitest' + +let delayCount = 0 +let delayStart = 0 + +it('retry with delay', { + retry: { + count: 2, + delay: 100, + }, +}, () => { + if (delayCount === 0) { + delayStart = Date.now() + } + delayCount += 1 + expect(delayCount).toBe(3) +}) + +it('verify delay was applied', () => { + const duration = Date.now() - delayStart + expect(delayCount).toBe(3) + // With 2 retries and 100ms delay each, should take at least 200ms + expect(duration).toBeGreaterThanOrEqual(200) +}) + +let zeroDelayCount = 0 + +it('retry with zero delay', { + retry: { + count: 2, + delay: 0, + }, +}, () => { + zeroDelayCount += 1 + expect(zeroDelayCount).toBe(3) +}) + +it('verify zero delay test passed', () => { + expect(zeroDelayCount).toBe(3) +}) + +describe('retry delay with describe', { + retry: { + count: 2, + delay: 50, + }, +}, () => { + let describeCount = 0 + it('test should inherit retryDelay from describe block', () => { + describeCount += 1 + expect(describeCount).toBe(2) + }) +})