diff --git a/docs/src/api/class-page.md b/docs/src/api/class-page.md index 5c2b7521971d8..55399a4d678f8 100644 --- a/docs/src/api/class-page.md +++ b/docs/src/api/class-page.md @@ -722,6 +722,11 @@ Initialize page agent with the llm provider and cache. - `cacheFile` ?<[string]> Cache file to use/generate code for performed actions into. Cache is not used if not specified (default). - `cacheOutFile` ?<[string]> When specified, generated entries are written into the `cacheOutFile` instead of updating the `cacheFile`. +### option: Page.agent.expect +* since: v1.58 +- `expect` <[Object]> + - `timeout` ?<[int]> Default timeout for expect calls in milliseconds, defaults to 5000ms. + ### option: Page.agent.limits * since: v1.58 - `limits` <[Object]> diff --git a/docs/src/api/class-pageagent.md b/docs/src/api/class-pageagent.md index 26242e8c4428d..0f04e2b04d851 100644 --- a/docs/src/api/class-pageagent.md +++ b/docs/src/api/class-pageagent.md @@ -35,6 +35,12 @@ await agent.expect('"0 items" to be reported'); Expectation to assert. +### option: PageAgent.expect.timeout +* since: v1.58 +- `timeout` <[float]> + +Expect timeout in milliseconds. Defaults to `5000`. The default value can be changed via `expect.timeout` option in the config, or by specifying the `expect` property of the [`option: Page.agent.expect`] option. Pass `0` to disable timeout. + ### option: PageAgent.expect.-inline- = %%-page-agent-call-options-v1.58-%% * since: v1.58 @@ -68,6 +74,13 @@ Task to perform using agentic loop. * since: v1.58 - `schema` <[z.ZodSchema]> +### option: PageAgent.extract.timeout +* since: v1.58 +- `timeout` <[float]> + +Extract timeout in milliseconds. Defaults to `5000`. The default value can be changed via `actionTimeout` option in the config, or by using the [`method: BrowserContext.setDefaultTimeout`] or +[`method: Page.setDefaultTimeout`] methods. Pass `0` to disable timeout. + ### option: PageAgent.extract.-inline- = %%-page-agent-call-options-v1.58-%% * since: v1.58 @@ -94,6 +107,13 @@ await agent.perform('Click submit button'); Task to perform using agentic loop. +### option: PageAgent.perform.timeout +* since: v1.58 +- `timeout` <[float]> + +Perform timeout in milliseconds. Defaults to `5000`. The default value can be changed via `actionTimeout` option in the config, or by using the [`method: BrowserContext.setDefaultTimeout`] or +[`method: Page.setDefaultTimeout`] methods. Pass `0` to disable timeout. + ### option: PageAgent.perform.-inline- = %%-page-agent-call-options-v1.58-%% * since: v1.58 diff --git a/docs/src/api/params.md b/docs/src/api/params.md index 9b7bf8f3d4af4..69257b93db23d 100644 --- a/docs/src/api/params.md +++ b/docs/src/api/params.md @@ -396,18 +396,11 @@ Maximum number of agentic actions to generate, defaults to context-wide value sp Maximum number of retries when generating each action, defaults to context-wide value specified in `agent` property. -## page-agent-timeout -* since: v1.58 -- `timeout` <[int]> - -Request timeout in milliseconds. Defaults to action timeout. Pass `0` to disable timeout. - ## page-agent-call-options-v1.58 - %%-page-agent-cache-key-%% - %%-page-agent-max-tokens-%% - %%-page-agent-max-actions-%% - %%-page-agent-max-action-retries-%% -- %%-page-agent-timeout-%% ## fetch-param-url - `url` <[string]> diff --git a/packages/injected/src/ariaSnapshot.ts b/packages/injected/src/ariaSnapshot.ts index 6f7348115baae..2ce324e81fdf0 100644 --- a/packages/injected/src/ariaSnapshot.ts +++ b/packages/injected/src/ariaSnapshot.ts @@ -39,6 +39,7 @@ let lastRef = 0; export type AriaTreeOptions = { mode: 'ai' | 'expect' | 'codegen' | 'autoexpect'; refPrefix?: string; + doNotRenderActive?: boolean; }; type InternalOptions = { @@ -59,7 +60,7 @@ function toInternalOptions(options: AriaTreeOptions): InternalOptions { refs: 'interactable', refPrefix: options.refPrefix, includeGenericRole: true, - renderActive: true, + renderActive: !options.doNotRenderActive, renderCursorPointer: true, }; } diff --git a/packages/injected/src/injectedScript.ts b/packages/injected/src/injectedScript.ts index ce18bf46aa252..2be0082e4d79a 100644 --- a/packages/injected/src/injectedScript.ts +++ b/packages/injected/src/injectedScript.ts @@ -50,8 +50,7 @@ import type { Builtins } from './utilityScript'; export type FrameExpectParams = Omit & { expectedValue?: any; timeoutForLogs?: number; - explicitTimeout?: number; - noPreChecks?: boolean; + noAutoWaiting?: boolean; }; export type ElementState = 'visible' | 'hidden' | 'enabled' | 'disabled' | 'editable' | 'checked' | 'unchecked' | 'indeterminate' | 'stable'; diff --git a/packages/playwright-client/types/types.d.ts b/packages/playwright-client/types/types.d.ts index 6e2a58a63d4d3..50dd490708e13 100644 --- a/packages/playwright-client/types/types.d.ts +++ b/packages/playwright-client/types/types.d.ts @@ -2112,6 +2112,13 @@ export interface Page { cacheOutFile?: string; }; + expect?: { + /** + * Default timeout for expect calls in milliseconds, defaults to 5000ms. + */ + timeout?: number; + }; + /** * Limits to use for the agentic loop. */ @@ -5440,7 +5447,10 @@ export interface PageAgent { maxTokens?: number; /** - * Request timeout in milliseconds. Defaults to action timeout. Pass `0` to disable timeout. + * Expect timeout in milliseconds. Defaults to `5000`. The default value can be changed via `expect.timeout` option in + * the config, or by specifying the `expect` property of the + * [`expect`](https://playwright.dev/docs/api/class-page#page-agent-option-expect) option. Pass `0` to disable + * timeout. */ timeout?: number; }): Promise; @@ -5482,7 +5492,11 @@ export interface PageAgent { maxTokens?: number; /** - * Request timeout in milliseconds. Defaults to action timeout. Pass `0` to disable timeout. + * Perform timeout in milliseconds. Defaults to `5000`. The default value can be changed via `actionTimeout` option in + * the config, or by using the + * [browserContext.setDefaultTimeout(timeout)](https://playwright.dev/docs/api/class-browsercontext#browser-context-set-default-timeout) + * or [page.setDefaultTimeout(timeout)](https://playwright.dev/docs/api/class-page#page-set-default-timeout) methods. + * Pass `0` to disable timeout. */ timeout?: number; }): Promise<{ diff --git a/packages/playwright-core/src/client/page.ts b/packages/playwright-core/src/client/page.ts index 4e3d827595712..adc7654de9599 100644 --- a/packages/playwright-core/src/client/page.ts +++ b/packages/playwright-core/src/client/page.ts @@ -854,6 +854,7 @@ export class Page extends ChannelOwner implements api.Page apiKey: options.provider?.apiKey, apiTimeout: options.provider?.apiTimeout, apiCacheFile: (options.provider as any)?._apiCacheFile, + doNotRenderActive: (options as any)._doNotRenderActive, model: options.provider?.model, cacheFile: options.cache?.cacheFile, cacheOutFile: options.cache?.cacheOutFile, @@ -864,7 +865,9 @@ export class Page extends ChannelOwner implements api.Page systemPrompt: options.systemPrompt, }; const { agent } = await this._channel.agent(params); - return PageAgent.from(agent); + const pageAgent = PageAgent.from(agent); + pageAgent._expectTimeout = options?.expect?.timeout; + return pageAgent; } async _snapshotForAI(options: TimeoutOptions & { track?: string } = {}): Promise<{ full: string, incremental?: string }> { diff --git a/packages/playwright-core/src/client/pageAgent.ts b/packages/playwright-core/src/client/pageAgent.ts index 2837f5bbb11f3..ec6a16cc9aeab 100644 --- a/packages/playwright-core/src/client/pageAgent.ts +++ b/packages/playwright-core/src/client/pageAgent.ts @@ -22,14 +22,9 @@ import { Page } from './page'; import type * as api from '../../types/types'; import type * as channels from '@protocol/channels'; -type PageAgentOptions = { - maxTokens?: number; - maxTurns?: number; - cacheKey?: string; -}; - export class PageAgent extends ChannelOwner implements api.PageAgent { private _page: Page; + _expectTimeout?: number; static from(channel: channels.PageAgentChannel): PageAgent { return (channel as any)._object; @@ -41,17 +36,20 @@ export class PageAgent extends ChannelOwner implement this._channel.on('turn', params => this.emit(Events.Page.AgentTurn, params)); } - async expect(expectation: string, options: PageAgentOptions = {}) { - await this._channel.expect({ expectation, ...options }); + async expect(expectation: string, options: channels.PageAgentExpectOptions = {}) { + const timeout = options.timeout ?? this._expectTimeout ?? 5000; + await this._channel.expect({ expectation, ...options, timeout }); } - async perform(task: string, options: PageAgentOptions = {}) { - const { usage } = await this._channel.perform({ task, ...options }); + async perform(task: string, options: channels.PageAgentPerformOptions = {}) { + const timeout = this._page._timeoutSettings.timeout(options); + const { usage } = await this._channel.perform({ task, ...options, timeout }); return { usage }; } - async extract(query: string, schema: Schema, options: PageAgentOptions = {}): Promise<{ result: any, usage: channels.AgentUsage }> { - const { result, usage } = await this._channel.extract({ query, schema: this._page._platform.zodToJsonSchema(schema), ...options }); + async extract(query: string, schema: Schema, options: channels.PageAgentExtractOptions = {}): Promise<{ result: any, usage: channels.AgentUsage }> { + const timeout = this._page._timeoutSettings.timeout(options); + const { result, usage } = await this._channel.extract({ query, schema: this._page._platform.zodToJsonSchema(schema), ...options, timeout }); return { result, usage }; } diff --git a/packages/playwright-core/src/protocol/validator.ts b/packages/playwright-core/src/protocol/validator.ts index 4ed0e4af71eb2..bc253a5a7be63 100644 --- a/packages/playwright-core/src/protocol/validator.ts +++ b/packages/playwright-core/src/protocol/validator.ts @@ -1495,6 +1495,7 @@ scheme.PageAgentParams = tObject({ apiCacheFile: tOptional(tString), cacheFile: tOptional(tString), cacheOutFile: tOptional(tString), + doNotRenderActive: tOptional(tBoolean), maxActions: tOptional(tInt), maxActionRetries: tOptional(tInt), maxTokens: tOptional(tInt), diff --git a/packages/playwright-core/src/server/agent/actionRunner.ts b/packages/playwright-core/src/server/agent/actionRunner.ts index 9f25d3fec282e..9773c29b55a03 100644 --- a/packages/playwright-core/src/server/agent/actionRunner.ts +++ b/packages/playwright-core/src/server/agent/actionRunner.ts @@ -143,14 +143,11 @@ async function innerRunAction(progress: Progress, mode: 'generate' | 'run', page } async function runExpect(frame: Frame, progress: Progress, mode: 'generate' | 'run', selector: string | undefined, options: FrameExpectParams, expected: string | RegExp, matcherName: string, expectation: string) { - // Pass explicit timeout to limit the single expect action inside the overall "agentic expect" multi-step progress. - const timeout = expectTimeout(mode); const result = await frame.expect(progress, selector, { ...options, - timeoutForLogs: timeout, - explicitTimeout: timeout, - // Disable pre-checks to avoid them timing out, model has seen the snapshot anyway. - noPreChecks: mode === 'generate', + // When generating, we want the expect to pass or fail immediately and give feedback to the model. + noAutoWaiting: mode === 'generate', + timeoutForLogs: mode === 'generate' ? undefined : progress.timeout, }); if (!result.matches === !options.isNot) { const received = matcherName === 'toMatchAriaSnapshot' ? '\n' + result.received.raw : result.received; @@ -162,7 +159,7 @@ async function runExpect(frame: Frame, progress: Progress, mode: 'generate' | 'r expectation, locator: selector ? asLocatorDescription('javascript', selector) : undefined, timedOut: result.timedOut, - timeout, + timeout: mode === 'generate' ? undefined : progress.timeout, printedExpected: options.isNot ? `Expected${expectedSuffix}: not ${expectedDisplay}` : `Expected${expectedSuffix}: ${expectedDisplay}`, printedReceived: result.errorMessage ? '' : `Received: ${received}`, errorMessage: result.errorMessage, @@ -266,7 +263,7 @@ export function traceParamsForAction(progress: Progress, action: actions.Action, expression: 'to.have.value', expectedText, isNot: !!action.isNot, - timeout: expectTimeout(mode), + timeout, }; return { type: 'Frame', method: 'expect', title: 'Expect Value', params }; } else if (action.type === 'checkbox' || action.type === 'radio') { @@ -275,7 +272,7 @@ export function traceParamsForAction(progress: Progress, action: actions.Action, selector: action.selector, expression: 'to.be.checked', isNot: !!action.isNot, - timeout: expectTimeout(mode), + timeout, }; return { type: 'Frame', method: 'expect', title: 'Expect Checked', params }; } else { @@ -287,7 +284,7 @@ export function traceParamsForAction(progress: Progress, action: actions.Action, selector: action.selector, expression: 'to.be.visible', isNot: !!action.isNot, - timeout: expectTimeout(mode), + timeout, }; return { type: 'Frame', method: 'expect', title: 'Expect Visible', params }; } @@ -298,7 +295,7 @@ export function traceParamsForAction(progress: Progress, action: actions.Action, expression: 'to.match.snapshot', expectedText: [], isNot: !!action.isNot, - timeout: expectTimeout(mode), + timeout, }; return { type: 'Frame', method: 'expect', title: 'Expect Aria Snapshot', params }; } @@ -310,7 +307,7 @@ export function traceParamsForAction(progress: Progress, action: actions.Action, expression: 'to.have.url', expectedText, isNot: !!action.isNot, - timeout: expectTimeout(mode), + timeout, }; return { type: 'Frame', method: 'expect', title: 'Expect URL', params }; } @@ -321,7 +318,7 @@ export function traceParamsForAction(progress: Progress, action: actions.Action, expression: 'to.have.title', expectedText, isNot: !!action.isNot, - timeout: expectTimeout(mode), + timeout, }; return { type: 'Frame', method: 'expect', title: 'Expect Title', params }; } @@ -341,7 +338,3 @@ function callMetadataForAction(progress: Progress, frame: Frame, action: actions }; return callMetadata; } - -function expectTimeout(mode: 'generate' | 'run') { - return mode === 'generate' ? 0 : 5000; -} diff --git a/packages/playwright-core/src/server/agent/context.ts b/packages/playwright-core/src/server/agent/context.ts index 6bb1b618f168a..41853f073d376 100644 --- a/packages/playwright-core/src/server/agent/context.ts +++ b/packages/playwright-core/src/server/agent/context.ts @@ -132,9 +132,14 @@ export class Context { return result; } + async takeSnapshot(progress: Progress) { + const { full } = await this.page.snapshotForAI(progress, { doNotRenderActive: this.agentParams.doNotRenderActive }); + // TODO: it seems like redactText should be here. + return full; + } + async snapshotResult(progress: Progress, error?: Error): Promise { - let { full } = await this.page.snapshotForAI(progress); - full = this._redactText(full); + const snapshot = this._redactText(await this.takeSnapshot(progress)); const text: string[] = []; if (error) @@ -142,12 +147,12 @@ export class Context { else text.push(`# Success`); - text.push(`# Page snapshot\n${full}`); + text.push(`# Page snapshot\n${snapshot}`); return { _meta: { 'dev.lowire/state': { - 'Page snapshot': full + 'Page snapshot': snapshot }, 'dev.lowire/history': error ? [{ category: 'error', diff --git a/packages/playwright-core/src/server/agent/pageAgent.ts b/packages/playwright-core/src/server/agent/pageAgent.ts index dfad33bf99329..7b22731aa58e0 100644 --- a/packages/playwright-core/src/server/agent/pageAgent.ts +++ b/packages/playwright-core/src/server/agent/pageAgent.ts @@ -50,7 +50,7 @@ export async function pageAgentPerform(progress: Progress, context: Context, use ### Task ${userTask} `; - + progress.disableTimeout(); await runLoop(progress, context, performTools, task, undefined, callParams); await updateCache(context, cacheKey); } @@ -68,7 +68,7 @@ export async function pageAgentExpect(progress: Progress, context: Context, expe ### Expectation ${expectation} `; - + progress.disableTimeout(); await runLoop(progress, context, expectTools, task, undefined, callParams); await updateCache(context, cacheKey); } @@ -88,19 +88,20 @@ ${query}`; async function runLoop(progress: Progress, context: Context, toolDefinitions: ToolDefinition[], userTask: string, resultSchema: loopTypes.Schema | undefined, params: CallParams): Promise<{ result: any }> { - const { page } = context; if (!context.agentParams.api || !context.agentParams.model) throw new Error(`This action requires the API and API key to be set on the page agent. Did you mean to --run-agents=missing?`); if (!context.agentParams.apiKey) throw new Error(`This action requires API key to be set on the page agent.`); + if (context.agentParams.apiEndpoint && !URL.canParse(context.agentParams.apiEndpoint)) + throw new Error(`Agent API endpoint "${context.agentParams.apiEndpoint}" is not a valid URL.`); - const { full } = await page.snapshotForAI(progress); + const snapshot = await context.takeSnapshot(progress); const { tools, callTool, reportedResult, refusedToPerformReason } = toolsForLoop(progress, context, toolDefinitions, { resultSchema, refuseToPerform: 'allow' }); const secrets = Object.fromEntries((context.agentParams.secrets || [])?.map(s => ([s.name, s.value]))); const apiCacheTextBefore = context.agentParams.apiCacheFile ? await fs.promises.readFile(context.agentParams.apiCacheFile, 'utf-8').catch(() => '{}') : '{}'; - const apiCacheBefore = JSON.parse(apiCacheTextBefore); + const apiCacheBefore = JSON.parse(apiCacheTextBefore || '{}'); const loop = new Loop({ api: context.agentParams.api as any, @@ -136,7 +137,7 @@ async function runLoop(progress: Progress, context: Context, toolDefinitions: To task.push(''); } task.push('### Page snapshot'); - task.push(full); + task.push(snapshot); task.push(''); const { error, usage } = await loop.run(task.join('\n'), { signal: progress.signal }); @@ -205,7 +206,13 @@ const allCaches = new Map(); async function cachedActions(cacheFile: string): Promise { let cache = allCaches.get(cacheFile); if (!cache) { - const json = await fs.promises.readFile(cacheFile, 'utf-8').then(text => JSON.parse(text)).catch(() => ({})); + const content = await fs.promises.readFile(cacheFile, 'utf-8').catch(() => ''); + let json: any; + try { + json = JSON.parse(content.trim() || '{}'); + } catch (error) { + throw new Error(`Failed to parse cache file ${cacheFile}:\n${error.message}`); + } const parsed = actions.cachedActionsSchema.safeParse(json); if (parsed.error) throw new Error(`Failed to parse cache file ${cacheFile}:\n${zod.prettifyError(parsed.error)}`); diff --git a/packages/playwright-core/src/server/frames.ts b/packages/playwright-core/src/server/frames.ts index 5431ea6a40e26..a09b41d050cff 100644 --- a/packages/playwright-core/src/server/frames.ts +++ b/packages/playwright-core/src/server/frames.ts @@ -1389,14 +1389,10 @@ export class Frame extends SdkObject { progress.metadata.error = { error: { name: 'Expect', message: 'Expect failed' } }; }; try { - // When explicit timeout is passed, constrain expect by it, in addition to regular progress abort. - const timeoutPromise = options.explicitTimeout !== undefined ? progress.wait(options.explicitTimeout).then(() => { throw new TimeoutError(`Timed out after ${options.explicitTimeout}ms`); }) : undefined; - timeoutPromise?.catch(() => { /* Prevent unhandled promise rejection */ }); - // Step 1: perform locator handlers checkpoint with a specified timeout. if (selector) progress.log(`waiting for ${this._asLocator(selector)}`); - if (!options.noPreChecks) + if (!options.noAutoWaiting) await this._page.performActionPreChecks(progress); // Step 2: perform one-shot expect check without a timeout. @@ -1404,19 +1400,19 @@ export class Frame extends SdkObject { // that should succeed when the locator is already visible. try { const resultOneShot = await this._expectInternal(progress, selector, options, lastIntermediateResult, true); - if (resultOneShot.matches !== options.isNot) + if (options.noAutoWaiting || resultOneShot.matches !== options.isNot) return resultOneShot; } catch (e) { - if (this.isNonRetriableError(e)) + if (options.noAutoWaiting || this.isNonRetriableError(e)) throw e; // Ignore any other errors from one-shot, we'll handle them during retries. } // Step 3: auto-retry expect with increasing timeouts. Bounded by the total remaining time. const result = await this.retryWithProgressAndTimeouts(progress, [100, 250, 500, 1000], async continuePolling => { - if (!options.noPreChecks) + if (!options.noAutoWaiting) await this._page.performActionPreChecks(progress); - const { matches, received } = await this._expectInternal(progress, selector, options, lastIntermediateResult, false, timeoutPromise); + const { matches, received } = await this._expectInternal(progress, selector, options, lastIntermediateResult, false); if (matches === options.isNot) { // Keep waiting in these cases: // expect(locator).conditionThatDoesNotMatch @@ -1446,9 +1442,9 @@ export class Frame extends SdkObject { } } - private async _expectInternal(progress: Progress, selector: string | undefined, options: FrameExpectParams, lastIntermediateResult: { received?: any, isSet: boolean, errorMessage?: string }, noAbort: boolean, timeoutPromise?: Promise) { + private async _expectInternal(progress: Progress, selector: string | undefined, options: FrameExpectParams, lastIntermediateResult: { received?: any, isSet: boolean, errorMessage?: string }, noAbort: boolean) { // The first expect check, a.k.a. one-shot, always finishes - even when progress is aborted. - const race = (p: Promise) => noAbort ? p : (timeoutPromise ? progress.race([p, timeoutPromise]) : progress.race(p)); + const race = (p: Promise) => noAbort ? p : progress.race(p); const selectorInFrame = selector ? await race(this.selectors.resolveFrameForSelector(selector, { strict: true })) : undefined; const { frame, info } = selectorInFrame || { frame: this, info: undefined }; diff --git a/packages/playwright-core/src/server/page.ts b/packages/playwright-core/src/server/page.ts index 1759727992d74..278b89a10475e 100644 --- a/packages/playwright-core/src/server/page.ts +++ b/packages/playwright-core/src/server/page.ts @@ -863,7 +863,7 @@ export class Page extends SdkObject { await Promise.all(this.frames().map(frame => frame.hideHighlight().catch(() => {}))); } - async snapshotForAI(progress: Progress, options: { track?: string } = {}): Promise<{ full: string, incremental?: string }> { + async snapshotForAI(progress: Progress, options: { track?: string, doNotRenderActive?: boolean } = {}): Promise<{ full: string, incremental?: string }> { const snapshot = await snapshotFrameForAI(progress, this.mainFrame(), options); return { full: snapshot.full.join('\n'), incremental: snapshot.incremental?.join('\n') }; } @@ -986,7 +986,7 @@ export class InitScript { } -async function snapshotFrameForAI(progress: Progress, frame: frames.Frame, options: { track?: string }): Promise<{ full: string[], incremental?: string[] }> { +async function snapshotFrameForAI(progress: Progress, frame: frames.Frame, options: { track?: string, doNotRenderActive?: boolean } = {}): Promise<{ full: string[], incremental?: string[] }> { // Only await the topmost navigations, inner frames will be empty when racing. const snapshot = await frame.retryWithProgressAndTimeouts(progress, [1000, 2000, 4000, 8000], async continuePolling => { try { @@ -997,7 +997,7 @@ async function snapshotFrameForAI(progress: Progress, frame: frames.Frame, optio if (!node) return true; return injected.incrementalAriaSnapshot(node, { mode: 'ai', ...options }); - }, { refPrefix: frame.seq ? 'f' + frame.seq : '', track: options.track })); + }, { refPrefix: frame.seq ? 'f' + frame.seq : '', track: options.track, doNotRenderActive: options.doNotRenderActive })); if (snapshotOrRetry === true) return continuePolling; return snapshotOrRetry; diff --git a/packages/playwright-core/src/server/progress.ts b/packages/playwright-core/src/server/progress.ts index b6cb8201fb27d..07b580bff6376 100644 --- a/packages/playwright-core/src/server/progress.ts +++ b/packages/playwright-core/src/server/progress.ts @@ -61,10 +61,14 @@ export class ProgressController { const deadline = timeout ? monotonicTime() + timeout : 0; assert(this._state === 'before'); this._state = 'running'; + let timer: NodeJS.Timeout | undefined; const progress: Progress = { timeout: timeout ?? 0, deadline, + disableTimeout: () => { + clearTimeout(timer); + }, log: message => { if (this._state === 'running') this.metadata.log.push(message); @@ -87,10 +91,10 @@ export class ProgressController { signal: this._controller.signal, }; - let timer: NodeJS.Timeout | undefined; if (deadline) { const timeoutError = new TimeoutError(`Timeout ${timeout}ms exceeded.`); timer = setTimeout(() => { + // TODO: migrate this to "progress.disableTimeout()". if (this.metadata.pauseStartTime && !this.metadata.pauseEndTime) return; if (this._state === 'running') { diff --git a/packages/playwright-core/types/types.d.ts b/packages/playwright-core/types/types.d.ts index 6e2a58a63d4d3..50dd490708e13 100644 --- a/packages/playwright-core/types/types.d.ts +++ b/packages/playwright-core/types/types.d.ts @@ -2112,6 +2112,13 @@ export interface Page { cacheOutFile?: string; }; + expect?: { + /** + * Default timeout for expect calls in milliseconds, defaults to 5000ms. + */ + timeout?: number; + }; + /** * Limits to use for the agentic loop. */ @@ -5440,7 +5447,10 @@ export interface PageAgent { maxTokens?: number; /** - * Request timeout in milliseconds. Defaults to action timeout. Pass `0` to disable timeout. + * Expect timeout in milliseconds. Defaults to `5000`. The default value can be changed via `expect.timeout` option in + * the config, or by specifying the `expect` property of the + * [`expect`](https://playwright.dev/docs/api/class-page#page-agent-option-expect) option. Pass `0` to disable + * timeout. */ timeout?: number; }): Promise; @@ -5482,7 +5492,11 @@ export interface PageAgent { maxTokens?: number; /** - * Request timeout in milliseconds. Defaults to action timeout. Pass `0` to disable timeout. + * Perform timeout in milliseconds. Defaults to `5000`. The default value can be changed via `actionTimeout` option in + * the config, or by using the + * [browserContext.setDefaultTimeout(timeout)](https://playwright.dev/docs/api/class-browsercontext#browser-context-set-default-timeout) + * or [page.setDefaultTimeout(timeout)](https://playwright.dev/docs/api/class-page#page-set-default-timeout) methods. + * Pass `0` to disable timeout. */ timeout?: number; }): Promise<{ diff --git a/packages/playwright/src/index.ts b/packages/playwright/src/index.ts index 82b978fb8caf2..874ba50ef2d00 100644 --- a/packages/playwright/src/index.ts +++ b/packages/playwright/src/index.ts @@ -477,6 +477,9 @@ const playwrightFixtures: Fixtures = ({ limits: agentOptions?.limits, secrets: agentOptions?.secrets, systemPrompt: agentOptions?.systemPrompt, + expect: { + timeout: testInfoImpl._projectInternal.expect?.timeout, + }, }); await use(agent); diff --git a/packages/protocol/src/channels.d.ts b/packages/protocol/src/channels.d.ts index a76dce8a907a8..86c22a67751ec 100644 --- a/packages/protocol/src/channels.d.ts +++ b/packages/protocol/src/channels.d.ts @@ -2590,6 +2590,7 @@ export type PageAgentParams = { apiCacheFile?: string, cacheFile?: string, cacheOutFile?: string, + doNotRenderActive?: boolean, maxActions?: number, maxActionRetries?: number, maxTokens?: number, @@ -2605,6 +2606,7 @@ export type PageAgentOptions = { apiCacheFile?: string, cacheFile?: string, cacheOutFile?: string, + doNotRenderActive?: boolean, maxActions?: number, maxActionRetries?: number, maxTokens?: number, diff --git a/packages/protocol/src/progress.d.ts b/packages/protocol/src/progress.d.ts index 8e389e0c31ea4..f7abd096ac5b9 100644 --- a/packages/protocol/src/progress.d.ts +++ b/packages/protocol/src/progress.d.ts @@ -37,6 +37,7 @@ import type { CallMetadata } from './callMetadata'; export interface Progress { timeout: number; deadline: number; + disableTimeout(): void; log(message: string): void; race(promise: Promise | Promise[]): Promise; wait(timeout: number): Promise; // timeout = 0 here means "wait 0 ms", not forever. diff --git a/packages/protocol/src/protocol.yml b/packages/protocol/src/protocol.yml index d4af228adaa12..9a66cd1beef50 100644 --- a/packages/protocol/src/protocol.yml +++ b/packages/protocol/src/protocol.yml @@ -2023,6 +2023,7 @@ Page: apiCacheFile: string? cacheFile: string? cacheOutFile: string? + doNotRenderActive: boolean? maxActions: int? maxActionRetries: int? maxTokens: int? diff --git a/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-generate.json b/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-generate.json index dcab20e88ce3a..af59d69ae2aa2 100644 --- a/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-generate.json +++ b/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-generate.json @@ -5,19 +5,19 @@ "content": [ { "type": "text", - "text": "I need to verify that an input has the value \"hello\", but according to the page snapshot, the textbox currently has the value \"bye\". I'll use the browser_expect_value tool to assert the expected value." + "text": "I need to verify that an input has the value \"hello\". Looking at the page snapshot, I can see there's a textbox with ref=e2 that currently shows \"bye\". I'll use the browser_expect_value tool to assert that this textbox should have the value \"hello\"." }, { "type": "tool_call", "name": "browser_expect_value", "arguments": { "type": "textbox", - "element": "input textbox", + "element": "textbox", "ref": "e2", "value": "hello", "_is_done": true }, - "id": "toolu_017soKmdATMEEVS4yxjbaLs4" + "id": "toolu_011sPLFthuXvN3mxhFhh4yQZ" } ], "stopReason": { @@ -26,7 +26,7 @@ }, "usage": { "input": 2204, - "output": 176 + "output": 191 } } } \ No newline at end of file diff --git a/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-run-from-agent-options.json b/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-run-from-agent-options.json new file mode 100644 index 0000000000000..1109e7715d4e2 --- /dev/null +++ b/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-run-from-agent-options.json @@ -0,0 +1,30 @@ +{ + "44bfbaa6fe0cb038adf37547ecb000d4f39e243f": { + "result": { + "role": "assistant", + "content": [ + { + "type": "text", + "text": "I'll verify that the submit button is visible on the page." + }, + { + "type": "tool_call", + "name": "browser_expect_visible", + "arguments": { + "role": "button", + "accessibleName": "Submit", + "_is_done": true + }, + "id": "toolu_012RgUS8buCjrQRgt4w2sYFf" + } + ], + "stopReason": { + "code": "ok" + } + }, + "usage": { + "input": 2203, + "output": 107 + } + } +} \ No newline at end of file diff --git a/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-run.json b/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-run.json index b4f7c09466e72..790f520de68b1 100644 --- a/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-run.json +++ b/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-run.json @@ -5,7 +5,7 @@ "content": [ { "type": "text", - "text": "I'll verify that the submit button is visible on the page." + "text": "I need to verify that the submit button is visible on the page. Based on the page snapshot, I can see there's a button with the accessible name \"Submit\" with reference e2." }, { "type": "tool_call", @@ -15,7 +15,7 @@ "accessibleName": "Submit", "_is_done": true }, - "id": "toolu_01Luk6EzYXfyshpYHsWqCzfF" + "id": "toolu_01AaRe6HUqqFdEuNGk3rVBQJ" } ], "stopReason": { @@ -24,7 +24,7 @@ }, "usage": { "input": 2203, - "output": 107 + "output": 133 } } } \ No newline at end of file diff --git a/tests/library/__llm_cache__/library-agent-expect-expectTitle-success.json b/tests/library/__llm_cache__/library-agent-expect-expectTitle-success.json index 353522d10b339..0f0ff37ab65b6 100644 --- a/tests/library/__llm_cache__/library-agent-expect-expectTitle-success.json +++ b/tests/library/__llm_cache__/library-agent-expect-expectTitle-success.json @@ -5,7 +5,7 @@ "content": [ { "type": "text", - "text": "I'll verify that the page title is \"My Page Title\" using the browser_expect_title tool." + "text": "I need to verify that the page title is \"My Page Title\". I'll use the browser_expect_title tool to check this." }, { "type": "tool_call", @@ -14,7 +14,7 @@ "title": "My Page Title", "_is_done": true }, - "id": "toolu_01QSwESxdJwhfaSFhvahxFqB" + "id": "toolu_01R1QyAwfxk2KSz3tsAx9RCA" } ], "stopReason": { @@ -23,7 +23,7 @@ }, "usage": { "input": 2195, - "output": 99 + "output": 105 } } } \ No newline at end of file diff --git a/tests/library/__llm_cache__/library-agent-expect-expectTitle-wrong-title-error.json b/tests/library/__llm_cache__/library-agent-expect-expectTitle-wrong-title-error.json index 2227c3d117ec0..a4ae141362c51 100644 --- a/tests/library/__llm_cache__/library-agent-expect-expectTitle-wrong-title-error.json +++ b/tests/library/__llm_cache__/library-agent-expect-expectTitle-wrong-title-error.json @@ -5,7 +5,7 @@ "content": [ { "type": "text", - "text": "I need to verify that the page title is \"Other Title\". I'll use the browser_expect_title tool to assert this condition." + "text": "I need to verify that the page title is \"Other Title\" using one of the browser_expect_* tools." }, { "type": "tool_call", @@ -14,7 +14,7 @@ "title": "Other Title", "_is_done": true }, - "id": "toolu_01BnWsHUg4s787xMJYmLUNDn" + "id": "toolu_017ZvXqNbcab9Uyjzw3VuByK" } ], "stopReason": { @@ -23,7 +23,7 @@ }, "usage": { "input": 2194, - "output": 104 + "output": 100 } } } \ No newline at end of file diff --git a/tests/library/__llm_cache__/library-agent-expect-expectURL-success.json b/tests/library/__llm_cache__/library-agent-expect-expectURL-success.json index 453653504d967..56fadb43dc1e4 100644 --- a/tests/library/__llm_cache__/library-agent-expect-expectURL-success.json +++ b/tests/library/__llm_cache__/library-agent-expect-expectURL-success.json @@ -1,20 +1,20 @@ { - "74935765e910e2883dc5925fd5f947bb528f174d": { + "5fd07fcf484a378d71b68dc09227463b80797da0": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "I see the issue - the URL contains a server prefix. Let me use a regex pattern to match the URL ending with /counter.html." + "text": "I'll verify that the page URL is /counter.html." }, { "type": "tool_call", "name": "browser_expect_url", "arguments": { - "regex": "/counter\\.html$/", + "url": "/counter.html", "_is_done": true }, - "id": "toolu_018XaEFPSm6aLzqA1BBqRNhu" + "id": "toolu_01PkjsZaM5eEGtjLtMixyJUk" } ], "stopReason": { @@ -22,26 +22,26 @@ } }, "usage": { - "input": 2447, - "output": 108 + "input": 2240, + "output": 91 } }, - "ee1b825529efd3c64fff821e9ce6f2e06433f6d3": { + "71a7a4f51b3994a200f239eda9b505454825a79f": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "I'll verify that the page URL is /counter.html using the browser_expect_url tool." + "text": "I need to use a regex pattern to match the URL since it includes a server prefix. Let me update the expectation to match the actual URL pattern." }, { "type": "tool_call", "name": "browser_expect_url", "arguments": { - "url": "/counter.html", + "regex": "/\\/counter\\.html$/", "_is_done": true }, - "id": "toolu_01PRTpTFD78UKubB5EzwTAii" + "id": "toolu_01Wwq7Pbe2DthqpmskgGfCzk" } ], "stopReason": { @@ -49,8 +49,8 @@ } }, "usage": { - "input": 2243, - "output": 99 + "input": 2433, + "output": 111 } } } \ No newline at end of file diff --git a/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex-error.json b/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex-error.json index 7e0cc5425c484..3f76a92de99db 100644 --- a/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex-error.json +++ b/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex-error.json @@ -1,38 +1,11 @@ { - "37de212b569c4a2d8412505c647ce7ad1f824387": { + "f0db973fe7c770b8bdb9ae3c8999544a3305b4b3": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "I need to verify that the page URL matches the `/counter` pattern. I'll use the `browser_expect_url` tool with a regex pattern." - }, - { - "type": "tool_call", - "name": "browser_expect_url", - "arguments": { - "regex": "/counter", - "_is_done": true - }, - "id": "toolu_01NFN4Fz1V3Bek9pKTbuEsBg" - } - ], - "stopReason": { - "code": "ok" - } - }, - "usage": { - "input": 2242, - "output": 108 - } - }, - "a786ca3af437fcb35753c1f7ceb7f9a90bc8deeb": { - "result": { - "role": "assistant", - "content": [ - { - "type": "text", - "text": "I need to fix the regex pattern format. It should end with a `/` and optional flags." + "text": "I'll verify that the page URL matches the /counter pattern using the browser_expect_url tool with a regex pattern." }, { "type": "tool_call", @@ -41,7 +14,7 @@ "regex": "/counter/", "_is_done": true }, - "id": "toolu_01SLaSETGg54hyKNATWckQVw" + "id": "toolu_01JxsK661QmxpScYeTMCfWTZ" } ], "stopReason": { @@ -49,8 +22,8 @@ } }, "usage": { - "input": 2440, - "output": 97 + "input": 2239, + "output": 102 } } } \ No newline at end of file diff --git a/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex.json b/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex.json index dd48e0d5fdcda..c8ae66692ec94 100644 --- a/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex.json +++ b/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex.json @@ -1,11 +1,11 @@ { - "37de212b569c4a2d8412505c647ce7ad1f824387": { + "f0db973fe7c770b8bdb9ae3c8999544a3305b4b3": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "I need to verify that the page URL matches the /counter pattern using a browser_expect assertion tool." + "text": "I need to verify that the page URL matches the `/counter` pattern. I'll use the `browser_expect_url` tool with a regex pattern." }, { "type": "tool_call", @@ -14,7 +14,7 @@ "regex": "/counter/", "_is_done": true }, - "id": "toolu_015RtjMDCtGsTUpu36HUk5fC" + "id": "toolu_01YVkbby1j3FBXk2o5Bi4cjP" } ], "stopReason": { @@ -22,8 +22,8 @@ } }, "usage": { - "input": 2242, - "output": 98 + "input": 2239, + "output": 109 } } } \ No newline at end of file diff --git a/tests/library/__llm_cache__/library-agent-expect-expectURL-wrong-URL-error.json b/tests/library/__llm_cache__/library-agent-expect-expectURL-wrong-URL-error.json index 353b806e0961c..b88a2b2d35cfc 100644 --- a/tests/library/__llm_cache__/library-agent-expect-expectURL-wrong-URL-error.json +++ b/tests/library/__llm_cache__/library-agent-expect-expectURL-wrong-URL-error.json @@ -1,20 +1,20 @@ { - "688127ac19026114a89d4e621b86bdeaa699418e": { + "5fd07fcf484a378d71b68dc09227463b80797da0": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "I see the issue - the actual URL includes the server prefix. Let me use a regex pattern to match the URL ending with /counter.html." + "text": "I'll verify that the page URL is /counter.html using the browser_expect_url tool." }, { "type": "tool_call", "name": "browser_expect_url", "arguments": { - "regex": "/\\/counter\\.html$/", + "url": "/counter.html", "_is_done": true }, - "id": "toolu_01V6sRugMvahkP21AHaffA11" + "id": "toolu_01BrgLhGzD2dYWeu75KGgxKv" } ], "stopReason": { @@ -22,26 +22,26 @@ } }, "usage": { - "input": 2447, - "output": 110 + "input": 2240, + "output": 99 } }, - "ee1b825529efd3c64fff821e9ce6f2e06433f6d3": { + "8315bc81a0dfc915cb8be72ad00102f627e370dd": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "I'll verify that the page URL is /counter.html using the browser_expect_url tool." + "text": "I see the issue. The actual URL includes the server portion. Let me use a regex pattern to match the URL ending with /counter.html." }, { "type": "tool_call", "name": "browser_expect_url", "arguments": { - "url": "/counter.html", + "regex": "/\\/counter\\.html$/", "_is_done": true }, - "id": "toolu_014UE4ajKxYwk8kM6MsURpks" + "id": "toolu_01KtWVz2EdJjx2V9viYeqs71" } ], "stopReason": { @@ -49,8 +49,8 @@ } }, "usage": { - "input": 2243, - "output": 99 + "input": 2441, + "output": 110 } } } \ No newline at end of file diff --git a/tests/library/__llm_cache__/library-agent-limits-should-respect-max-actions-limit.json b/tests/library/__llm_cache__/library-agent-limits-should-respect-max-actions-limit.json index c4e3d1c4c89e2..a1046b18663c2 100644 --- a/tests/library/__llm_cache__/library-agent-limits-should-respect-max-actions-limit.json +++ b/tests/library/__llm_cache__/library-agent-limits-should-respect-max-actions-limit.json @@ -1,11 +1,11 @@ { - "1b776f266d622f4553281b575e36acd4862f48b4": { + "37067acf73704df3b84beb06e5caed3eb9bf18ae": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "Now clicking the second time." + "text": "Now clicking the submit button for the second time." }, { "type": "tool_call", @@ -15,7 +15,7 @@ "ref": "e2", "_is_done": false }, - "id": "toolu_014LLfN7tHFxcAqX72VTHXVy" + "id": "toolu_01HrvXLTTXvo9BModqvnKT9X" } ], "stopReason": { @@ -23,17 +23,17 @@ } }, "usage": { - "input": 3194, - "output": 98 + "input": 3191, + "output": 102 } }, - "7b25e318de64dc9d2494432a6d311631e2c1edc0": { + "42f3f98abb30b1cd490e8afa7ed7f62dd63a0355": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "I'll click the submit button 5 times as requested. Let me start by clicking it the first time." + "text": "Clicking the submit button for the fourth time." }, { "type": "tool_call", @@ -43,7 +43,7 @@ "ref": "e2", "_is_done": false }, - "id": "toolu_01PWwDgpp2rUtvmpuTfgTNzn" + "id": "toolu_01Ac6wP23fB9JAMqdXJ4iHo9" } ], "stopReason": { @@ -51,17 +51,17 @@ } }, "usage": { - "input": 3047, - "output": 115 + "input": 3342, + "output": 102 } }, - "b9bb21161597f576eaef8aea85dae797c879fec9": { + "7b25e318de64dc9d2494432a6d311631e2c1edc0": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "Clicking the fourth time." + "text": "I'll click the submit button 5 times as requested. Let me start by clicking it the first time." }, { "type": "tool_call", @@ -71,7 +71,7 @@ "ref": "e2", "_is_done": false }, - "id": "toolu_01YRh2G5UeZqvPou8d311SZU" + "id": "toolu_013EzLkuM1hq5oKwFdwsfXgW" } ], "stopReason": { @@ -79,17 +79,17 @@ } }, "usage": { - "input": 3337, - "output": 98 + "input": 3047, + "output": 115 } }, - "bae9fc31434e4f1f36e5cd29bd84165e04ac32f2": { + "b0b9f5d4d75488a8be0bd393e38396f58e8bae3b": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "Clicking the third time." + "text": "Clicking the submit button for the third time." }, { "type": "tool_call", @@ -99,7 +99,7 @@ "ref": "e2", "_is_done": false }, - "id": "toolu_01JnLrvVrm3EzPwcBrzzDR1S" + "id": "toolu_01RC27VmiERhtpWN8DvhEXvg" } ], "stopReason": { @@ -107,8 +107,8 @@ } }, "usage": { - "input": 3267, - "output": 98 + "input": 3268, + "output": 102 } } } \ No newline at end of file diff --git a/tests/library/__llm_cache__/library-agent-perform-click-a-button.json b/tests/library/__llm_cache__/library-agent-perform-click-a-button.json index a20b50d717b5f..1759105d6a121 100644 --- a/tests/library/__llm_cache__/library-agent-perform-click-a-button.json +++ b/tests/library/__llm_cache__/library-agent-perform-click-a-button.json @@ -15,7 +15,7 @@ "ref": "e2", "_is_done": true }, - "id": "toolu_01VL7BkrHmWKoBoPpEqgdXBM" + "id": "toolu_011oz46csVKDo4fGJBzPX6Pu" } ], "stopReason": { diff --git a/tests/library/__llm_cache__/library-agent-perform-empty-cache-file-works.json b/tests/library/__llm_cache__/library-agent-perform-empty-cache-file-works.json new file mode 100644 index 0000000000000..2709b86f58861 --- /dev/null +++ b/tests/library/__llm_cache__/library-agent-perform-empty-cache-file-works.json @@ -0,0 +1,30 @@ +{ + "29a2028099170f3ab333459d1f9c78de2767e7b3": { + "result": { + "role": "assistant", + "content": [ + { + "type": "text", + "text": "I'll click the Test button for you." + }, + { + "type": "tool_call", + "name": "browser_click", + "arguments": { + "element": "Test button", + "ref": "e2", + "_is_done": true + }, + "id": "toolu_013jBJjsaeAjvjyjfWLYNFGb" + } + ], + "stopReason": { + "code": "ok" + } + }, + "usage": { + "input": 3043, + "output": 101 + } + } +} \ No newline at end of file diff --git a/tests/library/__llm_cache__/library-agent-perform-expect-value.json b/tests/library/__llm_cache__/library-agent-perform-expect-value.json index 0f6642a0c1912..3045824187285 100644 --- a/tests/library/__llm_cache__/library-agent-perform-expect-value.json +++ b/tests/library/__llm_cache__/library-agent-perform-expect-value.json @@ -16,7 +16,7 @@ "text": "bogus", "_is_done": false }, - "id": "toolu_01EHNuYTEoREY8mpiB8TDpao" + "id": "toolu_01Gpg2Zqam6FBo1hga4nU5m2" } ], "stopReason": { @@ -28,13 +28,13 @@ "output": 149 } }, - "de97ea962d4424a77d06eeb1f87c4fca0072f5b0": { + "30aa72b75cada58032f8e66e544580c2fa37b9f8": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "Perfect! I have successfully completed all three parts of the task:\n\n1. ✅ Entered \"bogus\" into the email field\n2. ✅ Verified the value is \"bogus\" (shown in the page snapshot as `textbox \"Email Address\" [active] [ref=e2]: bogus`)\n3. ✅ Confirmed the error message is displayed (shown as `generic [ref=e3]: \"Error: Invalid email address\"`)\n\nThe task has been completed successfully. The email field now contains \"bogus\" and the validation error message \"Error: Invalid email address\" is displayed on the page." + "text": "Perfect! I have successfully completed all three parts of the task:\n\n1. ✓ Entered \"bogus\" into the email field\n2. ✓ Verified the value is \"bogus\" (shown in the snapshot as `textbox \"Email Address\" [ref=e2]: bogus`)\n3. ✓ Confirmed the error message is displayed (shown as `\"Error: Invalid email address\"`)\n\nAll requirements have been met. The task is complete." }, { "type": "tool_call", @@ -42,7 +42,7 @@ "arguments": { "_is_done": true }, - "id": "toolu_01RF8r5xHaVkEMKMw8DdLRtX" + "id": "toolu_01K9CVYgsXmZxyvf3S8hvEag" } ], "stopReason": { @@ -50,8 +50,8 @@ } }, "usage": { - "input": 3288, - "output": 196 + "input": 3285, + "output": 162 } } } \ No newline at end of file diff --git a/tests/library/__llm_cache__/library-agent-perform-extract-task.json b/tests/library/__llm_cache__/library-agent-perform-extract-task.json index 44e82a8f961c1..23ad40342d5f3 100644 --- a/tests/library/__llm_cache__/library-agent-perform-extract-task.json +++ b/tests/library/__llm_cache__/library-agent-perform-extract-task.json @@ -23,7 +23,7 @@ ], "_is_done": true }, - "id": "toolu_01S326B2WNaGGZsxcdufA6ub" + "id": "toolu_01DvvdfNSormjy1wYd3tsX9B" } ], "stopReason": { @@ -59,7 +59,7 @@ ], "_is_done": true }, - "id": "toolu_017VDE6U6W8WedvBTxDLg3mf" + "id": "toolu_012Xfc5Au8c5bdhdNA9tmqFe" } ], "stopReason": { @@ -68,7 +68,7 @@ }, "usage": { "input": 1020, - "output": 132 + "output": 113 } } } \ No newline at end of file diff --git a/tests/library/__llm_cache__/library-agent-perform-perform-history.json b/tests/library/__llm_cache__/library-agent-perform-perform-history.json index 068a283179e95..69362c3ece709 100644 --- a/tests/library/__llm_cache__/library-agent-perform-perform-history.json +++ b/tests/library/__llm_cache__/library-agent-perform-perform-history.json @@ -1,5 +1,5 @@ { - "190ffa76902eab71ce416fea1894aa82dca2b883": { + "35faa6e330f5198be604e1d5b81cd0c1c1627a63": { "result": { "role": "assistant", "content": [ @@ -15,7 +15,7 @@ "ref": "e3", "_is_done": true }, - "id": "toolu_01ERb3Y3rLpQy6VCBtT9Lcsr" + "id": "toolu_01GmGw8peNBC1zWhPVjTM3UU" } ], "stopReason": { @@ -23,11 +23,11 @@ } }, "usage": { - "input": 3096, + "input": 3093, "output": 102 } }, - "500794034629b9a73d75bdb7ab58e72bf8adfca2": { + "d98ecb24c7732360ace82e5dabac9c9bf3266a71": { "result": { "role": "assistant", "content": [ @@ -43,7 +43,7 @@ "ref": "e3", "_is_done": true }, - "id": "toolu_01Tax2k8PnfvQvRj2s23Mc6m" + "id": "toolu_011PFYBHb9wVKpJLoMmNDNsz" } ], "stopReason": { @@ -51,7 +51,7 @@ } }, "usage": { - "input": 3083, + "input": 3080, "output": 101 } } diff --git a/tests/library/__llm_cache__/library-agent-perform-perform-reports-error.json b/tests/library/__llm_cache__/library-agent-perform-perform-reports-error.json index 99e0a69fa6b76..33bc8a034197e 100644 --- a/tests/library/__llm_cache__/library-agent-perform-perform-reports-error.json +++ b/tests/library/__llm_cache__/library-agent-perform-perform-reports-error.json @@ -1,20 +1,19 @@ { - "af3d8747bed52754774ce6409828a7ecbf22c747": { + "4804d46618cdfe9201831d9757cc2478ba4d3909": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "I can see the page snapshot only shows \"Wolf\" and \"Fox\" buttons, but not a \"Rabbit\" button. The Rabbit button is not currently visible in the page snapshot. This could mean:\n1. The button doesn't exist on the page\n2. The button is not yet loaded\n3. The button might appear after some interaction\n\nSince the task specifically asks me to click the Rabbit button but it's not present in the snapshot, I should report this issue." + "text": "I'll take a snapshot of the current page to locate the Rabbit button." }, { "type": "tool_call", - "name": "refuse_to_perform", + "name": "browser_snapshot", "arguments": { - "reason": "The Rabbit button is not present on the page. The page only contains \"Wolf\" and \"Fox\" buttons. The requested \"Rabbit\" button cannot be found in the current page snapshot.", - "_is_done": true + "_is_done": false }, - "id": "toolu_01WnpKKi4FUnGyFrRwi15Bmx" + "id": "toolu_012JEEitpPzZfnUeD8Wfa9Z3" } ], "stopReason": { @@ -22,25 +21,26 @@ } }, "usage": { - "input": 3203, - "output": 214 + "input": 3067, + "output": 72 } }, - "fb3d817d917f930cc19453843ed86fe6d4ad4e98": { + "889157cd583283667b9fd5d9add230316106c3a4": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "I need to take a snapshot of the page first to see the current state and locate the Rabbit button." + "text": "I can see the page snapshot only shows \"Wolf\" and \"Fox\" buttons, but there is no \"Rabbit\" button visible on the current page. Let me refuse to perform this action as the requested button is not available." }, { "type": "tool_call", - "name": "browser_snapshot", + "name": "refuse_to_perform", "arguments": { - "_is_done": false + "reason": "The Rabbit button is not present on the page. Only Wolf and Fox buttons are visible in the current page snapshot.", + "_is_done": true }, - "id": "toolu_01KnyXDHzhX6wn69vmG24yDa" + "id": "toolu_01RA82xhXBfG6hbJY8y1q278" } ], "stopReason": { @@ -48,8 +48,8 @@ } }, "usage": { - "input": 3070, - "output": 78 + "input": 3191, + "output": 144 } } } \ No newline at end of file diff --git a/tests/library/__llm_cache__/library-agent-perform-perform-run-timeout-inherited-from-page.json b/tests/library/__llm_cache__/library-agent-perform-perform-run-timeout-inherited-from-page.json new file mode 100644 index 0000000000000..1cd1ae76339cf --- /dev/null +++ b/tests/library/__llm_cache__/library-agent-perform-perform-run-timeout-inherited-from-page.json @@ -0,0 +1,30 @@ +{ + "554366043f6477d30a36655c10cbb4ec0eaa21d4": { + "result": { + "role": "assistant", + "content": [ + { + "type": "text", + "text": "I'll click the Fox button for you." + }, + { + "type": "tool_call", + "name": "browser_click", + "arguments": { + "element": "Fox button", + "ref": "e3", + "_is_done": true + }, + "id": "toolu_01TBMe2oPqGUpNuPnFnDeZJe" + } + ], + "stopReason": { + "code": "ok" + } + }, + "usage": { + "input": 3066, + "output": 101 + } + } +} \ No newline at end of file diff --git a/tests/library/__llm_cache__/library-agent-perform-perform-run-timeout.json b/tests/library/__llm_cache__/library-agent-perform-perform-run-timeout.json index ecb06f8d8fc2d..4eea493157517 100644 --- a/tests/library/__llm_cache__/library-agent-perform-perform-run-timeout.json +++ b/tests/library/__llm_cache__/library-agent-perform-perform-run-timeout.json @@ -1,5 +1,5 @@ { - "273f0802e7fc3fe0d3eb1cfa83366e1a1272f185": { + "554366043f6477d30a36655c10cbb4ec0eaa21d4": { "result": { "role": "assistant", "content": [ @@ -15,7 +15,7 @@ "ref": "e3", "_is_done": true }, - "id": "toolu_01YVFbgqSfhDKsS843HKxwor" + "id": "toolu_01XMd7RXQygLwiLd7eiPB9iV" } ], "stopReason": { @@ -23,7 +23,7 @@ } }, "usage": { - "input": 3069, + "input": 3066, "output": 101 } } diff --git a/tests/library/__llm_cache__/library-agent-perform-retrieve-a-secret.json b/tests/library/__llm_cache__/library-agent-perform-retrieve-a-secret.json index 99968d5392bce..3e44a2ca2b811 100644 --- a/tests/library/__llm_cache__/library-agent-perform-retrieve-a-secret.json +++ b/tests/library/__llm_cache__/library-agent-perform-retrieve-a-secret.json @@ -5,7 +5,7 @@ "content": [ { "type": "text", - "text": "I'll enter the x-secret-email into the email field." + "text": "I'll help you enter the secret email into the email field. Let me type the value into the Email Address textbox." }, { "type": "tool_call", @@ -16,7 +16,7 @@ "text": "%x-secret-email%", "_is_done": true }, - "id": "toolu_019NPT8YwSF9eaXHgPMRGAMZ" + "id": "toolu_01LxAKq2etJEHtoUe5pURHoa" } ], "stopReason": { @@ -25,7 +25,7 @@ }, "usage": { "input": 3051, - "output": 133 + "output": 144 } } } \ No newline at end of file diff --git a/tests/library/agent-expect.spec.ts b/tests/library/agent-expect.spec.ts index c7befd887ab86..f301a6c5ab1b9 100644 --- a/tests/library/agent-expect.spec.ts +++ b/tests/library/agent-expect.spec.ts @@ -196,16 +196,36 @@ test('expect timeout during run', async ({ context }) => { { const { page, agent } = await runAgent(context); await page.setContent(``); - const error = await agent.expect('submit button is visible', { timeout: 10000 }).catch(e => e); + const error = await agent.expect('submit button is visible', { timeout: 3000 }).catch(e => e); expect(stripAnsi(error.message)).toContain(`pageAgent.expect: expect(locator).toBeVisible() failed Locator: getByRole('button', { name: 'Submit' }) Expected: visible -Timeout: 5000ms +Timeout: 3000ms Error: element(s) not found Call log: - - Expect Visible with timeout 5000ms`); + - Expect Visible with timeout 3000ms`); + } +}); + +test('expect timeout during run from agent options', async ({ context }) => { + { + const { page, agent } = await generateAgent(context); + await page.setContent(``); + await agent.expect('submit button is visible'); + } + expect(await cacheObject()).toEqual({ + 'submit button is visible': { + actions: [expect.objectContaining({ method: 'expectVisible' })], + }, + }); + { + const { page, agent } = await runAgent(context, { expect: { timeout: 3000 } }); + await page.setContent(``); + const error = await agent.expect('submit button is visible').catch(e => e); + expect(stripAnsi(error.message)).toContain(`pageAgent.expect: expect(locator).toBeVisible() failed`); + expect(stripAnsi(error.message)).toContain(`Expect Visible with timeout 3000ms`); } }); diff --git a/tests/library/agent-helpers.ts b/tests/library/agent-helpers.ts index b145b1882d007..5a718082df48d 100644 --- a/tests/library/agent-helpers.ts +++ b/tests/library/agent-helpers.ts @@ -42,22 +42,25 @@ export async function generateAgent(context: BrowserContext, options: AgentOptio provider: { api: 'anthropic' as const, apiKey: process.env.AZURE_SONNET_API_KEY ?? 'dummy', - apiEndpoint: process.env.AZURE_SONNET_ENDPOINT ?? 'dummy', + apiEndpoint: process.env.AZURE_SONNET_ENDPOINT, model: 'claude-sonnet-4-5', ...{ _apiCacheFile: apiCacheFile } }, ...options, cache: { cacheFile: cacheFile(), - } + }, + ...{ _doNotRenderActive: true }, }); return { page, agent }; } -export async function runAgent(context: BrowserContext, options: { secrets?: Record } = {}) { +export async function runAgent(context: BrowserContext, options: AgentOptions = {}) { const page = await context.newPage(); const agent = await page.agent({ - cache: { cacheFile: cacheFile() } + ...options, + cache: { cacheFile: cacheFile() }, + ...{ _doNotRenderActive: true }, }); return { page, agent }; } diff --git a/tests/library/agent-perform.spec.ts b/tests/library/agent-perform.spec.ts index ba59b29658f99..9a14b409a20df 100644 --- a/tests/library/agent-perform.spec.ts +++ b/tests/library/agent-perform.spec.ts @@ -16,9 +16,10 @@ import { z as zod3 } from 'zod/v3'; import * as zod4 from 'zod'; +import fs from 'fs'; import { browserTest as test, expect } from '../config/browserTest'; -import { run, generateAgent, cacheObject, runAgent, setCacheObject } from './agent-helpers'; +import { run, generateAgent, cacheObject, runAgent, setCacheObject, cacheFile } from './agent-helpers'; // LOWIRE_NO_CACHE=1 to generate api caches // LOWIRE_FORCE_CACHE=1 to force api caches @@ -173,6 +174,28 @@ test('perform run timeout', async ({ context }) => { } }); +test('perform run timeout inherited from page', async ({ context }) => { + { + const { page, agent } = await generateAgent(context); + await page.setContent(` + + + `); + await agent.perform('click the Fox button'); + } + { + const { page, agent } = await runAgent(context); + await page.setContent(` + + + `); + page.setDefaultTimeout(3000); + const error = await agent.perform('click the Fox button').catch(e => e); + expect(error.message).toContain('Timeout 3000ms exceeded.'); + expect(error.message).toContain(`waiting for getByRole('button', { name: 'Fox' })`); + } +}); + test('invalid cache file throws error', async ({ context }) => { await setCacheObject({ 'some key': { @@ -191,6 +214,33 @@ Failed to parse cache file ${test.info().outputPath('agent-cache.json')}: `.trim()); }); +test('non-json cache file throws a nice error', async ({ context }) => { + await fs.promises.writeFile(cacheFile(), 'bogus', 'utf8'); + const { agent } = await runAgent(context); + const error = await agent.perform('click the Test button').catch(e => e); + expect(error.message).toContain(`Failed to parse cache file ${test.info().outputPath('agent-cache.json')}:`); + expect(error.message.toLowerCase()).toContain(`valid json`); +}); + +test('empty cache file works', async ({ context }) => { + await fs.promises.writeFile(cacheFile(), '', 'utf8'); + const { page, agent } = await generateAgent(context); + await page.setContent(``); + await agent.perform('click the Test button'); +}); + +test('missing apiKey throws a nice error', async ({ page }) => { + const agent = await page.agent({ provider: { api: 'anthropic', model: 'some model' } as any }); + const error = await agent.perform('click the Test button').catch(e => e); + expect(error.message).toContain(`This action requires API key to be set on the page agent`); +}); + +test('malformed apiEndpoint throws a nice error', async ({ page }) => { + const agent = await page.agent({ provider: { api: 'anthropic', model: 'some model', apiKey: 'some key', apiEndpoint: 'foobar' } }); + const error = await agent.perform('click the Test button').catch(e => e); + expect(error.message).toContain(`Agent API endpoint "foobar" is not a valid URL`); +}); + test('perform reports error', async ({ context }) => { const { page, agent } = await generateAgent(context); await page.setContent(`