diff --git a/docs/src/api/class-page.md b/docs/src/api/class-page.md index 45a0e2dc1cf90..5c2b7521971d8 100644 --- a/docs/src/api/class-page.md +++ b/docs/src/api/class-page.md @@ -722,15 +722,14 @@ Initialize page agent with the llm provider and cache. - `cacheFile` ?<[string]> Cache file to use/generate code for performed actions into. Cache is not used if not specified (default). - `cacheOutFile` ?<[string]> When specified, generated entries are written into the `cacheOutFile` instead of updating the `cacheFile`. -### option: Page.agent.maxTurns +### option: Page.agent.limits * since: v1.58 -- `maxTurns` <[int]> +- `limits` <[Object]> + - `maxTokens` ?<[int]> Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value. Defaults to unlimited. + - `maxActions` ?<[int]> Maximum number of agentic actions to generate, defaults to 10. + - `maxActionRetries` ?<[int]> Maximum number retries per action, defaults to 3. -Maximum number of agentic turns to take per call. Defaults to 10. - -### option: Page.agent.maxTokens -* since: v1.58 -- `maxTokens` ?<[int]> +Limits to use for the agentic loop. ### option: Page.agent.provider * since: v1.58 @@ -738,6 +737,7 @@ Maximum number of agentic turns to take per call. Defaults to 10. - `api` <[PageAgentAPI]<"openai"|"openai-compatible"|"anthropic"|"google">> API to use. - `apiEndpoint` ?<[string]> Endpoint to use if different from default. - `apiKey` <[string]> API key for the LLM provider. + - `apiTimeout` ?<[int]> Amount of time to wait for the provider to respond to each request. - `model` <[string]> Model identifier within the provider. Required in non-cache mode. ### option: Page.agent.secrets diff --git a/docs/src/api/params.md b/docs/src/api/params.md index bd319b19ad220..9b7bf8f3d4af4 100644 --- a/docs/src/api/params.md +++ b/docs/src/api/params.md @@ -384,11 +384,17 @@ By default, they are cached globally with the `task` as a key. This option allow Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value. Defaults to context-wide value specified in `agent` property. -## page-agent-max-turns +## page-agent-max-actions * since: v1.58 -- `maxTurns` <[int]> +- `maxActions` <[int]> -Maximum number of agentic turns during this call, defaults to context-wide value specified in `agent` property. +Maximum number of agentic actions to generate, defaults to context-wide value specified in `agent` property. + +## page-agent-max-action-retries +* since: v1.58 +- `maxActionRetries` <[int]> + +Maximum number of retries when generating each action, defaults to context-wide value specified in `agent` property. ## page-agent-timeout * since: v1.58 @@ -399,7 +405,8 @@ Request timeout in milliseconds. Defaults to action timeout. Pass `0` to disable ## page-agent-call-options-v1.58 - %%-page-agent-cache-key-%% - %%-page-agent-max-tokens-%% -- %%-page-agent-max-turns-%% +- %%-page-agent-max-actions-%% +- %%-page-agent-max-action-retries-%% - %%-page-agent-timeout-%% ## fetch-param-url diff --git a/docs/src/test-api/class-testoptions.md b/docs/src/test-api/class-testoptions.md index 194cd58616994..4c60a5c89617d 100644 --- a/docs/src/test-api/class-testoptions.md +++ b/docs/src/test-api/class-testoptions.md @@ -49,14 +49,19 @@ export default defineConfig({ ## property: TestOptions.agentOptions * since: v1.58 - type: <[Object]> - - `api` ?<[string]> LLM provider to use. Required in non-cache mode. - - `apiKey` ?<[string]> Key for the LLM provider. - - `apiEndpoint` ?<[string]> LLM provider endpoint. - - `model` ?<[string]> Model identifier within the provider. Required in non-cache mode. + - `provider` <[Object]> + - `api` <[PageAgentAPI]<"openai"|"openai-compatible"|"anthropic"|"google">> API to use. + - `apiEndpoint` ?<[string]> Endpoint to use if different from default. + - `apiKey` <[string]> API key for the LLM provider. + - `apiTimeout` ?<[int]> Amount of time to wait for the provider to respond to each request. + - `model` <[string]> Model identifier within the provider. Required in non-cache mode. - `cachePathTemplate` ?<[string]> Cache file template to use/generate code for performed actions into. - - `maxTurns` ?<[int]> Maximum number of agentic turns to take per call. Defaults to 10. - - `maxTokens` ?<[int]> Maximum number of tokens to consume per call. The agentic loop will stop after input + output tokens exceed this value. Defaults on unlimited. + - `limits` <[Object]> + - `maxTokens` ?<[int]> Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value. Defaults to unlimited. + - `maxActions` ?<[int]> Maximum number of agentic actions to generate, defaults to 10. + - `maxActionRetries` ?<[int]> Maximum number retries per action, defaults to 3. - `secrets` ?<[Object]<[string], [string]>> Secrets to hide from the LLM. + - `systemPrompt` <[string]> System prompt for the agent's loop. ## property: TestOptions.baseURL = %%-context-option-baseURL-%% * since: v1.10 diff --git a/examples/todomvc/tests/fixtures.ts b/examples/todomvc/tests/fixtures.ts index 0e317c47065e9..6226e718bf64f 100644 --- a/examples/todomvc/tests/fixtures.ts +++ b/examples/todomvc/tests/fixtures.ts @@ -6,10 +6,12 @@ export { expect } from '@playwright/test'; export const test = baseTest.extend({ agentOptions: { - api: 'anthropic', - apiKey: process.env.AZURE_SONNET_API_KEY!, - apiEndpoint: process.env.AZURE_SONNET_ENDPOINT!, - model: 'claude-sonnet-4-5', + provider: { + api: 'anthropic', + apiKey: process.env.AZURE_SONNET_API_KEY!, + apiEndpoint: process.env.AZURE_SONNET_ENDPOINT!, + model: 'claude-sonnet-4-5', + }, }, page: async ({ page }, use) => { await page.goto('https://demo.playwright.dev/todomvc'); diff --git a/package-lock.json b/package-lock.json index e3cd66d46e890..5160eea979cae 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,7 +18,7 @@ "@eslint/compat": "^1.3.2", "@eslint/eslintrc": "^3.3.1", "@eslint/js": "^9.34.0", - "@lowire/loop": "^0.0.19", + "@lowire/loop": "^0.0.23", "@modelcontextprotocol/sdk": "^1.17.5", "@octokit/graphql-schema": "^15.26.0", "@stylistic/eslint-plugin": "^5.2.3", @@ -1065,9 +1065,9 @@ } }, "node_modules/@lowire/loop": { - "version": "0.0.19", - "resolved": "https://registry.npmjs.org/@lowire/loop/-/loop-0.0.19.tgz", - "integrity": "sha512-uugjYtmA1mNSwHzbZEnvFedjB2EUX0s4NDbDbUcGuz6CH8fzZV2FFodY6fneik+r3GdLJtpvzpOP+cU/M0ba3w==", + "version": "0.0.23", + "resolved": "https://registry.npmjs.org/@lowire/loop/-/loop-0.0.23.tgz", + "integrity": "sha512-zw4yFzto2T9g+CiHfubzxut0JMxA053XQ2syveqPjqofvRZyumhJR11RIQjKrPO9rDbf2CFviKrYFt7kgru1VA==", "dev": true, "license": "Apache-2.0", "engines": { diff --git a/package.json b/package.json index 8db7c35f515c8..8138c730d905d 100644 --- a/package.json +++ b/package.json @@ -55,7 +55,7 @@ "@eslint/compat": "^1.3.2", "@eslint/eslintrc": "^3.3.1", "@eslint/js": "^9.34.0", - "@lowire/loop": "^0.0.19", + "@lowire/loop": "^0.0.23", "@modelcontextprotocol/sdk": "^1.17.5", "@octokit/graphql-schema": "^15.26.0", "@stylistic/eslint-plugin": "^5.2.3", diff --git a/packages/playwright-client/types/types.d.ts b/packages/playwright-client/types/types.d.ts index a9d3c3daff9fa..51dd92fb293b4 100644 --- a/packages/playwright-client/types/types.d.ts +++ b/packages/playwright-client/types/types.d.ts @@ -2110,12 +2110,26 @@ export interface Page { cacheOutFile?: string; }; - maxTokens?: number; - /** - * Maximum number of agentic turns to take per call. Defaults to 10. + * Limits to use for the agentic loop. */ - maxTurns?: number; + limits?: { + /** + * Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value. + * Defaults to unlimited. + */ + maxTokens?: number; + + /** + * Maximum number of agentic actions to generate, defaults to 10. + */ + maxActions?: number; + + /** + * Maximum number retries per action, defaults to 3. + */ + maxActionRetries?: number; + }; provider?: { /** @@ -2133,6 +2147,11 @@ export interface Page { */ apiKey: string; + /** + * Amount of time to wait for the provider to respond to each request. + */ + apiTimeout?: number; + /** * Model identifier within the provider. Required in non-cache mode. */ @@ -5402,15 +5421,21 @@ export interface PageAgent { cacheKey?: string; /** - * Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value. - * Defaults to context-wide value specified in `agent` property. + * Maximum number of retries when generating each action, defaults to context-wide value specified in `agent` + * property. */ - maxTokens?: number; + maxActionRetries?: number; + + /** + * Maximum number of agentic actions to generate, defaults to context-wide value specified in `agent` property. + */ + maxActions?: number; /** - * Maximum number of agentic turns during this call, defaults to context-wide value specified in `agent` property. + * Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value. + * Defaults to context-wide value specified in `agent` property. */ - maxTurns?: number; + maxTokens?: number; /** * Request timeout in milliseconds. Defaults to action timeout. Pass `0` to disable timeout. @@ -5438,15 +5463,21 @@ export interface PageAgent { cacheKey?: string; /** - * Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value. - * Defaults to context-wide value specified in `agent` property. + * Maximum number of retries when generating each action, defaults to context-wide value specified in `agent` + * property. */ - maxTokens?: number; + maxActionRetries?: number; + + /** + * Maximum number of agentic actions to generate, defaults to context-wide value specified in `agent` property. + */ + maxActions?: number; /** - * Maximum number of agentic turns during this call, defaults to context-wide value specified in `agent` property. + * Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value. + * Defaults to context-wide value specified in `agent` property. */ - maxTurns?: number; + maxTokens?: number; /** * Request timeout in milliseconds. Defaults to action timeout. Pass `0` to disable timeout. diff --git a/packages/playwright-core/ThirdPartyNotices.txt b/packages/playwright-core/ThirdPartyNotices.txt index 71cefc3d7d1e4..cf98b9ac14c69 100644 --- a/packages/playwright-core/ThirdPartyNotices.txt +++ b/packages/playwright-core/ThirdPartyNotices.txt @@ -4,7 +4,7 @@ THIRD-PARTY SOFTWARE NOTICES AND INFORMATION This project incorporates components from the projects listed below. The original copyright notices and the licenses under which Microsoft received such components are set forth below. Microsoft reserves all rights not expressly granted herein, whether by implication, estoppel or otherwise. -- @lowire/loop@0.0.19 (https://github.com/pavelfeldman/lowire) +- @lowire/loop@0.0.23 (https://github.com/pavelfeldman/lowire) - @modelcontextprotocol/sdk@1.24.2 (https://github.com/modelcontextprotocol/typescript-sdk) - accepts@2.0.0 (https://github.com/jshttp/accepts) - agent-base@7.1.4 (https://github.com/TooTallNate/proxy-agents) @@ -135,7 +135,7 @@ This project incorporates components from the projects listed below. The origina - zod-to-json-schema@3.25.0 (https://github.com/StefanTerdell/zod-to-json-schema) - zod@3.25.76 (https://github.com/colinhacks/zod) -%% @lowire/loop@0.0.19 NOTICES AND INFORMATION BEGIN HERE +%% @lowire/loop@0.0.23 NOTICES AND INFORMATION BEGIN HERE ========================================= Apache License Version 2.0, January 2004 @@ -339,7 +339,7 @@ Apache License See the License for the specific language governing permissions and limitations under the License. ========================================= -END OF @lowire/loop@0.0.19 AND INFORMATION +END OF @lowire/loop@0.0.23 AND INFORMATION %% @modelcontextprotocol/sdk@1.24.2 NOTICES AND INFORMATION BEGIN HERE ========================================= diff --git a/packages/playwright-core/bundles/mcp/package-lock.json b/packages/playwright-core/bundles/mcp/package-lock.json index 33a47d828b91d..bcc613bfc0275 100644 --- a/packages/playwright-core/bundles/mcp/package-lock.json +++ b/packages/playwright-core/bundles/mcp/package-lock.json @@ -8,16 +8,16 @@ "name": "mcp-bundle", "version": "0.0.1", "dependencies": { - "@lowire/loop": "^0.0.19", + "@lowire/loop": "^0.0.23", "@modelcontextprotocol/sdk": "^1.24.0", "zod": "^3.25.76", "zod-to-json-schema": "^3.24.6" } }, "node_modules/@lowire/loop": { - "version": "0.0.19", - "resolved": "https://registry.npmjs.org/@lowire/loop/-/loop-0.0.19.tgz", - "integrity": "sha512-uugjYtmA1mNSwHzbZEnvFedjB2EUX0s4NDbDbUcGuz6CH8fzZV2FFodY6fneik+r3GdLJtpvzpOP+cU/M0ba3w==", + "version": "0.0.23", + "resolved": "https://registry.npmjs.org/@lowire/loop/-/loop-0.0.23.tgz", + "integrity": "sha512-zw4yFzto2T9g+CiHfubzxut0JMxA053XQ2syveqPjqofvRZyumhJR11RIQjKrPO9rDbf2CFviKrYFt7kgru1VA==", "license": "Apache-2.0", "engines": { "node": ">=20" diff --git a/packages/playwright-core/bundles/mcp/package.json b/packages/playwright-core/bundles/mcp/package.json index f3209bf1bc0d5..fb90f525df212 100644 --- a/packages/playwright-core/bundles/mcp/package.json +++ b/packages/playwright-core/bundles/mcp/package.json @@ -3,7 +3,7 @@ "version": "0.0.1", "private": true, "dependencies": { - "@lowire/loop": "^0.0.19", + "@lowire/loop": "^0.0.23", "@modelcontextprotocol/sdk": "^1.24.0", "zod": "^3.25.76", "zod-to-json-schema": "^3.24.6" diff --git a/packages/playwright-core/src/client/page.ts b/packages/playwright-core/src/client/page.ts index f464643d1f4e1..4e3d827595712 100644 --- a/packages/playwright-core/src/client/page.ts +++ b/packages/playwright-core/src/client/page.ts @@ -852,12 +852,14 @@ export class Page extends ChannelOwner implements api.Page api: options.provider?.api, apiEndpoint: options.provider?.apiEndpoint, apiKey: options.provider?.apiKey, + apiTimeout: options.provider?.apiTimeout, apiCacheFile: (options.provider as any)?._apiCacheFile, model: options.provider?.model, cacheFile: options.cache?.cacheFile, cacheOutFile: options.cache?.cacheOutFile, - maxTokens: options.maxTokens, - maxTurns: options.maxTurns, + maxTokens: options.limits?.maxTokens, + maxActions: options.limits?.maxActions, + maxActionRetries: options.limits?.maxActionRetries, secrets: options.secrets ? Object.entries(options.secrets).map(([name, value]) => ({ name, value })) : undefined, systemPrompt: options.systemPrompt, }; diff --git a/packages/playwright-core/src/protocol/validator.ts b/packages/playwright-core/src/protocol/validator.ts index 62dca989ba68a..4ed0e4af71eb2 100644 --- a/packages/playwright-core/src/protocol/validator.ts +++ b/packages/playwright-core/src/protocol/validator.ts @@ -1491,10 +1491,12 @@ scheme.PageAgentParams = tObject({ api: tOptional(tString), apiKey: tOptional(tString), apiEndpoint: tOptional(tString), + apiTimeout: tOptional(tInt), apiCacheFile: tOptional(tString), cacheFile: tOptional(tString), cacheOutFile: tOptional(tString), - maxTurns: tOptional(tInt), + maxActions: tOptional(tInt), + maxActionRetries: tOptional(tInt), maxTokens: tOptional(tInt), model: tOptional(tString), secrets: tOptional(tArray(tType('NameValue'))), @@ -2907,7 +2909,8 @@ scheme.PageAgentTurnEvent = tObject({ }); scheme.PageAgentPerformParams = tObject({ task: tString, - maxTurns: tOptional(tInt), + maxActions: tOptional(tInt), + maxActionRetries: tOptional(tInt), maxTokens: tOptional(tInt), cacheKey: tOptional(tString), timeout: tOptional(tInt), @@ -2917,7 +2920,8 @@ scheme.PageAgentPerformResult = tObject({ }); scheme.PageAgentExpectParams = tObject({ expectation: tString, - maxTurns: tOptional(tInt), + maxActions: tOptional(tInt), + maxActionRetries: tOptional(tInt), maxTokens: tOptional(tInt), cacheKey: tOptional(tString), timeout: tOptional(tInt), @@ -2928,7 +2932,8 @@ scheme.PageAgentExpectResult = tObject({ scheme.PageAgentExtractParams = tObject({ query: tString, schema: tAny, - maxTurns: tOptional(tInt), + maxActions: tOptional(tInt), + maxActionRetries: tOptional(tInt), maxTokens: tOptional(tInt), cacheKey: tOptional(tString), timeout: tOptional(tInt), diff --git a/packages/playwright-core/src/server/agent/context.ts b/packages/playwright-core/src/server/agent/context.ts index 302b729341cd0..12aebb4f03ead 100644 --- a/packages/playwright-core/src/server/agent/context.ts +++ b/packages/playwright-core/src/server/agent/context.ts @@ -38,12 +38,14 @@ export class Context { readonly events: loopTypes.LoopEvents; private _actions: actions.ActionWithCode[] = []; private _history: HistoryItem[] = []; + private _budget: { tokens: number | undefined; }; - constructor(page: Page, agentParms: channels.PageAgentParams, events: loopTypes.LoopEvents) { + constructor(page: Page, agentParams: channels.PageAgentParams, events: loopTypes.LoopEvents) { this.page = page; - this.agentParams = agentParms; + this.agentParams = agentParams; this.sdkLanguage = page.browserContext._browser.sdkLanguage(); this.events = events; + this._budget = { tokens: agentParams.maxTokens }; } async runActionAndWait(progress: Progress, action: actions.Action) { @@ -79,6 +81,16 @@ export class Context { this._actions = []; } + consumeTokens(tokens: number) { + if (this._budget.tokens === undefined) + return; + this._budget.tokens = Math.max(0, this._budget.tokens - tokens); + } + + maxTokensRemaining(): number | undefined { + return this._budget.tokens; + } + async waitForCompletion(progress: Progress, callback: () => Promise, options?: { noWait?: boolean }): Promise { if (options?.noWait) return await callback(); diff --git a/packages/playwright-core/src/server/agent/pageAgent.ts b/packages/playwright-core/src/server/agent/pageAgent.ts index ff573aa9e765c..2578bea30a6eb 100644 --- a/packages/playwright-core/src/server/agent/pageAgent.ts +++ b/packages/playwright-core/src/server/agent/pageAgent.ts @@ -33,7 +33,8 @@ import type { Progress } from '../progress'; export type CallParams = { cacheKey?: string; maxTokens?: number; - maxTurns?: number; + maxActions?: number; + maxActionRetries?: number; }; export async function pageAgentPerform(progress: Progress, context: Context, userTask: string, callParams: CallParams) { @@ -105,9 +106,11 @@ async function runLoop(progress: Progress, context: Context, toolDefinitions: To api: context.agentParams.api as any, apiEndpoint: context.agentParams.apiEndpoint, apiKey: context.agentParams.apiKey, + apiTimeout: context.agentParams.apiTimeout ?? 0, model: context.agentParams.model, - maxTurns: params.maxTurns ?? context.agentParams.maxTurns, - maxTokens: params.maxTokens ?? context.agentParams.maxTokens, + maxTokens: params.maxTokens ?? context.maxTokensRemaining(), + maxToolCalls: params.maxActions ?? context.agentParams.maxActions ?? 10, + maxToolCallRetries: params.maxActionRetries ?? context.agentParams.maxActionRetries ?? 3, summarize: true, debug, callTool, @@ -136,8 +139,8 @@ async function runLoop(progress: Progress, context: Context, toolDefinitions: To task.push(full); task.push(''); - await loop.run(task.join('\n'), { signal: progress.signal }); - + const { error, usage } = await loop.run(task.join('\n'), { signal: progress.signal }); + context.consumeTokens(usage.input + usage.output); if (context.agentParams.apiCacheFile) { const apiCacheAfter = { ...apiCacheBefore, ...loop.cache() }; const sortedCache = Object.fromEntries(Object.entries(apiCacheAfter).sort(([a], [b]) => a.localeCompare(b))); @@ -148,6 +151,9 @@ async function runLoop(progress: Progress, context: Context, toolDefinitions: To } } + if (error) + throw new Error(`Agentic loop failed: ${error}`); + return { result: resultSchema ? reportedResult() : undefined }; } diff --git a/packages/playwright-core/src/server/bidi/bidiChromium.ts b/packages/playwright-core/src/server/bidi/bidiChromium.ts index 18071378b0076..7445f08269ba6 100644 --- a/packages/playwright-core/src/server/bidi/bidiChromium.ts +++ b/packages/playwright-core/src/server/bidi/bidiChromium.ts @@ -26,7 +26,6 @@ import { waitForReadyState } from '../chromium/chromium'; import type { BrowserOptions } from '../browser'; import type { SdkObject } from '../instrumentation'; -import type { ProtocolError } from '../protocolError'; import type { ConnectionTransport } from '../transport'; import type * as types from '../types'; @@ -55,16 +54,14 @@ export class BidiChromium extends BrowserType { } } - override doRewriteStartupLog(error: ProtocolError): ProtocolError { - if (!error.logs) - return error; - if (error.logs.includes('Missing X server')) - error.logs = '\n' + wrapInASCIIBox(kNoXServerRunningError, 1); + override doRewriteStartupLog(logs: string): string { + if (logs.includes('Missing X server')) + logs = '\n' + wrapInASCIIBox(kNoXServerRunningError, 1); // These error messages are taken from Chromium source code as of July, 2020: // https://github.com/chromium/chromium/blob/70565f67e79f79e17663ad1337dc6e63ee207ce9/content/browser/zygote_host/zygote_host_impl_linux.cc - if (!error.logs.includes('crbug.com/357670') && !error.logs.includes('No usable sandbox!') && !error.logs.includes('crbug.com/638180')) - return error; - error.logs = [ + if (!logs.includes('crbug.com/357670') && !logs.includes('No usable sandbox!') && !logs.includes('crbug.com/638180')) + return logs; + return [ `Chromium sandboxing failed!`, `================================`, `To avoid the sandboxing issue, do either of the following:`, @@ -73,7 +70,6 @@ export class BidiChromium extends BrowserType { `================================`, ``, ].join('\n'); - return error; } override amendEnvironment(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv { diff --git a/packages/playwright-core/src/server/bidi/bidiFirefox.ts b/packages/playwright-core/src/server/bidi/bidiFirefox.ts index 09406299cb6b2..199c9e72072da 100644 --- a/packages/playwright-core/src/server/bidi/bidiFirefox.ts +++ b/packages/playwright-core/src/server/bidi/bidiFirefox.ts @@ -26,7 +26,6 @@ import { ManualPromise } from '../../utils/isomorphic/manualPromise'; import type { BrowserOptions } from '../browser'; import type { SdkObject } from '../instrumentation'; -import type { ProtocolError } from '../protocolError'; import type { ConnectionTransport } from '../transport'; import type * as types from '../types'; import type { RecentLogsCollector } from '../utils/debugLogger'; @@ -45,15 +44,12 @@ export class BidiFirefox extends BrowserType { return BidiBrowser.connect(this.attribution.playwright, transport, options); } - override doRewriteStartupLog(error: ProtocolError): ProtocolError { - if (!error.logs) - return error; - // https://github.com/microsoft/playwright/issues/6500 - if (error.logs.includes(`as root in a regular user's session is not supported.`)) - error.logs = '\n' + wrapInASCIIBox(`Firefox is unable to launch if the $HOME folder isn't owned by the current user.\nWorkaround: Set the HOME=/root environment variable${process.env.GITHUB_ACTION ? ' in your GitHub Actions workflow file' : ''} when running Playwright.`, 1); - if (error.logs.includes('no DISPLAY environment variable specified')) - error.logs = '\n' + wrapInASCIIBox(kNoXServerRunningError, 1); - return error; + override doRewriteStartupLog(logs: string): string { + if (logs.includes(`as root in a regular user's session is not supported.`)) + logs = '\n' + wrapInASCIIBox(`Firefox is unable to launch if the $HOME folder isn't owned by the current user.\nWorkaround: Set the HOME=/root environment variable${process.env.GITHUB_ACTION ? ' in your GitHub Actions workflow file' : ''} when running Playwright.`, 1); + if (logs.includes('no DISPLAY environment variable specified')) + logs = '\n' + wrapInASCIIBox(kNoXServerRunningError, 1); + return logs; } override amendEnvironment(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv { diff --git a/packages/playwright-core/src/server/browserType.ts b/packages/playwright-core/src/server/browserType.ts index 3182f671b9c5e..98a2adec30e13 100644 --- a/packages/playwright-core/src/server/browserType.ts +++ b/packages/playwright-core/src/server/browserType.ts @@ -28,7 +28,7 @@ import { helper } from './helper'; import { SdkObject } from './instrumentation'; import { PipeTransport } from './pipeTransport'; import { envArrayToObject, launchProcess } from './utils/processLauncher'; -import { isProtocolError } from './protocolError'; +import { isProtocolError } from './protocolError'; import { registry } from './registry'; import { ClientCertificatesProxy } from './socksClientCertificatesInterceptor'; import { WebSocketTransport } from './transport'; @@ -37,7 +37,6 @@ import { RecentLogsCollector } from './utils/debugLogger'; import type { Browser, BrowserOptions, BrowserProcess } from './browser'; import type { BrowserContext } from './browserContext'; import type { Progress } from './progress'; -import type { ProtocolError } from './protocolError'; import type { BrowserName } from './registry'; import type { ConnectionTransport } from './transport'; import type * as types from './types'; @@ -264,8 +263,11 @@ export abstract class BrowserType extends SdkObject { this.waitForReadyState(options, browserLogsCollector), exitPromise.then(() => ({ wsEndpoint: undefined })), ]); - if (exitPromise.isDone()) - throw new Error(`Failed to launch the browser process.`); + if (exitPromise.isDone()) { + const log = helper.formatBrowserLogs(browserLogsCollector.recentLogs()); + const updatedLog = this.doRewriteStartupLog(log); + throw new Error(`Failed to launch the browser process.\nBrowser logs:\n${updatedLog}`); + } if (options.cdpPort !== undefined || !this.supportsPipeTransport()) { transport = await WebSocketTransport.connect(progress, wsEndpoint!); } else { @@ -321,7 +323,9 @@ export abstract class BrowserType extends SdkObject { private _rewriteStartupLog(error: Error): Error { if (!isProtocolError(error)) return error; - return this.doRewriteStartupLog(error); + if (error.logs) + error.logs = this.doRewriteStartupLog(error.logs); + return error; } async waitForReadyState(options: types.LaunchOptions, browserLogsCollector: RecentLogsCollector): Promise<{ wsEndpoint?: string }> { @@ -342,7 +346,7 @@ export abstract class BrowserType extends SdkObject { abstract defaultArgs(options: types.LaunchOptions, isPersistent: boolean, userDataDir: string): Promise; abstract connectToTransport(transport: ConnectionTransport, options: BrowserOptions, browserLogsCollector: RecentLogsCollector): Promise; abstract amendEnvironment(env: NodeJS.ProcessEnv, userDataDir: string, isPersistent: boolean, options: types.LaunchOptions): NodeJS.ProcessEnv; - abstract doRewriteStartupLog(error: ProtocolError): ProtocolError; + abstract doRewriteStartupLog(logs: string): string; abstract attemptToGracefullyCloseBrowser(transport: ConnectionTransport): void; } diff --git a/packages/playwright-core/src/server/chromium/chromium.ts b/packages/playwright-core/src/server/chromium/chromium.ts index cd442d3fddc77..b7d2dd294d5ef 100644 --- a/packages/playwright-core/src/server/chromium/chromium.ts +++ b/packages/playwright-core/src/server/chromium/chromium.ts @@ -42,7 +42,6 @@ import type { HTTPRequestParams } from '../utils/network'; import type { BrowserOptions, BrowserProcess } from '../browser'; import type { SdkObject } from '../instrumentation'; import type { Progress } from '../progress'; -import type { ProtocolError } from '../protocolError'; import type { ConnectionTransport, ProtocolRequest } from '../transport'; import type { BrowserContext } from '../browserContext'; import type * as types from '../types'; @@ -155,16 +154,14 @@ export class Chromium extends BrowserType { } } - override doRewriteStartupLog(error: ProtocolError): ProtocolError { - if (!error.logs) - return error; - if (error.logs.includes('Missing X server')) - error.logs = '\n' + wrapInASCIIBox(kNoXServerRunningError, 1); + override doRewriteStartupLog(logs: string): string { + if (logs.includes('Missing X server')) + logs = '\n' + wrapInASCIIBox(kNoXServerRunningError, 1); // These error messages are taken from Chromium source code as of July, 2020: // https://github.com/chromium/chromium/blob/70565f67e79f79e17663ad1337dc6e63ee207ce9/content/browser/zygote_host/zygote_host_impl_linux.cc - if (!error.logs.includes('crbug.com/357670') && !error.logs.includes('No usable sandbox!') && !error.logs.includes('crbug.com/638180')) - return error; - error.logs = [ + if (!logs.includes('crbug.com/357670') && !logs.includes('No usable sandbox!') && !logs.includes('crbug.com/638180')) + return logs; + return [ `Chromium sandboxing failed!`, `================================`, `To avoid the sandboxing issue, do either of the following:`, @@ -173,7 +170,6 @@ export class Chromium extends BrowserType { `================================`, ``, ].join('\n'); - return error; } override amendEnvironment(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv { diff --git a/packages/playwright-core/src/server/firefox/firefox.ts b/packages/playwright-core/src/server/firefox/firefox.ts index 1c0daef0fb6b7..489a6119c8182 100644 --- a/packages/playwright-core/src/server/firefox/firefox.ts +++ b/packages/playwright-core/src/server/firefox/firefox.ts @@ -26,7 +26,6 @@ import { ManualPromise } from '../../utils/isomorphic/manualPromise'; import type { Browser, BrowserOptions } from '../browser'; import type { SdkObject } from '../instrumentation'; -import type { ProtocolError } from '../protocolError'; import type { ConnectionTransport } from '../transport'; import type * as types from '../types'; import type { RecentLogsCollector } from '../utils/debugLogger'; @@ -58,15 +57,13 @@ export class Firefox extends BrowserType { return FFBrowser.connect(this.attribution.playwright, transport, options); } - override doRewriteStartupLog(error: ProtocolError): ProtocolError { - if (!error.logs) - return error; + override doRewriteStartupLog(logs: string): string { // https://github.com/microsoft/playwright/issues/6500 - if (error.logs.includes(`as root in a regular user's session is not supported.`)) - error.logs = '\n' + wrapInASCIIBox(`Firefox is unable to launch if the $HOME folder isn't owned by the current user.\nWorkaround: Set the HOME=/root environment variable${process.env.GITHUB_ACTION ? ' in your GitHub Actions workflow file' : ''} when running Playwright.`, 1); - if (error.logs.includes('no DISPLAY environment variable specified')) - error.logs = '\n' + wrapInASCIIBox(kNoXServerRunningError, 1); - return error; + if (logs.includes(`as root in a regular user's session is not supported.`)) + logs = '\n' + wrapInASCIIBox(`Firefox is unable to launch if the $HOME folder isn't owned by the current user.\nWorkaround: Set the HOME=/root environment variable${process.env.GITHUB_ACTION ? ' in your GitHub Actions workflow file' : ''} when running Playwright.`, 1); + if (logs.includes('no DISPLAY environment variable specified')) + logs = '\n' + wrapInASCIIBox(kNoXServerRunningError, 1); + return logs; } override amendEnvironment(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv { diff --git a/packages/playwright-core/src/server/webkit/webkit.ts b/packages/playwright-core/src/server/webkit/webkit.ts index 02e59bfdafa9a..c2daa135c9669 100644 --- a/packages/playwright-core/src/server/webkit/webkit.ts +++ b/packages/playwright-core/src/server/webkit/webkit.ts @@ -25,7 +25,6 @@ import { spawnAsync } from '../utils/spawnAsync'; import type { BrowserOptions } from '../browser'; import type { SdkObject } from '../instrumentation'; -import type { ProtocolError } from '../protocolError'; import type { ConnectionTransport } from '../transport'; import type * as types from '../types'; @@ -45,12 +44,10 @@ export class WebKit extends BrowserType { }; } - override doRewriteStartupLog(error: ProtocolError): ProtocolError { - if (!error.logs) - return error; - if (error.logs.includes('Failed to open display') || error.logs.includes('cannot open display')) - error.logs = '\n' + wrapInASCIIBox(kNoXServerRunningError, 1); - return error; + override doRewriteStartupLog(logs: string): string { + if (logs.includes('Failed to open display') || logs.includes('cannot open display')) + logs = '\n' + wrapInASCIIBox(kNoXServerRunningError, 1); + return logs; } override attemptToGracefullyCloseBrowser(transport: ConnectionTransport): void { diff --git a/packages/playwright-core/types/types.d.ts b/packages/playwright-core/types/types.d.ts index a9d3c3daff9fa..51dd92fb293b4 100644 --- a/packages/playwright-core/types/types.d.ts +++ b/packages/playwright-core/types/types.d.ts @@ -2110,12 +2110,26 @@ export interface Page { cacheOutFile?: string; }; - maxTokens?: number; - /** - * Maximum number of agentic turns to take per call. Defaults to 10. + * Limits to use for the agentic loop. */ - maxTurns?: number; + limits?: { + /** + * Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value. + * Defaults to unlimited. + */ + maxTokens?: number; + + /** + * Maximum number of agentic actions to generate, defaults to 10. + */ + maxActions?: number; + + /** + * Maximum number retries per action, defaults to 3. + */ + maxActionRetries?: number; + }; provider?: { /** @@ -2133,6 +2147,11 @@ export interface Page { */ apiKey: string; + /** + * Amount of time to wait for the provider to respond to each request. + */ + apiTimeout?: number; + /** * Model identifier within the provider. Required in non-cache mode. */ @@ -5402,15 +5421,21 @@ export interface PageAgent { cacheKey?: string; /** - * Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value. - * Defaults to context-wide value specified in `agent` property. + * Maximum number of retries when generating each action, defaults to context-wide value specified in `agent` + * property. */ - maxTokens?: number; + maxActionRetries?: number; + + /** + * Maximum number of agentic actions to generate, defaults to context-wide value specified in `agent` property. + */ + maxActions?: number; /** - * Maximum number of agentic turns during this call, defaults to context-wide value specified in `agent` property. + * Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value. + * Defaults to context-wide value specified in `agent` property. */ - maxTurns?: number; + maxTokens?: number; /** * Request timeout in milliseconds. Defaults to action timeout. Pass `0` to disable timeout. @@ -5438,15 +5463,21 @@ export interface PageAgent { cacheKey?: string; /** - * Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value. - * Defaults to context-wide value specified in `agent` property. + * Maximum number of retries when generating each action, defaults to context-wide value specified in `agent` + * property. */ - maxTokens?: number; + maxActionRetries?: number; + + /** + * Maximum number of agentic actions to generate, defaults to context-wide value specified in `agent` property. + */ + maxActions?: number; /** - * Maximum number of agentic turns during this call, defaults to context-wide value specified in `agent` property. + * Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value. + * Defaults to context-wide value specified in `agent` property. */ - maxTurns?: number; + maxTokens?: number; /** * Request timeout in milliseconds. Defaults to action timeout. Pass `0` to disable timeout. diff --git a/packages/playwright/src/index.ts b/packages/playwright/src/index.ts index 1a52a43897417..82b978fb8caf2 100644 --- a/packages/playwright/src/index.ts +++ b/packages/playwright/src/index.ts @@ -462,12 +462,7 @@ const playwrightFixtures: Fixtures = ({ const cacheFile = testInfoImpl.config.runAgents === 'all' ? undefined : await testInfoImpl._cloneStorage(resolvedCacheFile); const cacheOutFile = path.join(testInfoImpl.artifactsDir(), 'agent-cache-' + createGuid() + '.json'); - const provider = agentOptions?.api && testInfo.config.runAgents !== 'none' ? { - api: agentOptions.api as any, - apiEndpoint: agentOptions.apiEndpoint, - apiKey: agentOptions.apiKey, - model: agentOptions.model, - } : undefined; + const provider = agentOptions?.provider && testInfo.config.runAgents !== 'none' ? agentOptions.provider : undefined; if (provider) testInfo.setTimeout(0); @@ -479,9 +474,9 @@ const playwrightFixtures: Fixtures = ({ const agent = await page.agent({ provider, cache, - maxTokens: agentOptions?.maxTokens, - maxTurns: agentOptions?.maxTurns, + limits: agentOptions?.limits, secrets: agentOptions?.secrets, + systemPrompt: agentOptions?.systemPrompt, }); await use(agent); diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts index 16b81c92f0510..4119c6d1b1695 100644 --- a/packages/playwright/types/test.d.ts +++ b/packages/playwright/types/test.d.ts @@ -6955,15 +6955,22 @@ export type ScreenshotMode = 'off' | 'on' | 'only-on-failure' | 'on-first-failur export type TraceMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | 'on-all-retries' | 'retain-on-first-failure'; export type VideoMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry'; export type AgentOptions = { - api: string; - apiKey: string; - apiEndpoint?: string; - model: string; + provider?: { + api: 'openai' | 'openai-compatible' | 'anthropic' | 'google'; + apiEndpoint?: string; + apiKey: string; + apiTimeout?: number; + model: string; + }, + limits?: { + maxTokens?: number; + maxActions?: number; + maxActionRetries?: number; + }; cachePathTemplate?: string; - maxTurns?: number; - maxTokens?: number; runAgents?: 'all' | 'missing' | 'none'; secrets?: { [key: string]: string }; + systemPrompt?: string; }; /** diff --git a/packages/protocol/src/channels.d.ts b/packages/protocol/src/channels.d.ts index 38e0009d0d806..a76dce8a907a8 100644 --- a/packages/protocol/src/channels.d.ts +++ b/packages/protocol/src/channels.d.ts @@ -2586,10 +2586,12 @@ export type PageAgentParams = { api?: string, apiKey?: string, apiEndpoint?: string, + apiTimeout?: number, apiCacheFile?: string, cacheFile?: string, cacheOutFile?: string, - maxTurns?: number, + maxActions?: number, + maxActionRetries?: number, maxTokens?: number, model?: string, secrets?: NameValue[], @@ -2599,10 +2601,12 @@ export type PageAgentOptions = { api?: string, apiKey?: string, apiEndpoint?: string, + apiTimeout?: number, apiCacheFile?: string, cacheFile?: string, cacheOutFile?: string, - maxTurns?: number, + maxActions?: number, + maxActionRetries?: number, maxTokens?: number, model?: string, secrets?: NameValue[], @@ -5115,13 +5119,15 @@ export type PageAgentTurnEvent = { }; export type PageAgentPerformParams = { task: string, - maxTurns?: number, + maxActions?: number, + maxActionRetries?: number, maxTokens?: number, cacheKey?: string, timeout?: number, }; export type PageAgentPerformOptions = { - maxTurns?: number, + maxActions?: number, + maxActionRetries?: number, maxTokens?: number, cacheKey?: string, timeout?: number, @@ -5131,13 +5137,15 @@ export type PageAgentPerformResult = { }; export type PageAgentExpectParams = { expectation: string, - maxTurns?: number, + maxActions?: number, + maxActionRetries?: number, maxTokens?: number, cacheKey?: string, timeout?: number, }; export type PageAgentExpectOptions = { - maxTurns?: number, + maxActions?: number, + maxActionRetries?: number, maxTokens?: number, cacheKey?: string, timeout?: number, @@ -5148,13 +5156,15 @@ export type PageAgentExpectResult = { export type PageAgentExtractParams = { query: string, schema: any, - maxTurns?: number, + maxActions?: number, + maxActionRetries?: number, maxTokens?: number, cacheKey?: string, timeout?: number, }; export type PageAgentExtractOptions = { - maxTurns?: number, + maxActions?: number, + maxActionRetries?: number, maxTokens?: number, cacheKey?: string, timeout?: number, diff --git a/packages/protocol/src/protocol.yml b/packages/protocol/src/protocol.yml index edce8f286f432..d4af228adaa12 100644 --- a/packages/protocol/src/protocol.yml +++ b/packages/protocol/src/protocol.yml @@ -2019,10 +2019,12 @@ Page: api: string? apiKey: string? apiEndpoint: string? + apiTimeout: int? apiCacheFile: string? cacheFile: string? cacheOutFile: string? - maxTurns: int? + maxActions: int? + maxActionRetries: int? maxTokens: int? model: string? secrets: @@ -4374,7 +4376,8 @@ PageAgent: PageAgentOptions: type: mixin properties: - maxTurns: int? + maxActions: int? + maxActionRetries: int? maxTokens: int? cacheKey: string? timeout: int? diff --git a/tests/library/__llm_cache__/library-agent-limits-should-respect-max-actions-limit.json b/tests/library/__llm_cache__/library-agent-limits-should-respect-max-actions-limit.json new file mode 100644 index 0000000000000..068919159564e --- /dev/null +++ b/tests/library/__llm_cache__/library-agent-limits-should-respect-max-actions-limit.json @@ -0,0 +1,114 @@ +{ + "02fe9016aefaf25bffe8ae4a623014e933282db1": { + "result": { + "role": "assistant", + "content": [ + { + "type": "text", + "text": "Now clicking the second time." + }, + { + "type": "tool_call", + "name": "browser_click", + "arguments": { + "element": "Submit button", + "ref": "e2", + "_is_done": false + }, + "id": "toolu_01PVA67E6ey7gspaUHe6vKUb" + } + ], + "stopReason": { + "code": "ok" + } + }, + "usage": { + "input": 3044, + "output": 98 + } + }, + "9406488d923a063c065ff21e2a4f78c47cd932aa": { + "result": { + "role": "assistant", + "content": [ + { + "type": "text", + "text": "Now clicking the third time." + }, + { + "type": "tool_call", + "name": "browser_click", + "arguments": { + "element": "Submit button", + "ref": "e2", + "_is_done": false + }, + "id": "toolu_01HreVmfgh1urgtDcxbkDbsN" + } + ], + "stopReason": { + "code": "ok" + } + }, + "usage": { + "input": 3117, + "output": 98 + } + }, + "a8a2db95880cad9f7a0ce1d3e409e2d273685eba": { + "result": { + "role": "assistant", + "content": [ + { + "type": "text", + "text": "I'll click the submit button 5 times as requested. Let me start by clicking it the first time." + }, + { + "type": "tool_call", + "name": "browser_click", + "arguments": { + "element": "Submit button", + "ref": "e2", + "_is_done": false + }, + "id": "toolu_01Q2Bk335vL1wsswZGCsuaTe" + } + ], + "stopReason": { + "code": "ok" + } + }, + "usage": { + "input": 2897, + "output": 115 + } + }, + "efe9f77d726c2eff73d5c51d2cbc30c3171d6294": { + "result": { + "role": "assistant", + "content": [ + { + "type": "text", + "text": "Now clicking the fourth time." + }, + { + "type": "tool_call", + "name": "browser_click", + "arguments": { + "element": "Submit button", + "ref": "e2", + "_is_done": false + }, + "id": "toolu_01B29QmP39aFMi2uHevcZKU1" + } + ], + "stopReason": { + "code": "ok" + } + }, + "usage": { + "input": 3187, + "output": 98 + } + } +} \ No newline at end of file diff --git a/tests/library/__llm_cache__/library-agent-perform-click-a-button.json b/tests/library/__llm_cache__/library-agent-perform-click-a-button.json index 26f5b7cdc44f5..83a2310d62e30 100644 --- a/tests/library/__llm_cache__/library-agent-perform-click-a-button.json +++ b/tests/library/__llm_cache__/library-agent-perform-click-a-button.json @@ -1,5 +1,5 @@ { - "0706eb777330056e04cc3abb4715ed3fa1c9d4d5": { + "8c5697c11562d27e025c16eaba3a3d333198bb56": { "result": { "role": "assistant", "content": [ @@ -15,9 +15,12 @@ "ref": "e2", "_is_done": true }, - "id": "toolu_01W54FxjcWYV856RHP3V2xdj" + "id": "toolu_019UXybai8M1mCTeCBkEtzJc" } - ] + ], + "stopReason": { + "code": "ok" + } }, "usage": { "input": 2893, diff --git a/tests/library/__llm_cache__/library-agent-perform-expect-value.json b/tests/library/__llm_cache__/library-agent-perform-expect-value.json index c4cd9ce537ebf..8081774369641 100644 --- a/tests/library/__llm_cache__/library-agent-perform-expect-value.json +++ b/tests/library/__llm_cache__/library-agent-perform-expect-value.json @@ -1,5 +1,5 @@ { - "77c0828e4bffb298a3850504550ffcac23e6692e": { + "46258723cbb8dc1fbc9c390304cd506187897079": { "result": { "role": "assistant", "content": [ @@ -16,22 +16,25 @@ "text": "bogus", "_is_done": false }, - "id": "toolu_01FfLGgBK1CHZ8e54gQMukA6" + "id": "toolu_01V8GEC6fjiTt98e3k88ENyV" } - ] + ], + "stopReason": { + "code": "ok" + } }, "usage": { "input": 2926, "output": 149 } }, - "ca3ff77a849d4eacf88d299dc8727d63724a68aa": { + "5429c27bfbcae1d531a13010d3a5d4807251a7c4": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "Perfect! I have successfully completed all three parts of the task:\n\n1. ✅ **Entered \"bogus\" into the email field** - The text has been typed into the textbox\n2. ✅ **Verified the value is \"bogus\"** - The page snapshot shows the textbox contains \"bogus\"\n3. ✅ **Confirmed error message is displayed** - The error message \"Error: Invalid email address\" is now visible on the page\n\nThe task is complete." + "text": "Perfect! I've successfully completed all three parts of the task:\n\n1. ✅ **Entered \"bogus\" into the email field** - The text has been typed into the Email Address textbox\n2. ✅ **Verified the value is \"bogus\"** - The snapshot confirms the textbox contains \"bogus\"\n3. ✅ **Confirmed error message is displayed** - The error message \"Error: Invalid email address\" is now visible on the page\n\nThe task has been completed successfully. The invalid email \"bogus\" triggered the expected validation error message." }, { "type": "tool_call", @@ -39,13 +42,16 @@ "arguments": { "_is_done": true }, - "id": "toolu_018wjD4H2S5QHQ3rpE2sR7bM" + "id": "toolu_01WuNDf5vv1QM1SdgkAKnpVF" } - ] + ], + "stopReason": { + "code": "ok" + } }, "usage": { "input": 3138, - "output": 167 + "output": 184 } } } \ No newline at end of file diff --git a/tests/library/__llm_cache__/library-agent-perform-extract-task.json b/tests/library/__llm_cache__/library-agent-perform-extract-task.json index 74f9d15fbb0d1..46a3c383fc437 100644 --- a/tests/library/__llm_cache__/library-agent-perform-extract-task.json +++ b/tests/library/__llm_cache__/library-agent-perform-extract-task.json @@ -1,5 +1,5 @@ { - "b5f1ee58f1051afb059b3b33e8be41576a1324a4": { + "d02ae2240fac126bacb74133b970880eaaf3f915": { "result": { "role": "assistant", "content": [ @@ -23,9 +23,12 @@ ], "_is_done": true }, - "id": "toolu_01WvevmDTBxmKg79Ji6vQvpH" + "id": "toolu_01JtUAyrJqJSRwBsS7pn9fRY" } - ] + ], + "stopReason": { + "code": "ok" + } }, "usage": { "input": 888, diff --git a/tests/library/__llm_cache__/library-agent-perform-perform-history.json b/tests/library/__llm_cache__/library-agent-perform-perform-history.json index 00a85f8150edc..13be1898fc891 100644 --- a/tests/library/__llm_cache__/library-agent-perform-perform-history.json +++ b/tests/library/__llm_cache__/library-agent-perform-perform-history.json @@ -1,5 +1,5 @@ { - "4f9987bc22c6035501274532494c185d1270aba2": { + "7416c1b94dbb8a1f7405a276811783738c295295": { "result": { "role": "assistant", "content": [ @@ -15,16 +15,19 @@ "ref": "e3", "_is_done": true }, - "id": "toolu_013sypFp7tiRCLaR3i3oWrei" + "id": "toolu_017c7pdkXdqSjQMiaF7UKwa8" } - ] + ], + "stopReason": { + "code": "ok" + } }, "usage": { "input": 2946, "output": 102 } }, - "79e2d58996075c4ebe8eb9e557bd0cdb8b04211a": { + "96cdcffad5cf3bf3d064acdeb05fbb4968f27c02": { "result": { "role": "assistant", "content": [ @@ -40,9 +43,12 @@ "ref": "e3", "_is_done": true }, - "id": "toolu_01E6xBUYzQEdRrkWoobbUbFq" + "id": "toolu_01E1epU97Ub6RDTk4PeFuKke" } - ] + ], + "stopReason": { + "code": "ok" + } }, "usage": { "input": 2933, diff --git a/tests/library/__llm_cache__/library-agent-perform-perform-run-timeout.json b/tests/library/__llm_cache__/library-agent-perform-perform-run-timeout.json index 836cd0bc34487..01648843e9695 100644 --- a/tests/library/__llm_cache__/library-agent-perform-perform-run-timeout.json +++ b/tests/library/__llm_cache__/library-agent-perform-perform-run-timeout.json @@ -1,5 +1,5 @@ { - "c2f54e956e3131255f03c5484a637b580f7d695f": { + "c3664a96a82cba8678b7ca68973bb25c1d1207a4": { "result": { "role": "assistant", "content": [ @@ -15,9 +15,12 @@ "ref": "e3", "_is_done": true }, - "id": "toolu_01GipHDfLSETD8VyJt6D6ng5" + "id": "toolu_01FP2NCSpx8mnFeg16iX9gKi" } - ] + ], + "stopReason": { + "code": "ok" + } }, "usage": { "input": 2919, diff --git a/tests/library/__llm_cache__/library-agent-perform-retrieve-a-secret.json b/tests/library/__llm_cache__/library-agent-perform-retrieve-a-secret.json index f60e4427dd787..3c0ad14395152 100644 --- a/tests/library/__llm_cache__/library-agent-perform-retrieve-a-secret.json +++ b/tests/library/__llm_cache__/library-agent-perform-retrieve-a-secret.json @@ -1,5 +1,5 @@ { - "c1d41afc231b64caf136132dd841ca8f45876674": { + "8bd748693a579b9f17e2fb127fcb3af9706934d4": { "result": { "role": "assistant", "content": [ @@ -16,9 +16,12 @@ "text": "%x-secret-email%", "_is_done": true }, - "id": "toolu_018PD6jsENphY4ERa1XRF4ZN" + "id": "toolu_01YYGZnA96oGJMknVBJC4Cfj" } - ] + ], + "stopReason": { + "code": "ok" + } }, "usage": { "input": 2901, diff --git a/tests/library/agent-helpers.ts b/tests/library/agent-helpers.ts index b6d99de6a2b57..b145b1882d007 100644 --- a/tests/library/agent-helpers.ts +++ b/tests/library/agent-helpers.ts @@ -32,7 +32,9 @@ export async function setCacheObject(object: any) { await fs.promises.writeFile(cacheFile(), JSON.stringify(object, null, 2), 'utf8'); } -export async function generateAgent(context: BrowserContext, options: { secrets?: Record } = {}) { +type AgentOptions = Parameters[0]; + +export async function generateAgent(context: BrowserContext, options: AgentOptions = {}) { const apiCacheFile = path.join(__dirname, '__llm_cache__', sanitizeFileName(test.info().titlePath.join(' ')) + '.json'); const page = await context.newPage(); diff --git a/tests/library/agent-limits.spec.ts b/tests/library/agent-limits.spec.ts new file mode 100644 index 0000000000000..07d3f3014964c --- /dev/null +++ b/tests/library/agent-limits.spec.ts @@ -0,0 +1,54 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { browserTest as test, expect } from '../config/browserTest'; +import { generateAgent } from './agent-helpers'; + +test('should respect total max tokens limit', async ({ context }) => { + const { page, agent } = await generateAgent(context, { + limits: { + maxTokens: 123, + }, + }); + await page.setContent(` + + `); + const e = await agent.perform('Click submit button').catch(e => e); + expect(e.message.toLowerCase()).toContain('budget'); + expect(e.message).toContain('123'); +}); + +test('should respect call max tokens limit', async ({ context }) => { + const { page, agent } = await generateAgent(context); + await page.setContent(` + + `); + const e = await agent.perform('Click submit button', { maxTokens: 123 }).catch(e => e); + expect(e.message.toLowerCase()).toContain('budget'); + expect(e.message).toContain('123'); +}); + +test('should respect max actions limit', async ({ context }) => { + const { page, agent } = await generateAgent(context); + let clicked = 0; + await page.exposeFunction('clicked', () => ++clicked); + await page.setContent(` + + `); + const e = await agent.perform('Click the submit button 5 times', { maxActions: 3 }).catch(e => e); + expect(e.message).toContain('Failed to perform step, max tool calls (3) reached'); + expect(clicked).toBe(3); +}); diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts index bf47bdab5a27b..3a0eb2eb2d004 100644 --- a/utils/generate_types/overrides-test.d.ts +++ b/utils/generate_types/overrides-test.d.ts @@ -270,15 +270,22 @@ export type ScreenshotMode = 'off' | 'on' | 'only-on-failure' | 'on-first-failur export type TraceMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | 'on-all-retries' | 'retain-on-first-failure'; export type VideoMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry'; export type AgentOptions = { - api: string; - apiKey: string; - apiEndpoint?: string; - model: string; + provider?: { + api: 'openai' | 'openai-compatible' | 'anthropic' | 'google'; + apiEndpoint?: string; + apiKey: string; + apiTimeout?: number; + model: string; + }, + limits?: { + maxTokens?: number; + maxActions?: number; + maxActionRetries?: number; + }; cachePathTemplate?: string; - maxTurns?: number; - maxTokens?: number; runAgents?: 'all' | 'missing' | 'none'; secrets?: { [key: string]: string }; + systemPrompt?: string; }; export interface PlaywrightTestOptions {