diff --git a/docs/src/api/class-browser.md b/docs/src/api/class-browser.md index e61bf6930c4cf..7867ce5c8ba71 100644 --- a/docs/src/api/class-browser.md +++ b/docs/src/api/class-browser.md @@ -267,9 +267,6 @@ await browser.CloseAsync(); ### option: Browser.newContext.storageStatePath = %%-csharp-java-context-option-storage-state-path-%% * since: v1.9 -### option: Browser.newContext.agent = %%-context-option-agent-%% -* since: v1.58 - ## async method: Browser.newPage * since: v1.8 - returns: <[Page]> diff --git a/docs/src/api/class-page.md b/docs/src/api/class-page.md index 23e8aad37c892..ae88787901030 100644 --- a/docs/src/api/class-page.md +++ b/docs/src/api/class-page.md @@ -544,17 +544,6 @@ sequence of events is `request`, `response` and `requestfinished`. Emitted when [response] status and headers are received for a request. For a successful response, the sequence of events is `request`, `response` and `requestfinished`. -## event: Page.agentTurn -* since: v1.58 -- argument: <[Object]> - - `role` <[string]> - - `message` <[string]> - - `usage` ?<[Object]> - - `inputTokens` <[int]> - - `outputTokens` <[int]> - -Emitted when the agent makes a turn. - ## event: Page.webSocket * since: v1.9 - argument: <[WebSocket]> @@ -720,9 +709,42 @@ current working directory. Raw CSS content to be injected into frame. -## property: Page.agent +## async method: Page.agent * since: v1.58 -- type: <[PageAgent]> +- returns: <[PageAgent]> + +Initialize page agent with the llm provider and cache. + +### option: Page.agent.cache +* since: v1.58 +- `cache` <[Object]> + - `cacheFile` ?<[string]> Cache file to use/generate code for performed actions into. Cache is not used if not specified (default). + - `cacheOutFile` ?<[string]> When specified, generated entries are written into the `cacheOutFile` instead of updating the `cacheFile`. + +### option: Page.agent.maxTurns +* since: v1.58 +- `maxTurns` <[int]> + +Maximum number of agentic turns to take per call. Defaults to 10. + +### option: Page.agent.maxTokens +* since: v1.58 +- `maxTokens` ?<[int]> + +### option: Page.agent.provider +* since: v1.58 +- `provider` <[Object]> + - `api` <[PageAgentAPI]<"openai"|"openai-compatible"|"anthropic"|"google">> API to use. + - `apiEndpoint` ?<[string]> Endpoint to use if different from default. + - `apiKey` <[string]> API key for the LLM provider. + - `model` <[string]> Model identifier within the provider. Required in non-cache mode. + +### option: Page.agent.secrets +* since: v1.58 +- `secrets` ?<[Object]<[string], [string]>> + +Secrets to hide from the LLM. + ## async method: Page.bringToFront * since: v1.8 diff --git a/docs/src/api/class-pageagent.md b/docs/src/api/class-pageagent.md index d3f2f4ee917ce..4914a8e00b75c 100644 --- a/docs/src/api/class-pageagent.md +++ b/docs/src/api/class-pageagent.md @@ -1,6 +1,21 @@ # class: PageAgent * since: v1.58 +## event: PageAgent.turn +* since: v1.58 +- argument: <[Object]> + - `role` <[string]> + - `message` <[string]> + - `usage` ?<[Object]> + - `inputTokens` <[int]> + - `outputTokens` <[int]> + +Emitted when the agent makes a turn. + +## async method: PageAgent.dispose +* since: v1.58 + +Dispose this agent. ## async method: PageAgent.expect * since: v1.58 @@ -10,7 +25,7 @@ Expect certain condition to be met. **Usage** ```js -await page.agent.expect('"0 items" to be reported'); +await agent.expect('"0 items" to be reported'); ``` ### param: PageAgent.expect.expectation @@ -36,7 +51,7 @@ Extract information from the page using the agentic loop, return it in a given Z **Usage** ```js -await page.agent.extract('List of items in the cart', z.object({ +await agent.extract('List of items in the cart', z.object({ title: z.string().describe('Item title to extract'), price: z.string().describe('Item price to extract'), }).array()); @@ -69,7 +84,7 @@ Perform action using agentic loop. **Usage** ```js -await page.agent.perform('Click submit button'); +await agent.perform('Click submit button'); ``` ### param: PageAgent.perform.task diff --git a/docs/src/api/params.md b/docs/src/api/params.md index 64ed0173b7dde..12a8186fe3e9d 100644 --- a/docs/src/api/params.md +++ b/docs/src/api/params.md @@ -370,40 +370,6 @@ It makes the execution of the tests non-deterministic. Emulates consistent window screen size available inside web page via `window.screen`. Is only used when the [`option: viewport`] is set. -## context-option-agent -- `agent` <[Object]> - - `api` ?<[string]> API to use, `openapi`, `google` or `anthropic`. Required in non-cache mode. - - `apiEndpoint` ?<[string]> Endpoint to use if different from default. - - `apiKey` ?<[string]> API key for the LLM provider. - - `model` ?<[string]> Model identifier within the provider. Required in non-cache mode. - - `cacheFile` ?<[string]> Cache file to use/generate code for performed actions into. Cache is not used if not specified (default). - - `cacheOutFile` ?<[string]> When specified, generated entries are written into the `cacheOutFile` instead of updating the `cacheFile`. - - `secrets` ?<[Object]<[string], [string]>> Secrets to hide from the LLM. - - `maxTurns` ?<[int]> Maximum number of agentic turns to take per call. Defaults to 10. - - `maxTokens` ?<[int]> Maximum number of tokens to consume per call. The agentic loop will stop after input + output tokens exceed this value. Defaults on unlimited. - -Agent settings for [`property: Page.agent`]. - -## page-agent-api -* since: v1.58 -- `api` <[string]> - -API to use, `openapi`, `google` or `anthropic`. Required in non-cache mode. - -## page-agent-api-endpoint -* since: v1.58 -- `apiEndpoint` <[string]> - -Endpoint to use if different from default. - -## page-agent-api-key -* since: v1.58 -- `apiKey` <[string]> - -API key for the LLM provider. - -API version if relevant. - ## page-agent-cache-key * since: v1.58 - `cacheKey` <[string]> @@ -425,9 +391,6 @@ Defaults to context-wide value specified in `agent` property. Maximum number of agentic turns during this call, defaults to context-wide value specified in `agent` property. ## page-agent-call-options-v1.58 -- %%-page-agent-api-%% -- %%-page-agent-api-key-%% -- %%-page-agent-api-endpoint-%% - %%-page-agent-cache-key-%% - %%-page-agent-max-tokens-%% - %%-page-agent-max-turns-%% diff --git a/docs/src/test-api/class-fixtures.md b/docs/src/test-api/class-fixtures.md index 0c08b19da3b2a..9ec8e33f92cd3 100644 --- a/docs/src/test-api/class-fixtures.md +++ b/docs/src/test-api/class-fixtures.md @@ -18,6 +18,10 @@ Given the test above, Playwright Test will set up the `page` fixture before runn Playwright Test comes with builtin fixtures listed below, and you can add your own fixtures as well. Playwright Test also [provides options][TestOptions] to configure [`property: Fixtures.browser`], [`property: Fixtures.context`] and [`property: Fixtures.page`]. +## property: Fixtures.agent +* since: v1.58 +- type: <[PageAgent]> + ## property: Fixtures.browser * since: v1.10 - type: <[Browser]> diff --git a/docs/src/test-api/class-fullconfig.md b/docs/src/test-api/class-fullconfig.md index 634c48ba88c15..1f3ee92df7750 100644 --- a/docs/src/test-api/class-fullconfig.md +++ b/docs/src/test-api/class-fullconfig.md @@ -108,7 +108,7 @@ Base directory for all relative paths used in the reporters. * since: v1.58 - type: <['RunAgentsMode]<"all"|"missing"|"none">> -Whether to run LLM agent for [`property: Page.agent`]: +Whether to run LLM agent for [PageAgent]: * "all" disregards existing cache and performs all actions via LLM * "missing" only performs actions that don't have generated cache actions * "none" does not talk to LLM at all, relies on the cached actions (default) diff --git a/docs/src/test-api/class-testconfig.md b/docs/src/test-api/class-testconfig.md index 5a9251062ecdc..1d48d6be17423 100644 --- a/docs/src/test-api/class-testconfig.md +++ b/docs/src/test-api/class-testconfig.md @@ -518,7 +518,7 @@ export default defineConfig({ * since: v1.58 - type: ?<['RunAgentsMode]<"all"|"missing"|"none">> -Whether to run LLM agent for [`property: Page.agent`]: +Whether to run LLM agent for [PageAgent]: * "all" disregards existing cache and performs all actions via LLM * "missing" only performs actions that don't have generated cache actions * "none" does not talk to LLM at all, relies on the cached actions (default) diff --git a/docs/src/test-api/class-testoptions.md b/docs/src/test-api/class-testoptions.md index 8488da98455a1..194cd58616994 100644 --- a/docs/src/test-api/class-testoptions.md +++ b/docs/src/test-api/class-testoptions.md @@ -46,10 +46,12 @@ export default defineConfig({ }); ``` -## property: TestOptions.agent +## property: TestOptions.agentOptions * since: v1.58 - type: <[Object]> - - `provider` ?<[string]> LLM provider to use. Required in non-cache mode. + - `api` ?<[string]> LLM provider to use. Required in non-cache mode. + - `apiKey` ?<[string]> Key for the LLM provider. + - `apiEndpoint` ?<[string]> LLM provider endpoint. - `model` ?<[string]> Model identifier within the provider. Required in non-cache mode. - `cachePathTemplate` ?<[string]> Cache file template to use/generate code for performed actions into. - `maxTurns` ?<[int]> Maximum number of agentic turns to take per call. Defaults to 10. diff --git a/examples/todomvc/tests/fixtures.ts b/examples/todomvc/tests/fixtures.ts index c58c0055c58a0..0e317c47065e9 100644 --- a/examples/todomvc/tests/fixtures.ts +++ b/examples/todomvc/tests/fixtures.ts @@ -5,6 +5,12 @@ import { test as baseTest } from '@playwright/test'; export { expect } from '@playwright/test'; export const test = baseTest.extend({ + agentOptions: { + api: 'anthropic', + apiKey: process.env.AZURE_SONNET_API_KEY!, + apiEndpoint: process.env.AZURE_SONNET_ENDPOINT!, + model: 'claude-sonnet-4-5', + }, page: async ({ page }, use) => { await page.goto('https://demo.playwright.dev/todomvc'); await use(page); diff --git a/examples/todomvc/tests/perform/completing-todos/should-complete-multiple-todos.spec.ts b/examples/todomvc/tests/perform/completing-todos/should-complete-multiple-todos.spec.ts index dfaa105eccffd..18e0476f02da7 100644 --- a/examples/todomvc/tests/perform/completing-todos/should-complete-multiple-todos.spec.ts +++ b/examples/todomvc/tests/perform/completing-todos/should-complete-multiple-todos.spec.ts @@ -1,27 +1,18 @@ import { test } from '../../fixtures'; -test.use({ - agent: { - api: 'anthropic', - apiKey: process.env.AZURE_SONNET_API_KEY!, - apiEndpoint: process.env.AZURE_SONNET_ENDPOINT!, - model: 'claude-sonnet-4-5', - } -}); - -test('should complete multiple todos', async ({ page }) => { - await page.agent.expect(`The page loads with an empty todo list`); +test('should complete multiple todos', async ({ agent }) => { + await agent.expect(`The page loads with an empty todo list`); - await page.agent.perform(`Add three todos: 'Buy milk', 'Walk dog', 'Finish report'`); - await page.agent.expect(`All three todos are visible`); - await page.agent.expect(`Counter shows '3 items left'`); + await agent.perform(`Add three todos: 'Buy milk', 'Walk dog', 'Finish report'`); + await agent.expect(`All three todos are visible`); + await agent.expect(`Counter shows '3 items left'`); - await page.agent.perform(`Complete the first todo by clicking its checkbox`); - await page.agent.expect(`First todo is marked as complete`); - await page.agent.expect(`Counter shows '2 items left'`); + await agent.perform(`Complete the first todo by clicking its checkbox`); + await agent.expect(`First todo is marked as complete`); + await agent.expect(`Counter shows '2 items left'`); - await page.agent.perform(`Complete the third todo by clicking its checkbox`); - await page.agent.expect(`Third todo is marked as complete`); - await page.agent.expect(`Counter shows '1 item left'`); - await page.agent.expect(`The 'Clear completed' button appears`); + await agent.perform(`Complete the third todo by clicking its checkbox`); + await agent.expect(`Third todo is marked as complete`); + await agent.expect(`Counter shows '1 item left'`); + await agent.expect(`The 'Clear completed' button appears`); }); diff --git a/examples/todomvc/tests/perform/completing-todos/should-complete-single-todo.spec.ts b/examples/todomvc/tests/perform/completing-todos/should-complete-single-todo.spec.ts index 614c86866176e..2b8494a1d42c9 100644 --- a/examples/todomvc/tests/perform/completing-todos/should-complete-single-todo.spec.ts +++ b/examples/todomvc/tests/perform/completing-todos/should-complete-single-todo.spec.ts @@ -1,23 +1,14 @@ import { test } from '../../fixtures'; -test.use({ - agent: { - api: 'anthropic', - apiKey: process.env.AZURE_SONNET_API_KEY!, - apiEndpoint: process.env.AZURE_SONNET_ENDPOINT!, - model: 'claude-sonnet-4-5', - } -}); - -test('should complete single todo', async ({ page }) => { - await page.agent.expect(`The page loads with an empty todo list`); +test('should complete single todo', async ({ agent }) => { + await agent.expect(`The page loads with an empty todo list`); - await page.agent.perform(`Add a todo 'Buy groceries'`); - await page.agent.expect(`The todo appears as active`); - await page.agent.expect(`Counter shows '1 item left'`); + await agent.perform(`Add a todo 'Buy groceries'`); + await agent.expect(`The todo appears as active`); + await agent.expect(`Counter shows '1 item left'`); - await page.agent.perform(`Click the checkbox next to the todo`); - await page.agent.expect(`The checkbox is checked`); - await page.agent.expect(`Counter shows '0 items left'`); - await page.agent.expect(`The 'Clear completed' button appears in the footer`); + await agent.perform(`Click the checkbox next to the todo`); + await agent.expect(`The checkbox is checked`); + await agent.expect(`Counter shows '0 items left'`); + await agent.expect(`The 'Clear completed' button appears in the footer`); }); diff --git a/examples/todomvc/tests/perform/completing-todos/should-toggle-all-todos-complete.spec.ts b/examples/todomvc/tests/perform/completing-todos/should-toggle-all-todos-complete.spec.ts index 7623fb5499640..0f3b6e540a304 100644 --- a/examples/todomvc/tests/perform/completing-todos/should-toggle-all-todos-complete.spec.ts +++ b/examples/todomvc/tests/perform/completing-todos/should-toggle-all-todos-complete.spec.ts @@ -1,24 +1,15 @@ import { test } from '../../fixtures'; -test.use({ - agent: { - api: 'anthropic', - apiKey: process.env.AZURE_SONNET_API_KEY!, - apiEndpoint: process.env.AZURE_SONNET_ENDPOINT!, - model: 'claude-sonnet-4-5', - } -}); - -test('should toggle all todos complete', async ({ page }) => { - await page.agent.expect(`The page loads with an empty todo list`); - - await page.agent.perform(`Add three todos: 'Task 1', 'Task 2', 'Task 3'`); - await page.agent.expect(`All three todos are visible and active`); - await page.agent.expect(`Counter shows '3 items left'`); - - await page.agent.perform(`Click the 'Mark all as complete' checkbox`); - await page.agent.expect(`All three todos are marked as complete`); - await page.agent.expect(`All checkboxes are checked`); - await page.agent.expect(`Counter shows '0 items left'`); - await page.agent.expect(`The 'Clear completed' button appears`); +test('should toggle all todos complete', async ({ agent }) => { + await agent.expect(`The page loads with an empty todo list`); + + await agent.perform(`Add three todos: 'Task 1', 'Task 2', 'Task 3'`); + await agent.expect(`All three todos are visible and active`); + await agent.expect(`Counter shows '3 items left'`); + + await agent.perform(`Click the 'Mark all as complete' checkbox`); + await agent.expect(`All three todos are marked as complete`); + await agent.expect(`All checkboxes are checked`); + await agent.expect(`Counter shows '0 items left'`); + await agent.expect(`The 'Clear completed' button appears`); }); diff --git a/examples/todomvc/tests/perform/completing-todos/should-toggle-all-todos-incomplete.spec.ts b/examples/todomvc/tests/perform/completing-todos/should-toggle-all-todos-incomplete.spec.ts index 05767261a26b0..63a56828f1eff 100644 --- a/examples/todomvc/tests/perform/completing-todos/should-toggle-all-todos-incomplete.spec.ts +++ b/examples/todomvc/tests/perform/completing-todos/should-toggle-all-todos-incomplete.spec.ts @@ -1,24 +1,15 @@ import { test } from '../../fixtures'; -test.use({ - agent: { - api: 'anthropic', - apiKey: process.env.AZURE_SONNET_API_KEY!, - apiEndpoint: process.env.AZURE_SONNET_ENDPOINT!, - model: 'claude-sonnet-4-5', - } -}); - -test('should toggle all todos incomplete', async ({ page }) => { - await page.agent.expect(`The page loads with an empty todo list`); - - await page.agent.perform(`Add three todos: 'Task 1', 'Task 2', 'Task 3' and mark all as complete using the toggle all checkbox`); - await page.agent.expect(`All todos are marked as complete`); - await page.agent.expect(`Counter shows '0 items left'`); - - await page.agent.perform(`Click the 'Mark all as complete' checkbox again`); - await page.agent.expect(`All todos are marked as active`); - await page.agent.expect(`All checkboxes are unchecked`); - await page.agent.expect(`Counter shows '3 items left'`); - await page.agent.expect(`The 'Clear completed' button disappears`); +test('should toggle all todos incomplete', async ({ agent }) => { + await agent.expect(`The page loads with an empty todo list`); + + await agent.perform(`Add three todos: 'Task 1', 'Task 2', 'Task 3' and mark all as complete using the toggle all checkbox`); + await agent.expect(`All todos are marked as complete`); + await agent.expect(`Counter shows '0 items left'`); + + await agent.perform(`Click the 'Mark all as complete' checkbox again`); + await agent.expect(`All todos are marked as active`); + await agent.expect(`All checkboxes are unchecked`); + await agent.expect(`Counter shows '3 items left'`); + await agent.expect(`The 'Clear completed' button disappears`); }); diff --git a/examples/todomvc/tests/perform/completing-todos/should-uncomplete-completed-todo.spec.ts b/examples/todomvc/tests/perform/completing-todos/should-uncomplete-completed-todo.spec.ts index db4d2cc9745f7..96ef73e2e7b65 100644 --- a/examples/todomvc/tests/perform/completing-todos/should-uncomplete-completed-todo.spec.ts +++ b/examples/todomvc/tests/perform/completing-todos/should-uncomplete-completed-todo.spec.ts @@ -1,23 +1,14 @@ import { test } from '../../fixtures'; -test.use({ - agent: { - api: 'anthropic', - apiKey: process.env.AZURE_SONNET_API_KEY!, - apiEndpoint: process.env.AZURE_SONNET_ENDPOINT!, - model: 'claude-sonnet-4-5', - } -}); - -test('should uncomplete completed todo', async ({ page }) => { - await page.agent.expect(`The page loads with an empty todo list`); +test('should uncomplete completed todo', async ({ agent }) => { + await agent.expect(`The page loads with an empty todo list`); - await page.agent.perform(`Add a todo 'Buy groceries' and mark it as complete by clicking its checkbox`); - await page.agent.expect(`The todo is marked as complete`); - await page.agent.expect(`Counter shows '0 items left'`); + await agent.perform(`Add a todo 'Buy groceries' and mark it as complete by clicking its checkbox`); + await agent.expect(`The todo is marked as complete`); + await agent.expect(`Counter shows '0 items left'`); - await page.agent.perform(`Click the checkbox again to uncomplete it`); - await page.agent.expect(`The checkbox is unchecked`); - await page.agent.expect(`Counter shows '1 item left'`); - await page.agent.expect(`The 'Clear completed' button disappears`); + await agent.perform(`Click the checkbox again to uncomplete it`); + await agent.expect(`The checkbox is unchecked`); + await agent.expect(`Counter shows '1 item left'`); + await agent.expect(`The 'Clear completed' button disappears`); }); diff --git a/examples/todomvc/tests/perform/deleting-todos/should-clear-all-completed-todos.spec.ts b/examples/todomvc/tests/perform/deleting-todos/should-clear-all-completed-todos.spec.ts index 3f5804e027b9a..0491f03ce8bca 100644 --- a/examples/todomvc/tests/perform/deleting-todos/should-clear-all-completed-todos.spec.ts +++ b/examples/todomvc/tests/perform/deleting-todos/should-clear-all-completed-todos.spec.ts @@ -1,28 +1,19 @@ import { test } from '../../fixtures'; -test.use({ - agent: { - api: 'anthropic', - apiKey: process.env.AZURE_SONNET_API_KEY!, - apiEndpoint: process.env.AZURE_SONNET_ENDPOINT!, - model: 'claude-sonnet-4-5', - } -}); - -test('should clear all completed todos', async ({ page }) => { - await page.agent.expect(`The page loads with an empty todo list`); +test('should clear all completed todos', async ({ agent }) => { + await agent.expect(`The page loads with an empty todo list`); - await page.agent.perform(`Add three todos: 'Task 1', 'Task 2', 'Task 3'`); - await page.agent.expect(`All three todos are visible`); + await agent.perform(`Add three todos: 'Task 1', 'Task 2', 'Task 3'`); + await agent.expect(`All three todos are visible`); - await page.agent.perform(`Mark 'Task 1' and 'Task 3' as complete by clicking their checkboxes`); - await page.agent.expect(`Two todos are marked as complete`); - await page.agent.expect(`Counter shows '1 item left'`); - await page.agent.expect(`The 'Clear completed' button appears`); + await agent.perform(`Mark 'Task 1' and 'Task 3' as complete by clicking their checkboxes`); + await agent.expect(`Two todos are marked as complete`); + await agent.expect(`Counter shows '1 item left'`); + await agent.expect(`The 'Clear completed' button appears`); - await page.agent.perform(`Click the 'Clear completed' button`); - await page.agent.expect(`'Task 1' and 'Task 3' are removed from the list`); - await page.agent.expect(`Only 'Task 2' remains visible`); - await page.agent.expect(`Counter shows '1 item left'`); - await page.agent.expect(`The 'Clear completed' button disappears`); + await agent.perform(`Click the 'Clear completed' button`); + await agent.expect(`'Task 1' and 'Task 3' are removed from the list`); + await agent.expect(`Only 'Task 2' remains visible`); + await agent.expect(`Counter shows '1 item left'`); + await agent.expect(`The 'Clear completed' button disappears`); }); diff --git a/examples/todomvc/tests/perform/deleting-todos/should-delete-single-todo.spec.ts b/examples/todomvc/tests/perform/deleting-todos/should-delete-single-todo.spec.ts index 59610a89a9dd9..150e209a93538 100644 --- a/examples/todomvc/tests/perform/deleting-todos/should-delete-single-todo.spec.ts +++ b/examples/todomvc/tests/perform/deleting-todos/should-delete-single-todo.spec.ts @@ -1,26 +1,17 @@ import { test } from '../../fixtures'; -test.use({ - agent: { - api: 'anthropic', - apiKey: process.env.AZURE_SONNET_API_KEY!, - apiEndpoint: process.env.AZURE_SONNET_ENDPOINT!, - model: 'claude-sonnet-4-5', - } -}); - -test('should delete single todo', async ({ page }) => { - await page.agent.expect(`The page loads with an empty todo list`); +test('should delete single todo', async ({ agent }) => { + await agent.expect(`The page loads with an empty todo list`); - await page.agent.perform(`Add a todo 'Task to delete'`); - await page.agent.expect(`The todo appears in the list`); - await page.agent.expect(`Counter shows '1 item left'`); + await agent.perform(`Add a todo 'Task to delete'`); + await agent.expect(`The todo appears in the list`); + await agent.expect(`Counter shows '1 item left'`); - await page.agent.perform(`Hover over the todo item`); - await page.agent.expect(`A delete button (x) appears on the right side of the todo`); + await agent.perform(`Hover over the todo item`); + await agent.expect(`A delete button (x) appears on the right side of the todo`); - await page.agent.perform(`Click the delete button`); - await page.agent.expect(`The todo is removed from the list`); - await page.agent.expect(`The list is empty`); - await page.agent.expect(`The footer is hidden or shows '0 items left'`); + await agent.perform(`Click the delete button`); + await agent.expect(`The todo is removed from the list`); + await agent.expect(`The list is empty`); + await agent.expect(`The footer is hidden or shows '0 items left'`); }); diff --git a/examples/todomvc/tests/perform/deleting-todos/should-delete-specific-todo-from-multiple.spec.ts b/examples/todomvc/tests/perform/deleting-todos/should-delete-specific-todo-from-multiple.spec.ts index 4ccb1393b45e1..03020597d17ab 100644 --- a/examples/todomvc/tests/perform/deleting-todos/should-delete-specific-todo-from-multiple.spec.ts +++ b/examples/todomvc/tests/perform/deleting-todos/should-delete-specific-todo-from-multiple.spec.ts @@ -1,23 +1,14 @@ import { test } from '../../fixtures'; -test.use({ - agent: { - api: 'anthropic', - apiKey: process.env.AZURE_SONNET_API_KEY!, - apiEndpoint: process.env.AZURE_SONNET_ENDPOINT!, - model: 'claude-sonnet-4-5', - } -}); - -test('should delete specific todo from multiple', async ({ page }) => { - await page.agent.expect(`The page loads with an empty todo list`); +test('should delete specific todo from multiple', async ({ agent }) => { + await agent.expect(`The page loads with an empty todo list`); - await page.agent.perform(`Add three todos: 'Task 1', 'Task 2', 'Task 3'`); - await page.agent.expect(`All three todos appear in the list`); - await page.agent.expect(`Counter shows '3 items left'`); + await agent.perform(`Add three todos: 'Task 1', 'Task 2', 'Task 3'`); + await agent.expect(`All three todos appear in the list`); + await agent.expect(`Counter shows '3 items left'`); - await page.agent.perform(`Hover over 'Task 2' and click its delete button`); - await page.agent.expect(`'Task 2' is removed from the list`); - await page.agent.expect(`'Task 1' and 'Task 3' remain visible`); - await page.agent.expect(`Counter shows '2 items left'`); + await agent.perform(`Hover over 'Task 2' and click its delete button`); + await agent.expect(`'Task 2' is removed from the list`); + await agent.expect(`'Task 1' and 'Task 3' remain visible`); + await agent.expect(`Counter shows '2 items left'`); }); diff --git a/examples/todomvc/tests/perform/editing-todos/should-cancel-edit-on-escape.spec.ts b/examples/todomvc/tests/perform/editing-todos/should-cancel-edit-on-escape.spec.ts index 4e684a89d74b6..be4b2ef713647 100644 --- a/examples/todomvc/tests/perform/editing-todos/should-cancel-edit-on-escape.spec.ts +++ b/examples/todomvc/tests/perform/editing-todos/should-cancel-edit-on-escape.spec.ts @@ -1,25 +1,16 @@ import { test } from '../../fixtures'; -test.use({ - agent: { - api: 'anthropic', - apiKey: process.env.AZURE_SONNET_API_KEY!, - apiEndpoint: process.env.AZURE_SONNET_ENDPOINT!, - model: 'claude-sonnet-4-5', - } -}); - -test('should cancel edit on escape', async ({ page }) => { - await page.agent.expect(`The page loads with an empty todo list`); +test('should cancel edit on escape', async ({ agent }) => { + await agent.expect(`The page loads with an empty todo list`); - await page.agent.perform(`Add a todo 'Original text'`); - await page.agent.expect(`The todo appears in the list`); + await agent.perform(`Add a todo 'Original text'`); + await agent.expect(`The todo appears in the list`); - await page.agent.perform(`Double-click on the todo to enter edit mode`); - await page.agent.expect(`Edit textbox appears with 'Original text'`); + await agent.perform(`Double-click on the todo to enter edit mode`); + await agent.expect(`Edit textbox appears with 'Original text'`); - await page.agent.perform(`Change the text to 'Modified text' but press Escape instead of Enter`); - await page.agent.expect(`Edit mode is cancelled`); - await page.agent.expect(`The todo text reverts to 'Original text'`); - await page.agent.expect(`Changes are not saved`); + await agent.perform(`Change the text to 'Modified text' but press Escape instead of Enter`); + await agent.expect(`Edit mode is cancelled`); + await agent.expect(`The todo text reverts to 'Original text'`); + await agent.expect(`Changes are not saved`); }); diff --git a/examples/todomvc/tests/perform/editing-todos/should-delete-todo-when-edited-to-empty.spec.ts b/examples/todomvc/tests/perform/editing-todos/should-delete-todo-when-edited-to-empty.spec.ts index 122ae73bf241d..ffaa7fbb5fe0c 100644 --- a/examples/todomvc/tests/perform/editing-todos/should-delete-todo-when-edited-to-empty.spec.ts +++ b/examples/todomvc/tests/perform/editing-todos/should-delete-todo-when-edited-to-empty.spec.ts @@ -1,26 +1,17 @@ import { test } from '../../fixtures'; -test.use({ - agent: { - api: 'anthropic', - apiKey: process.env.AZURE_SONNET_API_KEY!, - apiEndpoint: process.env.AZURE_SONNET_ENDPOINT!, - model: 'claude-sonnet-4-5', - } -}); - -test('should delete todo when edited to empty', async ({ page }) => { - await page.agent.expect(`The page loads with an empty todo list`); +test('should delete todo when edited to empty', async ({ agent }) => { + await agent.expect(`The page loads with an empty todo list`); - await page.agent.perform(`Add a todo 'Temporary task'`); - await page.agent.expect(`The todo appears in the list`); - await page.agent.expect(`Counter shows '1 item left'`); + await agent.perform(`Add a todo 'Temporary task'`); + await agent.expect(`The todo appears in the list`); + await agent.expect(`Counter shows '1 item left'`); - await page.agent.perform(`Double-click on the todo to enter edit mode`); - await page.agent.expect(`Edit textbox appears`); + await agent.perform(`Double-click on the todo to enter edit mode`); + await agent.expect(`Edit textbox appears`); - await page.agent.perform(`Clear all the text and press Enter`); - await page.agent.expect(`The todo is deleted from the list`); - await page.agent.expect(`The list is empty`); - await page.agent.expect(`Counter shows '0 items left' or the footer is hidden`); + await agent.perform(`Clear all the text and press Enter`); + await agent.expect(`The todo is deleted from the list`); + await agent.expect(`The list is empty`); + await agent.expect(`Counter shows '0 items left' or the footer is hidden`); }); diff --git a/examples/todomvc/tests/perform/editing-todos/should-edit-todo-by-double-clicking.spec.ts b/examples/todomvc/tests/perform/editing-todos/should-edit-todo-by-double-clicking.spec.ts index 067af01a40946..a11d582c43171 100644 --- a/examples/todomvc/tests/perform/editing-todos/should-edit-todo-by-double-clicking.spec.ts +++ b/examples/todomvc/tests/perform/editing-todos/should-edit-todo-by-double-clicking.spec.ts @@ -1,27 +1,18 @@ import { test } from '../../fixtures'; -test.use({ - agent: { - api: 'anthropic', - apiKey: process.env.AZURE_SONNET_API_KEY!, - apiEndpoint: process.env.AZURE_SONNET_ENDPOINT!, - model: 'claude-sonnet-4-5', - } -}); - -test('should edit todo by double clicking', async ({ page }) => { - await page.agent.expect(`The page loads with an empty todo list`); +test('should edit todo by double clicking', async ({ agent }) => { + await agent.expect(`The page loads with an empty todo list`); - await page.agent.perform(`Add a todo 'Buy milk'`); - await page.agent.expect(`The todo appears in the list`); + await agent.perform(`Add a todo 'Buy milk'`); + await agent.expect(`The todo appears in the list`); - await page.agent.perform(`Double-click on the todo text`); - await page.agent.expect(`The todo enters edit mode`); - await page.agent.expect(`An edit textbox appears with the current text 'Buy milk'`); - await page.agent.expect(`The textbox is focused`); + await agent.perform(`Double-click on the todo text`); + await agent.expect(`The todo enters edit mode`); + await agent.expect(`An edit textbox appears with the current text 'Buy milk'`); + await agent.expect(`The textbox is focused`); - await page.agent.perform(`Change the text to 'Buy organic milk' and press Enter`); - await page.agent.expect(`The todo is updated to 'Buy organic milk'`); - await page.agent.expect(`Edit mode is exited`); - await page.agent.expect(`The updated text is displayed in the list`); + await agent.perform(`Change the text to 'Buy organic milk' and press Enter`); + await agent.expect(`The todo is updated to 'Buy organic milk'`); + await agent.expect(`Edit mode is exited`); + await agent.expect(`The updated text is displayed in the list`); }); diff --git a/examples/todomvc/tests/perform/editing-todos/should-save-edit-on-blur.spec.ts b/examples/todomvc/tests/perform/editing-todos/should-save-edit-on-blur.spec.ts index 2d0c59835aec8..cf429fb85b235 100644 --- a/examples/todomvc/tests/perform/editing-todos/should-save-edit-on-blur.spec.ts +++ b/examples/todomvc/tests/perform/editing-todos/should-save-edit-on-blur.spec.ts @@ -1,25 +1,16 @@ import { test } from '../../fixtures'; -test.use({ - agent: { - api: 'anthropic', - apiKey: process.env.AZURE_SONNET_API_KEY!, - apiEndpoint: process.env.AZURE_SONNET_ENDPOINT!, - model: 'claude-sonnet-4-5', - } -}); - -test('should save edit on blur', async ({ page }) => { - await page.agent.expect(`The page loads with an empty todo list`); +test('should save edit on blur', async ({ agent }) => { + await agent.expect(`The page loads with an empty todo list`); - await page.agent.perform(`Add a todo 'Call dentist'`); - await page.agent.expect(`The todo appears in the list`); + await agent.perform(`Add a todo 'Call dentist'`); + await agent.expect(`The todo appears in the list`); - await page.agent.perform(`Double-click on the todo to enter edit mode`); - await page.agent.expect(`Edit textbox appears`); + await agent.perform(`Double-click on the todo to enter edit mode`); + await agent.expect(`Edit textbox appears`); - await page.agent.perform(`Change the text to 'Schedule dentist appointment' and click elsewhere to blur the input`); - await page.agent.expect(`The changes are saved`); - await page.agent.expect(`The todo text is updated to 'Schedule dentist appointment'`); - await page.agent.expect(`Edit mode is exited`); + await agent.perform(`Change the text to 'Schedule dentist appointment' and click elsewhere to blur the input`); + await agent.expect(`The changes are saved`); + await agent.expect(`The todo text is updated to 'Schedule dentist appointment'`); + await agent.expect(`Edit mode is exited`); }); diff --git a/examples/todomvc/tests/perform/editing-todos/should-trim-whitespace-when-editing.spec.ts b/examples/todomvc/tests/perform/editing-todos/should-trim-whitespace-when-editing.spec.ts index 218a60872724a..4370afa23a292 100644 --- a/examples/todomvc/tests/perform/editing-todos/should-trim-whitespace-when-editing.spec.ts +++ b/examples/todomvc/tests/perform/editing-todos/should-trim-whitespace-when-editing.spec.ts @@ -1,23 +1,14 @@ import { test } from '../../fixtures'; -test.use({ - agent: { - api: 'anthropic', - apiKey: process.env.AZURE_SONNET_API_KEY!, - apiEndpoint: process.env.AZURE_SONNET_ENDPOINT!, - model: 'claude-sonnet-4-5', - } -}); - -test('should trim whitespace when editing', async ({ page }) => { - await page.agent.expect(`The page loads with an empty todo list`); +test('should trim whitespace when editing', async ({ agent }) => { + await agent.expect(`The page loads with an empty todo list`); - await page.agent.perform(`Add a todo 'Original task'`); - await page.agent.expect(`The todo appears in the list`); + await agent.perform(`Add a todo 'Original task'`); + await agent.expect(`The todo appears in the list`); - await page.agent.perform(`Double-click to edit and change text to ' Edited task ' (with leading and trailing spaces)`); - await page.agent.expect(`Edit textbox shows the text with spaces`); + await agent.perform(`Double-click to edit and change text to ' Edited task ' (with leading and trailing spaces)`); + await agent.expect(`Edit textbox shows the text with spaces`); - await page.agent.perform(`Press Enter to save`); - await page.agent.expect(`The todo is saved as 'Edited task' without leading or trailing whitespace`); + await agent.perform(`Press Enter to save`); + await agent.expect(`The todo is saved as 'Edited task' without leading or trailing whitespace`); }); diff --git a/examples/todomvc/tests/perform/filtering-todos/should-filter-active-todos.spec.ts b/examples/todomvc/tests/perform/filtering-todos/should-filter-active-todos.spec.ts index 2ee27d8a0e61e..e836bd141334d 100644 --- a/examples/todomvc/tests/perform/filtering-todos/should-filter-active-todos.spec.ts +++ b/examples/todomvc/tests/perform/filtering-todos/should-filter-active-todos.spec.ts @@ -1,27 +1,18 @@ import { test } from '../../fixtures'; -test.use({ - agent: { - api: 'anthropic', - apiKey: process.env.AZURE_SONNET_API_KEY!, - apiEndpoint: process.env.AZURE_SONNET_ENDPOINT!, - model: 'claude-sonnet-4-5', - } -}); - -test('should filter active todos', async ({ page }) => { - await page.agent.expect(`The page loads with an empty todo list`); +test('should filter active todos', async ({ agent }) => { + await agent.expect(`The page loads with an empty todo list`); - await page.agent.perform(`Add three todos: 'Active 1', 'Active 2', 'Will complete'`); - await page.agent.expect(`All three todos are visible`); + await agent.perform(`Add three todos: 'Active 1', 'Active 2', 'Will complete'`); + await agent.expect(`All three todos are visible`); - await page.agent.perform(`Mark 'Will complete' as completed by clicking its checkbox`); - await page.agent.expect(`One todo is marked as complete`); - await page.agent.expect(`Counter shows '2 items left'`); + await agent.perform(`Mark 'Will complete' as completed by clicking its checkbox`); + await agent.expect(`One todo is marked as complete`); + await agent.expect(`Counter shows '2 items left'`); - await page.agent.perform(`Click on the 'Active' filter link`); - await page.agent.expect(`The URL changes to #/active`); - await page.agent.expect(`Only 'Active 1' and 'Active 2' are displayed`); - await page.agent.expect(`'Will complete' is not visible`); - await page.agent.expect(`The 'Active' filter link is highlighted`); + await agent.perform(`Click on the 'Active' filter link`); + await agent.expect(`The URL changes to #/active`); + await agent.expect(`Only 'Active 1' and 'Active 2' are displayed`); + await agent.expect(`'Will complete' is not visible`); + await agent.expect(`The 'Active' filter link is highlighted`); }); diff --git a/examples/todomvc/tests/perform/filtering-todos/should-filter-completed-todos.spec.ts b/examples/todomvc/tests/perform/filtering-todos/should-filter-completed-todos.spec.ts index 5f987140b6048..28891f1b11fd2 100644 --- a/examples/todomvc/tests/perform/filtering-todos/should-filter-completed-todos.spec.ts +++ b/examples/todomvc/tests/perform/filtering-todos/should-filter-completed-todos.spec.ts @@ -1,26 +1,17 @@ import { test } from '../../fixtures'; -test.use({ - agent: { - api: 'anthropic', - apiKey: process.env.AZURE_SONNET_API_KEY!, - apiEndpoint: process.env.AZURE_SONNET_ENDPOINT!, - model: 'claude-sonnet-4-5', - } -}); - -test('should filter completed todos', async ({ page }) => { - await page.agent.expect(`The page loads with an empty todo list`); +test('should filter completed todos', async ({ agent }) => { + await agent.expect(`The page loads with an empty todo list`); - await page.agent.perform(`Add three todos: 'Active task', 'Completed 1', 'Completed 2'`); - await page.agent.expect(`All three todos are visible`); + await agent.perform(`Add three todos: 'Active task', 'Completed 1', 'Completed 2'`); + await agent.expect(`All three todos are visible`); - await page.agent.perform(`Mark 'Completed 1' and 'Completed 2' as completed by clicking their checkboxes`); - await page.agent.expect(`Two todos are marked as complete`); + await agent.perform(`Mark 'Completed 1' and 'Completed 2' as completed by clicking their checkboxes`); + await agent.expect(`Two todos are marked as complete`); - await page.agent.perform(`Click on the 'Completed' filter link`); - await page.agent.expect(`The URL changes to #/completed`); - await page.agent.expect(`Only 'Completed 1' and 'Completed 2' are displayed`); - await page.agent.expect(`'Active task' is not visible`); - await page.agent.expect(`The 'Completed' filter link is highlighted`); + await agent.perform(`Click on the 'Completed' filter link`); + await agent.expect(`The URL changes to #/completed`); + await agent.expect(`Only 'Completed 1' and 'Completed 2' are displayed`); + await agent.expect(`'Active task' is not visible`); + await agent.expect(`The 'Completed' filter link is highlighted`); }); diff --git a/examples/todomvc/tests/perform/filtering-todos/should-show-all-todos-with-all-filter.spec.ts b/examples/todomvc/tests/perform/filtering-todos/should-show-all-todos-with-all-filter.spec.ts index 9835b46d50973..ec31f5d73491f 100644 --- a/examples/todomvc/tests/perform/filtering-todos/should-show-all-todos-with-all-filter.spec.ts +++ b/examples/todomvc/tests/perform/filtering-todos/should-show-all-todos-with-all-filter.spec.ts @@ -1,25 +1,16 @@ import { test } from '../../fixtures'; -test.use({ - agent: { - api: 'anthropic', - apiKey: process.env.AZURE_SONNET_API_KEY!, - apiEndpoint: process.env.AZURE_SONNET_ENDPOINT!, - model: 'claude-sonnet-4-5', - } -}); - -test('should show all todos with all filter', async ({ page }) => { - await page.agent.expect(`The page loads with an empty todo list`); +test('should show all todos with all filter', async ({ agent }) => { + await agent.expect(`The page loads with an empty todo list`); - await page.agent.perform(`Add three todos and mark one as complete`); - await page.agent.expect(`Three todos exist, one completed and two active`); + await agent.perform(`Add three todos and mark one as complete`); + await agent.expect(`Three todos exist, one completed and two active`); - await page.agent.perform(`Navigate to the 'Active' filter by clicking on it`); - await page.agent.expect(`Only active todos are visible`); + await agent.perform(`Navigate to the 'Active' filter by clicking on it`); + await agent.expect(`Only active todos are visible`); - await page.agent.perform(`Click on the 'All' filter link`); - await page.agent.expect(`The URL changes to #/`); - await page.agent.expect(`All todos (both completed and active) are displayed`); - await page.agent.expect(`The 'All' filter link is highlighted`); + await agent.perform(`Click on the 'All' filter link`); + await agent.expect(`The URL changes to #/`); + await agent.expect(`All todos (both completed and active) are displayed`); + await agent.expect(`The 'All' filter link is highlighted`); }); diff --git a/examples/todomvc/tests/perform/persistence/should-persist-todos-after-page-reload.spec.ts b/examples/todomvc/tests/perform/persistence/should-persist-todos-after-page-reload.spec.ts index ac40e3a638893..2acc94427d32d 100644 --- a/examples/todomvc/tests/perform/persistence/should-persist-todos-after-page-reload.spec.ts +++ b/examples/todomvc/tests/perform/persistence/should-persist-todos-after-page-reload.spec.ts @@ -1,25 +1,16 @@ import { test } from '../../fixtures'; -test.use({ - agent: { - api: 'anthropic', - apiKey: process.env.AZURE_SONNET_API_KEY!, - apiEndpoint: process.env.AZURE_SONNET_ENDPOINT!, - model: 'claude-sonnet-4-5', - } -}); - -test('should persist todos after page reload', async ({ page }) => { - await page.agent.expect(`The page loads with an empty todo list`); +test('should persist todos after page reload', async ({ agent }) => { + await agent.expect(`The page loads with an empty todo list`); - await page.agent.perform(`Add three todos: 'Persistent 1', 'Persistent 2', 'Persistent 3'`); - await page.agent.expect(`All three todos appear in the list`); + await agent.perform(`Add three todos: 'Persistent 1', 'Persistent 2', 'Persistent 3'`); + await agent.expect(`All three todos appear in the list`); - await page.agent.perform(`Mark 'Persistent 2' as completed by clicking its checkbox`); - await page.agent.expect(`'Persistent 2' is marked as complete`); + await agent.perform(`Mark 'Persistent 2' as completed by clicking its checkbox`); + await agent.expect(`'Persistent 2' is marked as complete`); - await page.agent.perform(`Reload the page`); - await page.agent.expect(`All three todos are still present after reload`); - await page.agent.expect(`'Persistent 2' is still marked as complete`); - await page.agent.expect(`The counter shows '2 items left'`); + await agent.perform(`Reload the page`); + await agent.expect(`All three todos are still present after reload`); + await agent.expect(`'Persistent 2' is still marked as complete`); + await agent.expect(`The counter shows '2 items left'`); }); diff --git a/examples/todomvc/tests/perform/ui-state/should-hide-footer-when-no-todos.spec.ts b/examples/todomvc/tests/perform/ui-state/should-hide-footer-when-no-todos.spec.ts deleted file mode 100644 index c0dc31df3259d..0000000000000 --- a/examples/todomvc/tests/perform/ui-state/should-hide-footer-when-no-todos.spec.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { test } from '../../fixtures'; - -test.use({ - agent: { - api: 'anthropic', - apiKey: process.env.AZURE_SONNET_API_KEY!, - apiEndpoint: process.env.AZURE_SONNET_ENDPOINT!, - model: 'claude-sonnet-4-5', - } -}); - -test('should hide footer when no todos', async ({ page }) => { - await page.agent.expect(`The page loads with an empty todo list`); - await page.agent.expect(`The footer with counter and filters is not visible`); - await page.agent.expect(`Only the input field and heading are visible`); - - await page.agent.perform(`Add a todo 'First task'`); - await page.agent.expect(`The footer appears with counter '1 item left' and filter links`); - - await page.agent.perform(`Delete the todo by hovering over it and clicking the delete button`); - await page.agent.expect(`The footer is hidden again`); -}); diff --git a/examples/todomvc/tests/perform/ui-state/should-hide-footer-when-no-todos.spec.ts-cache.json b/examples/todomvc/tests/perform/ui-state/should-hide-footer-when-no-todos.spec.ts-cache.json deleted file mode 100644 index 128cc0b8e5690..0000000000000 --- a/examples/todomvc/tests/perform/ui-state/should-hide-footer-when-no-todos.spec.ts-cache.json +++ /dev/null @@ -1,85 +0,0 @@ -{ - "Add a todo 'First task'": { - "timestamp": 1767733696461, - "actions": [ - { - "method": "fill", - "selector": "internal:role=textbox[name=\"What needs to be done?\"i]", - "text": "First task", - "submit": true, - "code": "await page.getByRole('textbox', { name: 'What needs to be done?' }).fill('First task');\nawait page.keyboard.press('Enter');", - "intent": "I need to add a todo with the text 'First task'. I can see there's a textbox with the placeholder \"What needs to be done?\" that I can type into." - } - ] - }, - "Delete the todo by hovering over it and clicking the delete button": { - "timestamp": 1767733726006, - "actions": [ - { - "method": "hover", - "selector": "internal:testid=[data-testid=\"todo-item\"s] >> div", - "options": {}, - "code": "await page.getByTestId('todo-item').locator('div').hover();", - "intent": "I can see there's a todo item with \"First task\" text. I can see there's a \"×\" text which appears to be the delete button. According to TodoMVC conventions, the delete button typically appears when you hover over the todo item. Let me hover over the todo item (the listitem or the generic container) to reveal the delete button." - }, - { - "method": "click", - "selector": "internal:role=button[name=\"Delete\"i]", - "options": {}, - "code": "await page.getByRole('button', { name: 'Delete' }).click();", - "intent": "Perfect! After hovering over the todo item, the delete button is now visible. I can see it's a button labeled \"Delete\" with the \"×\" symbol (ref=e33). Now I'll click on it to delete the todo." - } - ] - }, - "Only the input field and heading are visible": { - "timestamp": 1767733686256, - "actions": [ - { - "method": "expectVisible", - "selector": "internal:role=contentinfo[name=\"\"i]", - "isNot": true, - "code": "await expect(page.getByRole('contentinfo', { name: '' })).not.toBeVisible();", - "intent": "I need to verify that only the input field and heading are visible on the page. Looking at the snapshot, I can see:\n\n1. A heading \"todos\" (ref=e7)\n2. A textbox \"What needs to be done?\" (ref=e8)\n3. Other elements like links, paragraphs in the contentinfo section\n\nThe expectation states that ONLY the input field and heading should be visible, which means other elements like the contentinfo section should not be visible. I'll verify that the contentinfo element is not visible." - } - ] - }, - "The footer appears with counter '1 item left' and filter links": { - "timestamp": 1767733702303, - "actions": [] - }, - "The footer is hidden again": { - "timestamp": 1767733740504, - "actions": [ - { - "method": "expectVisible", - "selector": "internal:role=contentinfo[name=\"\"i]", - "isNot": true, - "code": "await expect(page.getByRole('contentinfo', { name: '' })).not.toBeVisible();", - "intent": "I need to verify that the footer is hidden. Looking at the page snapshot, I can see a `contentinfo` element (which is the semantic footer element) with ref=e9. The expectation is that this footer should be hidden.\n\nI'll use the `browser_expect_visible` tool with the `isNot` parameter set to true to assert that the footer is NOT visible (i.e., hidden)." - } - ] - }, - "The footer with counter and filters is not visible": { - "timestamp": 1767733675122, - "actions": [ - { - "method": "expectVisible", - "selector": "internal:role=contentinfo[name=\"\"i]", - "isNot": true, - "code": "await expect(page.getByRole('contentinfo', { name: '' })).not.toBeVisible();", - "intent": "I need to verify that the footer with counter and filters is not visible. Looking at the page snapshot, I can see there is a `contentinfo` element (ref=e9) which appears to be a footer, but it contains information about the app (instructions, creator, and TodoMVC link) rather than a counter and filters for todos.\n\nThe footer with counter and filters that would typically appear in a TodoMVC app (showing item count and All/Active/Completed filters) is not present in this snapshot. I'll assert that such an element is not visible by checking for a common pattern - the contentinfo that would contain todo-related controls." - } - ] - }, - "The page loads with an empty todo list": { - "timestamp": 1767733662280, - "actions": [ - { - "method": "expectVisible", - "selector": "internal:role=textbox[name=\"What needs to be done?\"i]", - "code": "await expect(page.getByRole('textbox', { name: 'What needs to be done?' })).toBeVisible();", - "intent": "I need to verify that the page loads with an empty todo list. Looking at the page snapshot, I can see there's a textbox \"What needs to be done?\" which is the input field for adding todos, but I don't see any todo items listed on the page. This indicates the todo list is indeed empty.\n\nTo assert this condition, I should verify that the textbox for adding todos is visible, which confirms the page has loaded properly with the todo functionality, and the absence of any todo items in the snapshot confirms the list is empty." - } - ] - } -} \ No newline at end of file diff --git a/packages/playwright-client/types/types.d.ts b/packages/playwright-client/types/types.d.ts index 644abf75bbe7e..81f9f4bad1f64 100644 --- a/packages/playwright-client/types/types.d.ts +++ b/packages/playwright-client/types/types.d.ts @@ -1018,21 +1018,6 @@ export interface Page { */ behavior?: 'wait'|'ignoreErrors'|'default' }): Promise; - /** - * Emitted when the agent makes a turn. - */ - on(event: 'agentturn', listener: (data: { - role: string; - - message: string; - - usage?: { - inputTokens: number; - - outputTokens: number; - }; - }) => any): this; - /** * Emitted when the page closes. */ @@ -1240,21 +1225,6 @@ export interface Page { */ on(event: 'worker', listener: (worker: Worker) => any): this; - /** - * Adds an event listener that will be automatically removed after it is triggered once. See `addListener` for more information about this event. - */ - once(event: 'agentturn', listener: (data: { - role: string; - - message: string; - - usage?: { - inputTokens: number; - - outputTokens: number; - }; - }) => any): this; - /** * Adds an event listener that will be automatically removed after it is triggered once. See `addListener` for more information about this event. */ @@ -1350,21 +1320,6 @@ export interface Page { */ once(event: 'worker', listener: (worker: Worker) => any): this; - /** - * Emitted when the agent makes a turn. - */ - addListener(event: 'agentturn', listener: (data: { - role: string; - - message: string; - - usage?: { - inputTokens: number; - - outputTokens: number; - }; - }) => any): this; - /** * Emitted when the page closes. */ @@ -1572,21 +1527,6 @@ export interface Page { */ addListener(event: 'worker', listener: (worker: Worker) => any): this; - /** - * Removes an event listener added by `on` or `addListener`. - */ - removeListener(event: 'agentturn', listener: (data: { - role: string; - - message: string; - - usage?: { - inputTokens: number; - - outputTokens: number; - }; - }) => any): this; - /** * Removes an event listener added by `on` or `addListener`. */ @@ -1682,21 +1622,6 @@ export interface Page { */ removeListener(event: 'worker', listener: (worker: Worker) => any): this; - /** - * Removes an event listener added by `on` or `addListener`. - */ - off(event: 'agentturn', listener: (data: { - role: string; - - message: string; - - usage?: { - inputTokens: number; - - outputTokens: number; - }; - }) => any): this; - /** * Removes an event listener added by `on` or `addListener`. */ @@ -1792,21 +1717,6 @@ export interface Page { */ off(event: 'worker', listener: (worker: Worker) => any): this; - /** - * Emitted when the agent makes a turn. - */ - prependListener(event: 'agentturn', listener: (data: { - role: string; - - message: string; - - usage?: { - inputTokens: number; - - outputTokens: number; - }; - }) => any): this; - /** * Emitted when the page closes. */ @@ -2183,6 +2093,58 @@ export interface Page { url?: string; }): Promise; + /** + * Initialize page agent with the llm provider and cache. + * @param options + */ + agent(options?: { + cache?: { + /** + * Cache file to use/generate code for performed actions into. Cache is not used if not specified (default). + */ + cacheFile?: string; + + /** + * When specified, generated entries are written into the `cacheOutFile` instead of updating the `cacheFile`. + */ + cacheOutFile?: string; + }; + + maxTokens?: number; + + /** + * Maximum number of agentic turns to take per call. Defaults to 10. + */ + maxTurns?: number; + + provider?: { + /** + * API to use. + */ + api: "openai"|"openai-compatible"|"anthropic"|"google"; + + /** + * Endpoint to use if different from default. + */ + apiEndpoint?: string; + + /** + * API key for the LLM provider. + */ + apiKey: string; + + /** + * Model identifier within the provider. Required in non-cache mode. + */ + model: string; + }; + + /** + * Secrets to hide from the LLM. + */ + secrets?: { [key: string]: string; }; + }): Promise; + /** * Brings page to front (activates tab). */ @@ -4823,41 +4785,6 @@ export interface Page { height: number; }; - /** - * Emitted when the agent makes a turn. - */ - waitForEvent(event: 'agentturn', optionsOrPredicate?: { predicate?: (data: { - role: string; - - message: string; - - usage?: { - inputTokens: number; - - outputTokens: number; - }; - }) => boolean | Promise, timeout?: number } | ((data: { - role: string; - - message: string; - - usage?: { - inputTokens: number; - - outputTokens: number; - }; - }) => boolean | Promise)): Promise<{ - role: string; - - message: string; - - usage?: { - inputTokens: number; - - outputTokens: number; - }; - }>; - /** * Emitted when the page closes. */ @@ -5303,8 +5230,6 @@ export interface Page { */ workers(): Array; - agent: PageAgent; - /** * Playwright has ability to mock clock and passage of time. */ @@ -5346,7 +5271,7 @@ export interface PageAgent { * **Usage** * * ```js - * await page.agent.extract('List of items in the cart', z.object({ + * await agent.extract('List of items in the cart', z.object({ * title: z.string().describe('Item title to extract'), * price: z.string().describe('Item price to extract'), * }).array()); @@ -5357,36 +5282,114 @@ export interface PageAgent { * @param options */ extract(query: string, schema: Schema): Promise>; + /** + * Emitted when the agent makes a turn. + */ + on(event: 'turn', listener: (data: { + role: string; + + message: string; + + usage?: { + inputTokens: number; + + outputTokens: number; + }; + }) => any): this; + + /** + * Adds an event listener that will be automatically removed after it is triggered once. See `addListener` for more information about this event. + */ + once(event: 'turn', listener: (data: { + role: string; + + message: string; + + usage?: { + inputTokens: number; + + outputTokens: number; + }; + }) => any): this; + + /** + * Emitted when the agent makes a turn. + */ + addListener(event: 'turn', listener: (data: { + role: string; + + message: string; + + usage?: { + inputTokens: number; + + outputTokens: number; + }; + }) => any): this; + + /** + * Removes an event listener added by `on` or `addListener`. + */ + removeListener(event: 'turn', listener: (data: { + role: string; + + message: string; + + usage?: { + inputTokens: number; + + outputTokens: number; + }; + }) => any): this; + + /** + * Removes an event listener added by `on` or `addListener`. + */ + off(event: 'turn', listener: (data: { + role: string; + + message: string; + + usage?: { + inputTokens: number; + + outputTokens: number; + }; + }) => any): this; + + /** + * Emitted when the agent makes a turn. + */ + prependListener(event: 'turn', listener: (data: { + role: string; + + message: string; + + usage?: { + inputTokens: number; + + outputTokens: number; + }; + }) => any): this; + + /** + * Dispose this agent. + */ + dispose(): Promise; + /** * Expect certain condition to be met. * * **Usage** * * ```js - * await page.agent.expect('"0 items" to be reported'); + * await agent.expect('"0 items" to be reported'); * ``` * * @param expectation Expectation to assert. * @param options */ expect(expectation: string, options?: { - /** - * API to use, `openapi`, `google` or `anthropic`. Required in non-cache mode. - */ - api?: string; - - /** - * Endpoint to use if different from default. - */ - apiEndpoint?: string; - - /** - * API key for the LLM provider. - * - * API version if relevant. - */ - apiKey?: string; - /** * All the agentic actions are converted to the Playwright calls and are cached. By default, they are cached globally * with the `task` as a key. This option allows controlling the cache key explicitly. @@ -5411,30 +5414,13 @@ export interface PageAgent { * **Usage** * * ```js - * await page.agent.perform('Click submit button'); + * await agent.perform('Click submit button'); * ``` * * @param task Task to perform using agentic loop. * @param options */ perform(task: string, options?: { - /** - * API to use, `openapi`, `google` or `anthropic`. Required in non-cache mode. - */ - api?: string; - - /** - * Endpoint to use if different from default. - */ - apiEndpoint?: string; - - /** - * API key for the LLM provider. - * - * API version if relevant. - */ - apiKey?: string; - /** * All the agentic actions are converted to the Playwright calls and are cached. By default, they are cached globally * with the `task` as a key. This option allows controlling the cache key explicitly. @@ -5460,6 +5446,8 @@ export interface PageAgent { outputTokens: number; }; }>; + + [Symbol.asyncDispose](): Promise; } /** @@ -22304,57 +22292,6 @@ export interface BrowserContextOptions { */ acceptDownloads?: boolean; - /** - * Agent settings for [page.agent](https://playwright.dev/docs/api/class-page#page-agent). - */ - agent?: { - /** - * API to use, `openapi`, `google` or `anthropic`. Required in non-cache mode. - */ - api?: string; - - /** - * Endpoint to use if different from default. - */ - apiEndpoint?: string; - - /** - * API key for the LLM provider. - */ - apiKey?: string; - - /** - * Model identifier within the provider. Required in non-cache mode. - */ - model?: string; - - /** - * Cache file to use/generate code for performed actions into. Cache is not used if not specified (default). - */ - cacheFile?: string; - - /** - * When specified, generated entries are written into the `cacheOutFile` instead of updating the `cacheFile`. - */ - cacheOutFile?: string; - - /** - * Secrets to hide from the LLM. - */ - secrets?: { [key: string]: string; }; - - /** - * Maximum number of agentic turns to take per call. Defaults to 10. - */ - maxTurns?: number; - - /** - * Maximum number of tokens to consume per call. The agentic loop will stop after input + output tokens exceed this - * value. Defaults on unlimited. - */ - maxTokens?: number; - }; - /** * When using [page.goto(url[, options])](https://playwright.dev/docs/api/class-page#page-goto), * [page.route(url, handler[, options])](https://playwright.dev/docs/api/class-page#page-route), diff --git a/packages/playwright-core/src/client/browserContext.ts b/packages/playwright-core/src/client/browserContext.ts index b940f373c7ad7..fedd4c2b7ed9f 100644 --- a/packages/playwright-core/src/client/browserContext.ts +++ b/packages/playwright-core/src/client/browserContext.ts @@ -600,7 +600,6 @@ export async function prepareBrowserContextParams(platform: Platform, options: B network.validateHeaders(options.extraHTTPHeaders); const contextParams: channels.BrowserNewContextParams = { ...options, - agent: toAgentProtocol(options.agent), viewport: options.viewport === null ? undefined : options.viewport, noDefaultViewport: options.viewport === null, extraHTTPHeaders: options.extraHTTPHeaders ? headersObjectToArray(options.extraHTTPHeaders) : undefined, @@ -624,13 +623,6 @@ export async function prepareBrowserContextParams(platform: Platform, options: B return contextParams; } -function toAgentProtocol(agent?: BrowserContextOptions['agent']): channels.BrowserNewContextParams['agent'] { - if (!agent) - return undefined; - const secrets = agent.secrets ? Object.entries(agent.secrets).map(([name, value]) => ({ name, value })) : undefined; - return { ...agent, secrets }; -} - function toAcceptDownloadsProtocol(acceptDownloads?: boolean) { if (acceptDownloads === undefined) return undefined; diff --git a/packages/playwright-core/src/client/connection.ts b/packages/playwright-core/src/client/connection.ts index 589e063244d84..fcd56ee6e6032 100644 --- a/packages/playwright-core/src/client/connection.ts +++ b/packages/playwright-core/src/client/connection.ts @@ -41,6 +41,7 @@ import { Worker } from './worker'; import { WritableStream } from './writableStream'; import { ValidationError, findValidator } from '../protocol/validator'; import { rewriteErrorMessage } from '../utils/isomorphic/stackTrace'; +import { PageAgent } from './pageAgent'; import type { ClientInstrumentation } from './clientInstrumentation'; import type { HeadersArray } from './types'; @@ -295,6 +296,9 @@ export class Connection extends EventEmitter { case 'Page': result = new Page(parent, type, guid, initializer); break; + case 'PageAgent': + result = new PageAgent(parent, type, guid, initializer); + break; case 'Playwright': result = new Playwright(parent, type, guid, initializer); break; diff --git a/packages/playwright-core/src/client/events.ts b/packages/playwright-core/src/client/events.ts index 36a3c750933ad..e3e4df611ccfe 100644 --- a/packages/playwright-core/src/client/events.ts +++ b/packages/playwright-core/src/client/events.ts @@ -79,6 +79,11 @@ export const Events = { Worker: 'worker', }, + + PageAgent: { + Turn: 'turn', + }, + WebSocket: { Close: 'close', Error: 'socketerror', diff --git a/packages/playwright-core/src/client/page.ts b/packages/playwright-core/src/client/page.ts index bfab6af21de89..ad4cf8001175c 100644 --- a/packages/playwright-core/src/client/page.ts +++ b/packages/playwright-core/src/client/page.ts @@ -89,7 +89,6 @@ export class Page extends ChannelOwner implements api.Page _routes: RouteHandler[] = []; _webSocketRoutes: WebSocketRouteHandler[] = []; - readonly agent: PageAgent; readonly coverage: Coverage; readonly keyboard: Keyboard; readonly mouse: Mouse; @@ -122,7 +121,6 @@ export class Page extends ChannelOwner implements api.Page this._browserContext = parent as unknown as BrowserContext; this._timeoutSettings = new TimeoutSettings(this._platform, this._browserContext._timeoutSettings); - this.agent = new PageAgent(this); this.keyboard = new Keyboard(this); this.mouse = new Mouse(this); this.request = this._browserContext.request; @@ -136,7 +134,6 @@ export class Page extends ChannelOwner implements api.Page this._closed = initializer.isClosed; this._opener = Page.fromNullable(initializer.opener); - this._channel.on('agentTurn', params => this.emit(Events.Page.AgentTurn, params)); this._channel.on('bindingCall', ({ binding }) => this._onBinding(BindingCall.from(binding))); this._channel.on('close', () => this._onClose()); this._channel.on('crash', () => this._onCrash()); @@ -849,6 +846,22 @@ export class Page extends ChannelOwner implements api.Page return result.pdf; } + async agent(options: Parameters[0] = {}) { + const params: channels.PageAgentParams = { + api: options.provider?.api, + apiEndpoint: options.provider?.apiEndpoint, + apiKey: options.provider?.apiKey, + model: options.provider?.model, + cacheFile: options.cache?.cacheFile, + cacheOutFile: options.cache?.cacheOutFile, + maxTokens: options.maxTokens, + maxTurns: options.maxTurns, + secrets: options.secrets ? Object.entries(options.secrets).map(([name, value]) => ({ name, value })) : undefined, + }; + const { agent } = await this._channel.agent(params); + return PageAgent.from(agent); + } + async _snapshotForAI(options: TimeoutOptions & { track?: string } = {}): Promise<{ full: string, incremental?: string }> { return await this._channel.snapshotForAI({ timeout: this._timeoutSettings.timeout(options), track: options.track }); } diff --git a/packages/playwright-core/src/client/pageAgent.ts b/packages/playwright-core/src/client/pageAgent.ts index aea06c91bc435..80a2338e9bf4e 100644 --- a/packages/playwright-core/src/client/pageAgent.ts +++ b/packages/playwright-core/src/client/pageAgent.ts @@ -15,38 +15,52 @@ * limitations under the License. */ +import { ChannelOwner } from './channelOwner'; +import { Events } from './events'; +import { Page } from './page'; + import type * as api from '../../types/types'; -import type { Page } from './page'; import type z from 'zod'; +import type * as channels from '@protocol/channels'; type PageAgentOptions = { - api?: string; - apiEndpoint?: string; - apiKey?: string; - model?: string; maxTokens?: number; maxTurns?: number; cacheKey?: string; }; -export class PageAgent implements api.PageAgent { +export class PageAgent extends ChannelOwner implements api.PageAgent { private _page: Page; - constructor(page: Page) { - this._page = page; + static from(channel: channels.PageAgentChannel): PageAgent { + return (channel as any)._object; + } + + constructor(parent: ChannelOwner, type: string, guid: string, initializer: channels.PageAgentInitializer) { + super(parent, type, guid, initializer); + this._page = Page.from(initializer.page); + this._channel.on('turn', params => this.emit(Events.Page.AgentTurn, params)); } async expect(expectation: string, options: PageAgentOptions = {}) { - await this._page._channel.agentExpect({ expectation, ...options }); + await this._channel.expect({ expectation, ...options }); } async perform(task: string, options: PageAgentOptions = {}) { - const { usage } = await this._page._channel.agentPerform({ task, ...options }); + const { usage } = await this._channel.perform({ task, ...options }); return { usage }; } async extract(query: string, schema: Schema, options: PageAgentOptions = {}): Promise> { - const { result, usage } = await this._page._channel.agentExtract({ query, schema: this._page._platform.zodToJsonSchema(schema), ...options }); + const { result, usage } = await this._channel.extract({ query, schema: this._page._platform.zodToJsonSchema(schema), ...options }); return { result, usage }; } + + async dispose() { + await this._channel.dispose(); + } + + async [Symbol.asyncDispose]() { + await this.dispose(); + } } diff --git a/packages/playwright-core/src/client/types.ts b/packages/playwright-core/src/client/types.ts index 265a1e8e5950d..b9e2423613350 100644 --- a/packages/playwright-core/src/client/types.ts +++ b/packages/playwright-core/src/client/types.ts @@ -58,10 +58,7 @@ export type ClientCertificate = { passphrase?: string; }; -export type AgentOptions = Omit, 'secrets'> & { secrets?: Record }; - export type BrowserContextOptions = Omit & { - agent?: AgentOptions; viewport?: Size | null; extraHTTPHeaders?: Headers; logger?: Logger; diff --git a/packages/playwright-core/src/protocol/validator.ts b/packages/playwright-core/src/protocol/validator.ts index 95764b9e463dc..5481128be88dd 100644 --- a/packages/playwright-core/src/protocol/validator.ts +++ b/packages/playwright-core/src/protocol/validator.ts @@ -602,17 +602,6 @@ scheme.BrowserTypeLaunchPersistentContextParams = tObject({ serviceWorkers: tOptional(tEnum(['allow', 'block'])), selectorEngines: tOptional(tArray(tType('SelectorEngine'))), testIdAttributeName: tOptional(tString), - agent: tOptional(tObject({ - api: tOptional(tString), - apiKey: tOptional(tString), - apiEndpoint: tOptional(tString), - model: tOptional(tString), - cacheFile: tOptional(tString), - cacheOutFile: tOptional(tString), - secrets: tOptional(tArray(tType('NameValue'))), - maxTurns: tOptional(tInt), - maxTokens: tOptional(tInt), - })), userDataDir: tString, slowMo: tOptional(tFloat), }); @@ -706,17 +695,6 @@ scheme.BrowserNewContextParams = tObject({ serviceWorkers: tOptional(tEnum(['allow', 'block'])), selectorEngines: tOptional(tArray(tType('SelectorEngine'))), testIdAttributeName: tOptional(tString), - agent: tOptional(tObject({ - api: tOptional(tString), - apiKey: tOptional(tString), - apiEndpoint: tOptional(tString), - model: tOptional(tString), - cacheFile: tOptional(tString), - cacheOutFile: tOptional(tString), - secrets: tOptional(tArray(tType('NameValue'))), - maxTurns: tOptional(tInt), - maxTokens: tOptional(tInt), - })), proxy: tOptional(tObject({ server: tString, bypass: tOptional(tString), @@ -788,17 +766,6 @@ scheme.BrowserNewContextForReuseParams = tObject({ serviceWorkers: tOptional(tEnum(['allow', 'block'])), selectorEngines: tOptional(tArray(tType('SelectorEngine'))), testIdAttributeName: tOptional(tString), - agent: tOptional(tObject({ - api: tOptional(tString), - apiKey: tOptional(tString), - apiEndpoint: tOptional(tString), - model: tOptional(tString), - cacheFile: tOptional(tString), - cacheOutFile: tOptional(tString), - secrets: tOptional(tArray(tType('NameValue'))), - maxTurns: tOptional(tInt), - maxTokens: tOptional(tInt), - })), proxy: tOptional(tObject({ server: tString, bypass: tOptional(tString), @@ -847,6 +814,7 @@ scheme.WorkerWaitForEventInfoParams = tType('EventTargetWaitForEventInfoParams') scheme.WebSocketWaitForEventInfoParams = tType('EventTargetWaitForEventInfoParams'); scheme.ElectronApplicationWaitForEventInfoParams = tType('EventTargetWaitForEventInfoParams'); scheme.AndroidDeviceWaitForEventInfoParams = tType('EventTargetWaitForEventInfoParams'); +scheme.PageAgentWaitForEventInfoParams = tType('EventTargetWaitForEventInfoParams'); scheme.EventTargetWaitForEventInfoResult = tOptional(tObject({})); scheme.BrowserContextWaitForEventInfoResult = tType('EventTargetWaitForEventInfoResult'); scheme.PageWaitForEventInfoResult = tType('EventTargetWaitForEventInfoResult'); @@ -854,6 +822,7 @@ scheme.WorkerWaitForEventInfoResult = tType('EventTargetWaitForEventInfoResult') scheme.WebSocketWaitForEventInfoResult = tType('EventTargetWaitForEventInfoResult'); scheme.ElectronApplicationWaitForEventInfoResult = tType('EventTargetWaitForEventInfoResult'); scheme.AndroidDeviceWaitForEventInfoResult = tType('EventTargetWaitForEventInfoResult'); +scheme.PageAgentWaitForEventInfoResult = tType('EventTargetWaitForEventInfoResult'); scheme.BrowserContextInitializer = tObject({ isChromium: tBoolean, requestContext: tChannel(['APIRequestContext']), @@ -915,17 +884,6 @@ scheme.BrowserContextInitializer = tObject({ serviceWorkers: tOptional(tEnum(['allow', 'block'])), selectorEngines: tOptional(tArray(tType('SelectorEngine'))), testIdAttributeName: tOptional(tString), - agent: tOptional(tObject({ - api: tOptional(tString), - apiKey: tOptional(tString), - apiEndpoint: tOptional(tString), - model: tOptional(tString), - cacheFile: tOptional(tString), - cacheOutFile: tOptional(tString), - secrets: tOptional(tArray(tType('NameValue'))), - maxTurns: tOptional(tInt), - maxTokens: tOptional(tInt), - })), }), }); scheme.BrowserContextBindingCallEvent = tObject({ @@ -1191,14 +1149,6 @@ scheme.PageInitializer = tObject({ isClosed: tBoolean, opener: tOptional(tChannel(['Page'])), }); -scheme.PageAgentTurnEvent = tObject({ - role: tString, - message: tString, - usage: tOptional(tObject({ - inputTokens: tInt, - outputTokens: tInt, - })), -}); scheme.PageBindingCallEvent = tObject({ binding: tChannel(['BindingCall']), }); @@ -1537,43 +1487,19 @@ scheme.PageUpdateSubscriptionParams = tObject({ enabled: tBoolean, }); scheme.PageUpdateSubscriptionResult = tOptional(tObject({})); -scheme.PageAgentPerformParams = tObject({ - task: tString, +scheme.PageAgentParams = tObject({ api: tOptional(tString), - apiEndpoint: tOptional(tString), apiKey: tOptional(tString), - maxTurns: tOptional(tInt), - maxTokens: tOptional(tInt), - cacheKey: tOptional(tString), -}); -scheme.PageAgentPerformResult = tObject({ - usage: tType('AgentUsage'), -}); -scheme.PageAgentExpectParams = tObject({ - expectation: tString, - api: tOptional(tString), apiEndpoint: tOptional(tString), - apiKey: tOptional(tString), + model: tOptional(tString), + cacheFile: tOptional(tString), + cacheOutFile: tOptional(tString), + secrets: tOptional(tArray(tType('NameValue'))), maxTurns: tOptional(tInt), maxTokens: tOptional(tInt), - cacheKey: tOptional(tString), }); -scheme.PageAgentExpectResult = tObject({ - usage: tType('AgentUsage'), -}); -scheme.PageAgentExtractParams = tObject({ - query: tString, - schema: tAny, - api: tOptional(tString), - apiEndpoint: tOptional(tString), - apiKey: tOptional(tString), - maxTurns: tOptional(tInt), - maxTokens: tOptional(tInt), - cacheKey: tOptional(tString), -}); -scheme.PageAgentExtractResult = tObject({ - result: tAny, - usage: tType('AgentUsage'), +scheme.PageAgentResult = tObject({ + agent: tChannel(['PageAgent']), }); scheme.FrameInitializer = tObject({ url: tString, @@ -2863,17 +2789,6 @@ scheme.AndroidDeviceLaunchBrowserParams = tObject({ serviceWorkers: tOptional(tEnum(['allow', 'block'])), selectorEngines: tOptional(tArray(tType('SelectorEngine'))), testIdAttributeName: tOptional(tString), - agent: tOptional(tObject({ - api: tOptional(tString), - apiKey: tOptional(tString), - apiEndpoint: tOptional(tString), - model: tOptional(tString), - cacheFile: tOptional(tString), - cacheOutFile: tOptional(tString), - secrets: tOptional(tArray(tType('NameValue'))), - maxTurns: tOptional(tInt), - maxTokens: tOptional(tInt), - })), pkg: tOptional(tString), args: tOptional(tArray(tString)), proxy: tOptional(tObject({ @@ -2977,6 +2892,48 @@ scheme.JsonPipeSendParams = tObject({ scheme.JsonPipeSendResult = tOptional(tObject({})); scheme.JsonPipeCloseParams = tOptional(tObject({})); scheme.JsonPipeCloseResult = tOptional(tObject({})); +scheme.PageAgentInitializer = tObject({ + page: tChannel(['Page']), +}); +scheme.PageAgentTurnEvent = tObject({ + role: tString, + message: tString, + usage: tOptional(tObject({ + inputTokens: tInt, + outputTokens: tInt, + })), +}); +scheme.PageAgentPerformParams = tObject({ + task: tString, + maxTurns: tOptional(tInt), + maxTokens: tOptional(tInt), + cacheKey: tOptional(tString), +}); +scheme.PageAgentPerformResult = tObject({ + usage: tType('AgentUsage'), +}); +scheme.PageAgentExpectParams = tObject({ + expectation: tString, + maxTurns: tOptional(tInt), + maxTokens: tOptional(tInt), + cacheKey: tOptional(tString), +}); +scheme.PageAgentExpectResult = tObject({ + usage: tType('AgentUsage'), +}); +scheme.PageAgentExtractParams = tObject({ + query: tString, + schema: tAny, + maxTurns: tOptional(tInt), + maxTokens: tOptional(tInt), + cacheKey: tOptional(tString), +}); +scheme.PageAgentExtractResult = tObject({ + result: tAny, + usage: tType('AgentUsage'), +}); +scheme.PageAgentDisposeParams = tOptional(tObject({})); +scheme.PageAgentDisposeResult = tOptional(tObject({})); scheme.AgentUsage = tObject({ turns: tInt, inputTokens: tInt, diff --git a/packages/playwright-core/src/server/agent/context.ts b/packages/playwright-core/src/server/agent/context.ts index d515121f28f5f..8bb10692d92d6 100644 --- a/packages/playwright-core/src/server/agent/context.ts +++ b/packages/playwright-core/src/server/agent/context.ts @@ -23,24 +23,22 @@ import type * as loopTypes from '@lowire/loop'; import type * as actions from './actions'; import type { Page } from '../page'; import type { Progress } from '../progress'; -import type { BrowserContextOptions } from '../types'; import type { Language } from '../../utils/isomorphic/locatorGenerators.ts'; import type { ToolDefinition } from './tool'; - -type AgentOptions = BrowserContextOptions['agent']; +import type * as channels from '@protocol/channels'; export class Context { - readonly options: AgentOptions; readonly page: Page; readonly actions: actions.ActionWithCode[] = []; readonly sdkLanguage: Language; readonly progress: Progress; + readonly options: channels.PageAgentParams; private _callIntent: string | undefined; - constructor(apiCallProgress: Progress, page: Page) { + constructor(apiCallProgress: Progress, page: Page, options: channels.PageAgentParams) { this.progress = apiCallProgress; this.page = page; - this.options = page.browserContext._options.agent; + this.options = options; this.sdkLanguage = page.browserContext._browser.sdkLanguage(); } @@ -143,13 +141,6 @@ export class Context { })); } - limits(options: { maxTurns?: number, maxTokens?: number } = {}): { maxTurns: number | undefined, maxTokens: number | undefined } { - return { - maxTurns: options.maxTurns ?? this.options?.maxTurns ?? 10, - maxTokens: options.maxTokens ?? this.options?.maxTokens ?? undefined, - }; - } - private _redactText(text: string): string { const secrets = this.options?.secrets; if (!secrets) diff --git a/packages/playwright-core/src/server/agent/pageAgent.ts b/packages/playwright-core/src/server/agent/pageAgent.ts index fbad40bf73095..878dca1fea8fa 100644 --- a/packages/playwright-core/src/server/agent/pageAgent.ts +++ b/packages/playwright-core/src/server/agent/pageAgent.ts @@ -21,38 +21,13 @@ import { debug } from '../../utilsBundle'; import { Loop } from '../../mcpBundle'; import { runAction } from './actionRunner'; import { Context } from './context'; -import { Page } from '../page'; import performTools from './performTools'; import expectTools from './expectTools'; -import type { Progress } from '../progress'; import type * as channels from '@protocol/channels'; -import type * as loopTypes from '@lowire/loop'; import type * as actions from './actions'; import type { ToolDefinition } from './tool'; - -type Usage = { - turns: number, - inputTokens: number, - outputTokens: number, -}; - -const emptyUsage: Usage = { turns: 0, inputTokens: 0, outputTokens: 0 }; - -export async function pageAgentPerformWithEvents(progress: Progress, page: Page, options: channels.PageAgentPerformParams): Promise<{ usage: Usage, actions: actions.ActionWithCode[] }> { - const context = new Context(progress, page); - const usageContainer = { value: emptyUsage }; - const eventSupport = eventSupportHooks(page, usageContainer); - - await pageAgentPerform(context, { - ...eventSupport, - ...options, - }); - return { - usage: usageContainer.value, - actions: context.actions, - }; -} +import type * as loopTypes from '@lowire/loop'; export async function pageAgentPerform(context: Context, options: loopTypes.LoopEvents & channels.PageAgentPerformParams) { const cacheKey = (options.cacheKey ?? options.task).trim(); @@ -70,33 +45,12 @@ ${options.task} await runLoop(context, performTools, task, undefined, options); await updateCache(context, cacheKey); - return { actions: context.actions }; -} - -export async function pageAgentExpectWithEvents(progress: Progress, page: Page, options: channels.PageAgentExpectParams): Promise<{ usage: Usage, actions: actions.ActionWithCode[] }> { - const context = new Context(progress, page); - const usageContainer = { value: emptyUsage }; - const eventSupport = eventSupportHooks(page, usageContainer); - - await pageAgentExpect(context, { - ...eventSupport, - ...options, - }); - return { - usage: usageContainer.value, - actions: context.actions, - }; } export async function pageAgentExpect(context: Context, options: loopTypes.LoopEvents & channels.PageAgentExpectParams) { const cacheKey = (options.cacheKey ?? options.expectation).trim(); - const cachedActions = await cachedPerform(context, cacheKey); - if (cachedActions) { - return { - usage: emptyUsage, - actions: cachedActions, - }; - } + if (await cachedPerform(context, cacheKey)) + return; const task = ` ### Instructions @@ -111,25 +65,7 @@ ${options.expectation} await updateCache(context, cacheKey); } -export async function pageAgentExtractWithEvents(progress: Progress, page: Page, options: channels.PageAgentExtractParams): Promise<{ - result: any - usage: Usage, -}> { - const context = new Context(progress, page); - const usageContainer = { value: emptyUsage }; - const eventSupport = eventSupportHooks(page, usageContainer); - - const task = ` -### Instructions -Extract the following information from the page. Do not perform any actions, just extract the information. - -### Query -${options.query}`; - const { result } = await runLoop(context, [], task, options.schema, { ...eventSupport, ...options }); - return { result, usage: usageContainer.value }; -} - -async function runLoop(context: Context, toolDefinitions: ToolDefinition[], userTask: string, resultSchema: loopTypes.Schema | undefined, options: loopTypes.LoopEvents & { +export async function runLoop(context: Context, toolDefinitions: ToolDefinition[], userTask: string, resultSchema: loopTypes.Schema | undefined, options: loopTypes.LoopEvents & { api?: string, apiEndpoint?: string, apiKey?: string, @@ -140,31 +76,24 @@ async function runLoop(context: Context, toolDefinitions: ToolDefinition[], user result: any }> { const { page } = context; - const browserContext = page.browserContext; - - const api = options.api ?? browserContext._options.agent?.api; - const apiEndpoint = options.apiEndpoint ?? browserContext._options.agent?.apiEndpoint; - const apiKey = options.apiKey ?? browserContext._options.agent?.apiKey; - const model = options.model ?? browserContext._options.agent?.model; - if (!api || !apiKey || !model) + if (!context.options?.api || !context.options?.apiKey || !context.options?.model) throw new Error(`This action requires the API and API key to be set on the browser context`); const { full } = await page.snapshotForAI(context.progress); const { tools, callTool } = toolsForLoop(context, toolDefinitions, { resultSchema }); - const limits = context.limits(options); const loop = new Loop({ - api: api as any, - apiEndpoint, - apiKey, - model, + api: context.options.api as any, + apiEndpoint: context.options.apiEndpoint, + apiKey: context.options.apiKey, + model: context.options.model, + maxTurns: context.options.maxTurns, + maxTokens: context.options.maxTokens, summarize: true, debug, callTool, tools, - ...limits, - ...options }); const task = `${userTask} @@ -236,39 +165,3 @@ async function cachedActions(cacheFile: string): Promise { } return cache; } - -export function eventSupportHooks(page: Page, usageContainer: { value: Usage }): loopTypes.LoopEvents { - return { - onBeforeTurn(params: { conversation: loopTypes.Conversation }) { - const userMessage = params.conversation.messages.find(m => m.role === 'user'); - page.emit(Page.Events.AgentTurn, { role: 'user', message: userMessage?.content ?? '' }); - return 'continue' as const; - }, - - onAfterTurn(params: { assistantMessage: loopTypes.AssistantMessage, totalUsage: loopTypes.Usage }) { - const usage = { inputTokens: params.totalUsage.input, outputTokens: params.totalUsage.output }; - const intent = params.assistantMessage.content.filter(c => c.type === 'text').map(c => c.text).join('\n'); - page.emit(Page.Events.AgentTurn, { role: 'assistant', message: intent, usage }); - if (!params.assistantMessage.content.filter(c => c.type === 'tool_call').length) - page.emit(Page.Events.AgentTurn, { role: 'assistant', message: `no tool calls`, usage }); - usageContainer.value = { turns: usageContainer.value.turns + 1, inputTokens: usageContainer.value.inputTokens + usage.inputTokens, outputTokens: usageContainer.value.outputTokens + usage.outputTokens }; - return 'continue' as const; - }, - - onBeforeToolCall(params: { toolCall: loopTypes.ToolCallContentPart }) { - page.emit(Page.Events.AgentTurn, { role: 'assistant', message: `call tool "${params.toolCall.name}"` }); - return 'continue' as const; - }, - - onAfterToolCall(params: { toolCall: loopTypes.ToolCallContentPart, result: loopTypes.ToolResult }) { - const suffix = params.toolCall.result?.isError ? 'failed' : 'succeeded'; - page.emit(Page.Events.AgentTurn, { role: 'user', message: `tool "${params.toolCall.name}" ${suffix}` }); - return 'continue' as const; - }, - - onToolCallError(params: { toolCall: loopTypes.ToolCallContentPart, error: Error }) { - page.emit(Page.Events.AgentTurn, { role: 'user', message: `tool "${params.toolCall.name}" failed: ${params.error.message}` }); - return 'continue' as const; - } - }; -} diff --git a/packages/playwright-core/src/server/dispatchers/browserDispatcher.ts b/packages/playwright-core/src/server/dispatchers/browserDispatcher.ts index 684dcacca91a7..bda9e1f42650d 100644 --- a/packages/playwright-core/src/server/dispatchers/browserDispatcher.ts +++ b/packages/playwright-core/src/server/dispatchers/browserDispatcher.ts @@ -77,7 +77,6 @@ export class BrowserDispatcher extends Dispatcher { - delete params.agent; const context = await this._object.newContextForReuse(progress, params); const contextDispatcher = BrowserContextDispatcher.from(this, context); this._dispatchEvent('context', { context: contextDispatcher }); diff --git a/packages/playwright-core/src/server/dispatchers/pageAgentDispatcher.ts b/packages/playwright-core/src/server/dispatchers/pageAgentDispatcher.ts new file mode 100644 index 0000000000000..fbf3cfc53d584 --- /dev/null +++ b/packages/playwright-core/src/server/dispatchers/pageAgentDispatcher.ts @@ -0,0 +1,135 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the 'License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Dispatcher } from './dispatcher'; +import { pageAgentExpect, pageAgentPerform, runLoop } from '../agent/pageAgent'; +import { SdkObject } from '../instrumentation'; +import { Context } from '../agent/context'; + +import type { PageDispatcher } from './pageDispatcher'; +import type { DispatcherScope } from './dispatcher'; +import type * as channels from '@protocol/channels'; +import type { Progress } from '@protocol/progress'; +import type { Page } from '../page'; +import type * as loopTypes from '@lowire/loop'; + +export class PageAgentDispatcher extends Dispatcher implements channels.PageAgentChannel { + _type_PageAgent = true; + _type_EventTarget = true; + private _page: Page; + private _agentParams: channels.PageAgentParams; + private _usage: Usage = { turns: 0, inputTokens: 0, outputTokens: 0 }; + + constructor(scope: PageDispatcher, options: channels.PageAgentParams) { + super(scope, new SdkObject(scope._object, 'pageAgent'), 'PageAgent', { page: scope }); + this._page = scope._object; + this._agentParams = options; + } + + async perform(params: channels.PageAgentPerformParams, progress: Progress): Promise { + const resolvedParams = resolveCallOptions(this._agentParams, params); + const context = new Context(progress, this._page, resolvedParams); + + await pageAgentPerform(context, { + ...this._eventSupport(), + ...resolvedParams, + task: params.task, + }); + return { usage: this._usage }; + } + + async expect(params: channels.PageAgentExpectParams, progress: Progress): Promise { + const resolvedParams = resolveCallOptions(this._agentParams, params); + const context = new Context(progress, this._page, resolvedParams); + + await pageAgentExpect(context, { + ...this._eventSupport(), + ...resolvedParams, + expectation: params.expectation, + }); + return { usage: this._usage }; + } + + async extract(params: channels.PageAgentExtractParams, progress: Progress): Promise { + const resolvedParams = resolveCallOptions(this._agentParams, params); + const context = new Context(progress, this._page, resolvedParams); + + const task = ` + ### Instructions + Extract the following information from the page. Do not perform any actions, just extract the information. + + ### Query + ${params.query}`; + const { result } = await runLoop(context, [], task, params.schema, { + ...this._eventSupport(), + ...resolvedParams, + }); + return { result, usage: this._usage }; + } + + async dispose(params: channels.PageAgentDisposeParams, progress: Progress): Promise { + } + + private _eventSupport(): loopTypes.LoopEvents { + const self = this; + return { + onBeforeTurn(params: { conversation: loopTypes.Conversation }) { + const userMessage = params.conversation.messages.find(m => m.role === 'user'); + self._dispatchEvent('turn', { role: 'user', message: userMessage?.content ?? '' }); + return 'continue' as const; + }, + + onAfterTurn(params: { assistantMessage: loopTypes.AssistantMessage, totalUsage: loopTypes.Usage }) { + const usage = { inputTokens: params.totalUsage.input, outputTokens: params.totalUsage.output }; + const intent = params.assistantMessage.content.filter(c => c.type === 'text').map(c => c.text).join('\n'); + self._dispatchEvent('turn', { role: 'assistant', message: intent, usage }); + if (!params.assistantMessage.content.filter(c => c.type === 'tool_call').length) + self._dispatchEvent('turn', { role: 'assistant', message: `no tool calls`, usage }); + self._usage = { turns: self._usage.turns + 1, inputTokens: self._usage.inputTokens + usage.inputTokens, outputTokens: self._usage.outputTokens + usage.outputTokens }; + return 'continue' as const; + }, + + onBeforeToolCall(params: { toolCall: loopTypes.ToolCallContentPart }) { + self._dispatchEvent('turn', { role: 'assistant', message: `call tool "${params.toolCall.name}"` }); + return 'continue' as const; + }, + + onAfterToolCall(params: { toolCall: loopTypes.ToolCallContentPart, result: loopTypes.ToolResult }) { + const suffix = params.toolCall.result?.isError ? 'failed' : 'succeeded'; + self._dispatchEvent('turn', { role: 'user', message: `tool "${params.toolCall.name}" ${suffix}` }); + return 'continue' as const; + }, + + onToolCallError(params: { toolCall: loopTypes.ToolCallContentPart, error: Error }) { + self._dispatchEvent('turn', { role: 'user', message: `tool "${params.toolCall.name}" failed: ${params.error.message}` }); + return 'continue' as const; + } + }; + } +} + +function resolveCallOptions(agentParams: channels.PageAgentParams, callParams: channels.PageAgentPerformParams | channels.PageAgentExpectParams | channels.PageAgentExtractParams): channels.PageAgentParams { + return { + ...agentParams, + ...callParams, + }; +} + +type Usage = { + turns: number, + inputTokens: number, + outputTokens: number, +}; diff --git a/packages/playwright-core/src/server/dispatchers/pageDispatcher.ts b/packages/playwright-core/src/server/dispatchers/pageDispatcher.ts index 6e6a59eac6df3..22f65daf00c5f 100644 --- a/packages/playwright-core/src/server/dispatchers/pageDispatcher.ts +++ b/packages/playwright-core/src/server/dispatchers/pageDispatcher.ts @@ -27,7 +27,7 @@ import { RouteDispatcher, WebSocketDispatcher } from './networkDispatchers'; import { WebSocketRouteDispatcher } from './webSocketRouteDispatcher'; import { SdkObject } from '../instrumentation'; import { urlMatches } from '../../utils/isomorphic/urlMatch'; -import { pageAgentPerformWithEvents, pageAgentExpectWithEvents, pageAgentExtractWithEvents } from '../agent/pageAgent'; +import { PageAgentDispatcher } from './pageAgentDispatcher'; import type { Artifact } from '../artifact'; import type { BrowserContext } from '../browserContext'; @@ -94,7 +94,6 @@ export class PageDispatcher extends Dispatcher this._dispatchEvent('agentTurn', params)); this.addObjectListener(Page.Events.Close, () => { this._dispatchEvent('close'); this._dispose(); @@ -322,18 +321,6 @@ export class PageDispatcher extends Dispatcher { - return await pageAgentPerformWithEvents(progress, this._page, params); - } - - async agentExpect(params: channels.PageAgentExpectParams, progress: Progress): Promise { - return await pageAgentExpectWithEvents(progress, this._page, params); - } - - async agentExtract(params: channels.PageAgentExtractParams, progress: Progress): Promise { - return await pageAgentExtractWithEvents(progress, this._page, params); - } - async requests(params: channels.PageRequestsParams, progress: Progress): Promise { // Send all future requests to the client, so that it can reliably receive all of them. // Otherwise, if subscription is added in a different task from this call (either before or after), @@ -374,6 +361,10 @@ export class PageDispatcher extends Dispatcher { + return { agent: new PageAgentDispatcher(this, params) }; + } + _onFrameAttached(frame: Frame) { this._dispatchEvent('frameAttached', { frame: FrameDispatcher.from(this.parentScope(), frame) }); } diff --git a/packages/playwright-core/src/server/page.ts b/packages/playwright-core/src/server/page.ts index 2adb01e405971..1759727992d74 100644 --- a/packages/playwright-core/src/server/page.ts +++ b/packages/playwright-core/src/server/page.ts @@ -120,7 +120,6 @@ type ExpectScreenshotOptions = ImageComparatorOptions & ScreenshotOptions & { }; const PageEvent = { - AgentTurn: 'agentturn', Close: 'close', Crash: 'crash', Download: 'download', @@ -137,7 +136,6 @@ const PageEvent = { } as const; export type PageEventMap = { - [PageEvent.AgentTurn]: [agentTurn: { role: string, message: string, usage?: { inputTokens: number, outputTokens: number } }]; [PageEvent.Close]: []; [PageEvent.Crash]: []; [PageEvent.Download]: [download: Download]; diff --git a/packages/playwright-core/src/utils/isomorphic/protocolMetainfo.ts b/packages/playwright-core/src/utils/isomorphic/protocolMetainfo.ts index 6c413c338beaf..b86c3fad0e4f7 100644 --- a/packages/playwright-core/src/utils/isomorphic/protocolMetainfo.ts +++ b/packages/playwright-core/src/utils/isomorphic/protocolMetainfo.ts @@ -66,6 +66,7 @@ export const methodMetainfo = new Map; - /** - * Emitted when the agent makes a turn. - */ - on(event: 'agentturn', listener: (data: { - role: string; - - message: string; - - usage?: { - inputTokens: number; - - outputTokens: number; - }; - }) => any): this; - /** * Emitted when the page closes. */ @@ -1240,21 +1225,6 @@ export interface Page { */ on(event: 'worker', listener: (worker: Worker) => any): this; - /** - * Adds an event listener that will be automatically removed after it is triggered once. See `addListener` for more information about this event. - */ - once(event: 'agentturn', listener: (data: { - role: string; - - message: string; - - usage?: { - inputTokens: number; - - outputTokens: number; - }; - }) => any): this; - /** * Adds an event listener that will be automatically removed after it is triggered once. See `addListener` for more information about this event. */ @@ -1350,21 +1320,6 @@ export interface Page { */ once(event: 'worker', listener: (worker: Worker) => any): this; - /** - * Emitted when the agent makes a turn. - */ - addListener(event: 'agentturn', listener: (data: { - role: string; - - message: string; - - usage?: { - inputTokens: number; - - outputTokens: number; - }; - }) => any): this; - /** * Emitted when the page closes. */ @@ -1572,21 +1527,6 @@ export interface Page { */ addListener(event: 'worker', listener: (worker: Worker) => any): this; - /** - * Removes an event listener added by `on` or `addListener`. - */ - removeListener(event: 'agentturn', listener: (data: { - role: string; - - message: string; - - usage?: { - inputTokens: number; - - outputTokens: number; - }; - }) => any): this; - /** * Removes an event listener added by `on` or `addListener`. */ @@ -1682,21 +1622,6 @@ export interface Page { */ removeListener(event: 'worker', listener: (worker: Worker) => any): this; - /** - * Removes an event listener added by `on` or `addListener`. - */ - off(event: 'agentturn', listener: (data: { - role: string; - - message: string; - - usage?: { - inputTokens: number; - - outputTokens: number; - }; - }) => any): this; - /** * Removes an event listener added by `on` or `addListener`. */ @@ -1792,21 +1717,6 @@ export interface Page { */ off(event: 'worker', listener: (worker: Worker) => any): this; - /** - * Emitted when the agent makes a turn. - */ - prependListener(event: 'agentturn', listener: (data: { - role: string; - - message: string; - - usage?: { - inputTokens: number; - - outputTokens: number; - }; - }) => any): this; - /** * Emitted when the page closes. */ @@ -2183,6 +2093,58 @@ export interface Page { url?: string; }): Promise; + /** + * Initialize page agent with the llm provider and cache. + * @param options + */ + agent(options?: { + cache?: { + /** + * Cache file to use/generate code for performed actions into. Cache is not used if not specified (default). + */ + cacheFile?: string; + + /** + * When specified, generated entries are written into the `cacheOutFile` instead of updating the `cacheFile`. + */ + cacheOutFile?: string; + }; + + maxTokens?: number; + + /** + * Maximum number of agentic turns to take per call. Defaults to 10. + */ + maxTurns?: number; + + provider?: { + /** + * API to use. + */ + api: "openai"|"openai-compatible"|"anthropic"|"google"; + + /** + * Endpoint to use if different from default. + */ + apiEndpoint?: string; + + /** + * API key for the LLM provider. + */ + apiKey: string; + + /** + * Model identifier within the provider. Required in non-cache mode. + */ + model: string; + }; + + /** + * Secrets to hide from the LLM. + */ + secrets?: { [key: string]: string; }; + }): Promise; + /** * Brings page to front (activates tab). */ @@ -4823,41 +4785,6 @@ export interface Page { height: number; }; - /** - * Emitted when the agent makes a turn. - */ - waitForEvent(event: 'agentturn', optionsOrPredicate?: { predicate?: (data: { - role: string; - - message: string; - - usage?: { - inputTokens: number; - - outputTokens: number; - }; - }) => boolean | Promise, timeout?: number } | ((data: { - role: string; - - message: string; - - usage?: { - inputTokens: number; - - outputTokens: number; - }; - }) => boolean | Promise)): Promise<{ - role: string; - - message: string; - - usage?: { - inputTokens: number; - - outputTokens: number; - }; - }>; - /** * Emitted when the page closes. */ @@ -5303,8 +5230,6 @@ export interface Page { */ workers(): Array; - agent: PageAgent; - /** * Playwright has ability to mock clock and passage of time. */ @@ -5346,7 +5271,7 @@ export interface PageAgent { * **Usage** * * ```js - * await page.agent.extract('List of items in the cart', z.object({ + * await agent.extract('List of items in the cart', z.object({ * title: z.string().describe('Item title to extract'), * price: z.string().describe('Item price to extract'), * }).array()); @@ -5357,36 +5282,114 @@ export interface PageAgent { * @param options */ extract(query: string, schema: Schema): Promise>; + /** + * Emitted when the agent makes a turn. + */ + on(event: 'turn', listener: (data: { + role: string; + + message: string; + + usage?: { + inputTokens: number; + + outputTokens: number; + }; + }) => any): this; + + /** + * Adds an event listener that will be automatically removed after it is triggered once. See `addListener` for more information about this event. + */ + once(event: 'turn', listener: (data: { + role: string; + + message: string; + + usage?: { + inputTokens: number; + + outputTokens: number; + }; + }) => any): this; + + /** + * Emitted when the agent makes a turn. + */ + addListener(event: 'turn', listener: (data: { + role: string; + + message: string; + + usage?: { + inputTokens: number; + + outputTokens: number; + }; + }) => any): this; + + /** + * Removes an event listener added by `on` or `addListener`. + */ + removeListener(event: 'turn', listener: (data: { + role: string; + + message: string; + + usage?: { + inputTokens: number; + + outputTokens: number; + }; + }) => any): this; + + /** + * Removes an event listener added by `on` or `addListener`. + */ + off(event: 'turn', listener: (data: { + role: string; + + message: string; + + usage?: { + inputTokens: number; + + outputTokens: number; + }; + }) => any): this; + + /** + * Emitted when the agent makes a turn. + */ + prependListener(event: 'turn', listener: (data: { + role: string; + + message: string; + + usage?: { + inputTokens: number; + + outputTokens: number; + }; + }) => any): this; + + /** + * Dispose this agent. + */ + dispose(): Promise; + /** * Expect certain condition to be met. * * **Usage** * * ```js - * await page.agent.expect('"0 items" to be reported'); + * await agent.expect('"0 items" to be reported'); * ``` * * @param expectation Expectation to assert. * @param options */ expect(expectation: string, options?: { - /** - * API to use, `openapi`, `google` or `anthropic`. Required in non-cache mode. - */ - api?: string; - - /** - * Endpoint to use if different from default. - */ - apiEndpoint?: string; - - /** - * API key for the LLM provider. - * - * API version if relevant. - */ - apiKey?: string; - /** * All the agentic actions are converted to the Playwright calls and are cached. By default, they are cached globally * with the `task` as a key. This option allows controlling the cache key explicitly. @@ -5411,30 +5414,13 @@ export interface PageAgent { * **Usage** * * ```js - * await page.agent.perform('Click submit button'); + * await agent.perform('Click submit button'); * ``` * * @param task Task to perform using agentic loop. * @param options */ perform(task: string, options?: { - /** - * API to use, `openapi`, `google` or `anthropic`. Required in non-cache mode. - */ - api?: string; - - /** - * Endpoint to use if different from default. - */ - apiEndpoint?: string; - - /** - * API key for the LLM provider. - * - * API version if relevant. - */ - apiKey?: string; - /** * All the agentic actions are converted to the Playwright calls and are cached. By default, they are cached globally * with the `task` as a key. This option allows controlling the cache key explicitly. @@ -5460,6 +5446,8 @@ export interface PageAgent { outputTokens: number; }; }>; + + [Symbol.asyncDispose](): Promise; } /** @@ -22304,57 +22292,6 @@ export interface BrowserContextOptions { */ acceptDownloads?: boolean; - /** - * Agent settings for [page.agent](https://playwright.dev/docs/api/class-page#page-agent). - */ - agent?: { - /** - * API to use, `openapi`, `google` or `anthropic`. Required in non-cache mode. - */ - api?: string; - - /** - * Endpoint to use if different from default. - */ - apiEndpoint?: string; - - /** - * API key for the LLM provider. - */ - apiKey?: string; - - /** - * Model identifier within the provider. Required in non-cache mode. - */ - model?: string; - - /** - * Cache file to use/generate code for performed actions into. Cache is not used if not specified (default). - */ - cacheFile?: string; - - /** - * When specified, generated entries are written into the `cacheOutFile` instead of updating the `cacheFile`. - */ - cacheOutFile?: string; - - /** - * Secrets to hide from the LLM. - */ - secrets?: { [key: string]: string; }; - - /** - * Maximum number of agentic turns to take per call. Defaults to 10. - */ - maxTurns?: number; - - /** - * Maximum number of tokens to consume per call. The agentic loop will stop after input + output tokens exceed this - * value. Defaults on unlimited. - */ - maxTokens?: number; - }; - /** * When using [page.goto(url[, options])](https://playwright.dev/docs/api/class-page#page-goto), * [page.route(url, handler[, options])](https://playwright.dev/docs/api/class-page#page-route), diff --git a/packages/playwright/src/index.ts b/packages/playwright/src/index.ts index 004d6625ee7e2..0e7ee8375c420 100644 --- a/packages/playwright/src/index.ts +++ b/packages/playwright/src/index.ts @@ -152,7 +152,7 @@ const playwrightFixtures: Fixtures = ({ }, { option: true, box: true }], serviceWorkers: [({ contextOptions }, use) => use(contextOptions.serviceWorkers ?? 'allow'), { option: true, box: true }], contextOptions: [{}, { option: true, box: true }], - agent: [({}, use) => use(undefined), { option: true, box: true }], + agentOptions: [({}, use) => use(undefined), { option: true, box: true }], _combinedContextOptions: [async ({ acceptDownloads, @@ -245,13 +245,13 @@ const playwrightFixtures: Fixtures = ({ playwright._defaultContextNavigationTimeout = undefined; }, { auto: 'all-hooks-included', title: 'context configuration', box: true } as any], - _setupArtifacts: [async ({ playwright, screenshot, _combinedContextOptions, agent }, use, testInfo) => { + _setupArtifacts: [async ({ playwright, screenshot, _combinedContextOptions }, use, testInfo) => { // This fixture has a separate zero-timeout slot to ensure that artifact collection // happens even after some fixtures or hooks time out. // Now that default test timeout is known, we can replace zero with an actual value. testInfo.setTimeout(testInfo.project.timeout); - const artifactsRecorder = new ArtifactsRecorder(playwright, tracing().artifactsDir(), screenshot, agent); + const artifactsRecorder = new ArtifactsRecorder(playwright, tracing().artifactsDir(), screenshot); await artifactsRecorder.willStartTest(testInfo as TestInfoImpl); const tracingGroupSteps: TestStepInternal[] = []; @@ -310,7 +310,6 @@ const playwrightFixtures: Fixtures = ({ if (!(key in options)) options[key as keyof BrowserContextOptions] = value; } - await artifactsRecorder.willCreateBrowserContext(options); }, runBeforeCreateRequestContext: async (options: APIRequestContextOptions) => { for (const [key, value] of Object.entries(_combinedContextOptions)) { @@ -456,6 +455,43 @@ const playwrightFixtures: Fixtures = ({ await use(page); }, + agent: async ({ page, agentOptions }, use, testInfo) => { + const testInfoImpl = testInfo as TestInfoImpl; + const cachePathTemplate = agentOptions?.cachePathTemplate ?? '{testDir}/{testFilePath}-cache.json'; + const resolvedCacheFile = testInfoImpl._applyPathTemplate(cachePathTemplate, '', '.json'); + const cacheFile = testInfoImpl.config.runAgents === 'all' ? undefined : await testInfoImpl._cloneStorage(resolvedCacheFile); + const cacheOutFile = path.join(testInfoImpl.artifactsDir(), 'agent-cache-' + createGuid() + '.json'); + + const provider = agentOptions?.api && testInfo.config.runAgents !== 'none' ? { + api: agentOptions.api as any, + apiEndpoint: agentOptions.apiEndpoint, + apiKey: agentOptions.apiKey, + model: agentOptions.model, + } : undefined; + + const cache = { + cacheFile, + cacheOutFile, + }; + + const agent = await page.agent({ + provider, + cache, + maxTokens: agentOptions?.maxTokens, + maxTurns: agentOptions?.maxTurns, + secrets: agentOptions?.secrets, + }); + + await use(agent); + + if (!resolvedCacheFile || !cacheOutFile) + return; + if (testInfo.status !== 'passed') + return; + + await testInfoImpl._upstreamStorage(resolvedCacheFile, cacheOutFile); + }, + request: async ({ playwright }, use) => { const request = await playwright.request.newContext(); await use(request); @@ -644,14 +680,10 @@ class ArtifactsRecorder { private _screenshotRecorder: SnapshotRecorder; private _pageSnapshot: string | undefined; - private _agent: PlaywrightTestOptions['agent']; - private _agentCacheFile: string | undefined; - private _agentCacheOutFile: string | undefined; - constructor(playwright: PlaywrightImpl, artifactsDir: string, screenshot: ScreenshotOption, agent: PlaywrightTestOptions['agent']) { + constructor(playwright: PlaywrightImpl, artifactsDir: string, screenshot: ScreenshotOption) { this._playwright = playwright; this._artifactsDir = artifactsDir; - this._agent = agent; const screenshotOptions = typeof screenshot === 'string' ? undefined : screenshot; this._startedCollectingArtifacts = Symbol('startedCollectingArtifacts'); @@ -678,15 +710,10 @@ class ArtifactsRecorder { await this._startTraceChunkOnContextCreation(context, context.tracing); } - async willCreateBrowserContext(options: BrowserContextOptions) { - await this._cloneAgentCache(options); - } - async willCloseBrowserContext(context: BrowserContextImpl) { await this._stopTracing(context, context.tracing); await this._screenshotRecorder.captureTemporary(context); await this._takePageSnapshot(context); - await this._upstreamAgentCache(context); } private async _takePageSnapshot(context: BrowserContextImpl) { @@ -708,42 +735,6 @@ class ArtifactsRecorder { } catch {} } - private async _cloneAgentCache(options: BrowserContextOptions) { - if (!this._agent) - return; - - const cachePathTemplate = this._agent.cachePathTemplate ?? '{testDir}/{testFilePath}-cache.json'; - this._agentCacheFile = this._testInfo._applyPathTemplate(cachePathTemplate, '', '.json'); - this._agentCacheOutFile = path.join(this._testInfo.artifactsDir(), 'agent-cache-' + createGuid() + '.json'); - - const cacheFile = this._testInfo.config.runAgents === 'all' ? undefined : await this._testInfo._cloneStorage(this._agentCacheFile); - const apiProps = this._testInfo.config.runAgents !== 'none' ? { - api: this._agent.api, - apiEndpoint: this._agent.apiEndpoint, - apiKey: this._agent.apiKey, - model: this._agent.model, - } : { - api: undefined, - apiEndpoint: undefined, - apiKey: undefined, - model: undefined, - }; - options.agent = { - ...this._agent, - ...apiProps, - cacheFile, - cacheOutFile: this._agentCacheOutFile, - }; - } - - private async _upstreamAgentCache(context: BrowserContextImpl) { - if (!this._agentCacheFile || !this._agentCacheOutFile) - return; - if (this._testInfo.status !== 'passed') - return; - await this._testInfo._upstreamStorage(this._agentCacheFile, this._agentCacheOutFile); - } - async didCreateRequestContext(context: APIRequestContextImpl) { await this._startTraceChunkOnContextCreation(context, context._tracing); } diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts index 5bd9894c4b8b8..108a9119b7484 100644 --- a/packages/playwright/types/test.d.ts +++ b/packages/playwright/types/test.d.ts @@ -15,7 +15,7 @@ * limitations under the License. */ -import type { APIRequestContext, Browser, BrowserContext, BrowserContextOptions, Page, LaunchOptions, ViewportSize, Geolocation, HTTPCredentials, Locator, APIResponse, PageScreenshotOptions } from 'playwright-core'; +import type { APIRequestContext, Browser, BrowserContext, BrowserContextOptions, Page, PageAgent, LaunchOptions, ViewportSize, Geolocation, HTTPCredentials, Locator, APIResponse, PageScreenshotOptions } from 'playwright-core'; export * from 'playwright-core'; export type BlobReporterOptions = { outputDir?: string, fileName?: string }; @@ -1612,7 +1612,7 @@ interface TestConfig { retries?: number; /** - * Whether to run LLM agent for [page.agent](https://playwright.dev/docs/api/class-page#page-agent): + * Whether to run LLM agent for [PageAgent](https://playwright.dev/docs/api/class-pageagent): * - "all" disregards existing cache and performs all actions via LLM * - "missing" only performs actions that don't have generated cache actions * - "none" does not talk to LLM at all, relies on the cached actions (default) @@ -2076,7 +2076,7 @@ export interface FullConfig { rootDir: string; /** - * Whether to run LLM agent for [page.agent](https://playwright.dev/docs/api/class-page#page-agent): + * Whether to run LLM agent for [PageAgent](https://playwright.dev/docs/api/class-pageagent): * - "all" disregards existing cache and performs all actions via LLM * - "missing" only performs actions that don't have generated cache actions * - "none" does not talk to LLM at all, relies on the cached actions (default) @@ -6950,7 +6950,7 @@ export interface PlaywrightWorkerOptions { export type ScreenshotMode = 'off' | 'on' | 'only-on-failure' | 'on-first-failure'; export type TraceMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | 'on-all-retries' | 'retain-on-first-failure'; export type VideoMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry'; -export type Agent = { +export type AgentOptions = { api: string; apiKey: string; apiEndpoint?: string; @@ -7001,7 +7001,7 @@ export type Agent = { * */ export interface PlaywrightTestOptions { - agent: Agent | undefined; + agentOptions: AgentOptions | undefined; /** * Whether to automatically download all the attachments. Defaults to `true` where all the downloads are accepted. * @@ -7711,6 +7711,7 @@ export interface PlaywrightTestArgs { * */ request: APIRequestContext; + agent: PageAgent; } type ExcludeProps = { diff --git a/packages/protocol/src/channels.d.ts b/packages/protocol/src/channels.d.ts index d808ff3dcd4fb..bcf21f876f153 100644 --- a/packages/protocol/src/channels.d.ts +++ b/packages/protocol/src/channels.d.ts @@ -26,6 +26,7 @@ export interface Channel { // ----------- Initializer Traits ----------- export type InitializerTraits = + T extends PageAgentChannel ? PageAgentInitializer : T extends JsonPipeChannel ? JsonPipeInitializer : T extends AndroidDeviceChannel ? AndroidDeviceInitializer : T extends AndroidSocketChannel ? AndroidSocketInitializer : @@ -63,6 +64,7 @@ export type InitializerTraits = // ----------- Event Traits ----------- export type EventsTraits = + T extends PageAgentChannel ? PageAgentEvents : T extends JsonPipeChannel ? JsonPipeEvents : T extends AndroidDeviceChannel ? AndroidDeviceEvents : T extends AndroidSocketChannel ? AndroidSocketEvents : @@ -100,6 +102,7 @@ export type EventsTraits = // ----------- EventTarget Traits ----------- export type EventTargetTraits = + T extends PageAgentChannel ? PageAgentEventTarget : T extends JsonPipeChannel ? JsonPipeEventTarget : T extends AndroidDeviceChannel ? AndroidDeviceEventTarget : T extends AndroidSocketChannel ? AndroidSocketEventTarget : @@ -1008,17 +1011,6 @@ export type BrowserTypeLaunchPersistentContextParams = { serviceWorkers?: 'allow' | 'block', selectorEngines?: SelectorEngine[], testIdAttributeName?: string, - agent?: { - api?: string, - apiKey?: string, - apiEndpoint?: string, - model?: string, - cacheFile?: string, - cacheOutFile?: string, - secrets?: NameValue[], - maxTurns?: number, - maxTokens?: number, - }, userDataDir: string, slowMo?: number, }; @@ -1101,17 +1093,6 @@ export type BrowserTypeLaunchPersistentContextOptions = { serviceWorkers?: 'allow' | 'block', selectorEngines?: SelectorEngine[], testIdAttributeName?: string, - agent?: { - api?: string, - apiKey?: string, - apiEndpoint?: string, - model?: string, - cacheFile?: string, - cacheOutFile?: string, - secrets?: NameValue[], - maxTurns?: number, - maxTokens?: number, - }, slowMo?: number, }; export type BrowserTypeLaunchPersistentContextResult = { @@ -1235,17 +1216,6 @@ export type BrowserNewContextParams = { serviceWorkers?: 'allow' | 'block', selectorEngines?: SelectorEngine[], testIdAttributeName?: string, - agent?: { - api?: string, - apiKey?: string, - apiEndpoint?: string, - model?: string, - cacheFile?: string, - cacheOutFile?: string, - secrets?: NameValue[], - maxTurns?: number, - maxTokens?: number, - }, proxy?: { server: string, bypass?: string, @@ -1314,17 +1284,6 @@ export type BrowserNewContextOptions = { serviceWorkers?: 'allow' | 'block', selectorEngines?: SelectorEngine[], testIdAttributeName?: string, - agent?: { - api?: string, - apiKey?: string, - apiEndpoint?: string, - model?: string, - cacheFile?: string, - cacheOutFile?: string, - secrets?: NameValue[], - maxTurns?: number, - maxTokens?: number, - }, proxy?: { server: string, bypass?: string, @@ -1396,17 +1355,6 @@ export type BrowserNewContextForReuseParams = { serviceWorkers?: 'allow' | 'block', selectorEngines?: SelectorEngine[], testIdAttributeName?: string, - agent?: { - api?: string, - apiKey?: string, - apiEndpoint?: string, - model?: string, - cacheFile?: string, - cacheOutFile?: string, - secrets?: NameValue[], - maxTurns?: number, - maxTokens?: number, - }, proxy?: { server: string, bypass?: string, @@ -1475,17 +1423,6 @@ export type BrowserNewContextForReuseOptions = { serviceWorkers?: 'allow' | 'block', selectorEngines?: SelectorEngine[], testIdAttributeName?: string, - agent?: { - api?: string, - apiKey?: string, - apiEndpoint?: string, - model?: string, - cacheFile?: string, - cacheOutFile?: string, - secrets?: NameValue[], - maxTurns?: number, - maxTokens?: number, - }, proxy?: { server: string, bypass?: string, @@ -1621,17 +1558,6 @@ export type BrowserContextInitializer = { serviceWorkers?: 'allow' | 'block', selectorEngines?: SelectorEngine[], testIdAttributeName?: string, - agent?: { - api?: string, - apiKey?: string, - apiEndpoint?: string, - model?: string, - cacheFile?: string, - cacheOutFile?: string, - secrets?: NameValue[], - maxTurns?: number, - maxTokens?: number, - }, }, }; export interface BrowserContextEventTarget { @@ -2096,7 +2022,6 @@ export type PageInitializer = { opener?: PageChannel, }; export interface PageEventTarget { - on(event: 'agentTurn', callback: (params: PageAgentTurnEvent) => void): this; on(event: 'bindingCall', callback: (params: PageBindingCallEvent) => void): this; on(event: 'close', callback: (params: PageCloseEvent) => void): this; on(event: 'crash', callback: (params: PageCrashEvent) => void): this; @@ -2153,18 +2078,8 @@ export interface PageChannel extends PageEventTarget, EventTargetChannel { stopCSSCoverage(params?: PageStopCSSCoverageParams, progress?: Progress): Promise; bringToFront(params?: PageBringToFrontParams, progress?: Progress): Promise; updateSubscription(params: PageUpdateSubscriptionParams, progress?: Progress): Promise; - agentPerform(params: PageAgentPerformParams, progress?: Progress): Promise; - agentExpect(params: PageAgentExpectParams, progress?: Progress): Promise; - agentExtract(params: PageAgentExtractParams, progress?: Progress): Promise; + agent(params: PageAgentParams, progress?: Progress): Promise; } -export type PageAgentTurnEvent = { - role: string, - message: string, - usage?: { - inputTokens: number, - outputTokens: number, - }, -}; export type PageBindingCallEvent = { binding: BindingCallChannel, }; @@ -2667,71 +2582,33 @@ export type PageUpdateSubscriptionOptions = { }; export type PageUpdateSubscriptionResult = void; -export type PageAgentPerformParams = { - task: string, +export type PageAgentParams = { api?: string, - apiEndpoint?: string, apiKey?: string, - maxTurns?: number, - maxTokens?: number, - cacheKey?: string, -}; -export type PageAgentPerformOptions = { - api?: string, apiEndpoint?: string, - apiKey?: string, + model?: string, + cacheFile?: string, + cacheOutFile?: string, + secrets?: NameValue[], maxTurns?: number, maxTokens?: number, - cacheKey?: string, -}; -export type PageAgentPerformResult = { - usage: AgentUsage, }; -export type PageAgentExpectParams = { - expectation: string, +export type PageAgentOptions = { api?: string, - apiEndpoint?: string, apiKey?: string, - maxTurns?: number, - maxTokens?: number, - cacheKey?: string, -}; -export type PageAgentExpectOptions = { - api?: string, apiEndpoint?: string, - apiKey?: string, + model?: string, + cacheFile?: string, + cacheOutFile?: string, + secrets?: NameValue[], maxTurns?: number, maxTokens?: number, - cacheKey?: string, }; -export type PageAgentExpectResult = { - usage: AgentUsage, -}; -export type PageAgentExtractParams = { - query: string, - schema: any, - api?: string, - apiEndpoint?: string, - apiKey?: string, - maxTurns?: number, - maxTokens?: number, - cacheKey?: string, -}; -export type PageAgentExtractOptions = { - api?: string, - apiEndpoint?: string, - apiKey?: string, - maxTurns?: number, - maxTokens?: number, - cacheKey?: string, -}; -export type PageAgentExtractResult = { - result: any, - usage: AgentUsage, +export type PageAgentResult = { + agent: PageAgentChannel, }; export interface PageEvents { - 'agentTurn': PageAgentTurnEvent; 'bindingCall': PageBindingCallEvent; 'close': PageCloseEvent; 'crash': PageCrashEvent; @@ -4994,17 +4871,6 @@ export type AndroidDeviceLaunchBrowserParams = { serviceWorkers?: 'allow' | 'block', selectorEngines?: SelectorEngine[], testIdAttributeName?: string, - agent?: { - api?: string, - apiKey?: string, - apiEndpoint?: string, - model?: string, - cacheFile?: string, - cacheOutFile?: string, - secrets?: NameValue[], - maxTurns?: number, - maxTokens?: number, - }, pkg?: string, args?: string[], proxy?: { @@ -5071,17 +4937,6 @@ export type AndroidDeviceLaunchBrowserOptions = { serviceWorkers?: 'allow' | 'block', selectorEngines?: SelectorEngine[], testIdAttributeName?: string, - agent?: { - api?: string, - apiKey?: string, - apiEndpoint?: string, - model?: string, - cacheFile?: string, - cacheOutFile?: string, - secrets?: NameValue[], - maxTurns?: number, - maxTokens?: number, - }, pkg?: string, args?: string[], proxy?: { @@ -5231,6 +5086,80 @@ export interface JsonPipeEvents { 'closed': JsonPipeClosedEvent; } +// ----------- PageAgent ----------- +export type PageAgentInitializer = { + page: PageChannel, +}; +export interface PageAgentEventTarget { + on(event: 'turn', callback: (params: PageAgentTurnEvent) => void): this; +} +export interface PageAgentChannel extends PageAgentEventTarget, EventTargetChannel { + _type_PageAgent: boolean; + perform(params: PageAgentPerformParams, progress?: Progress): Promise; + expect(params: PageAgentExpectParams, progress?: Progress): Promise; + extract(params: PageAgentExtractParams, progress?: Progress): Promise; + dispose(params?: PageAgentDisposeParams, progress?: Progress): Promise; +} +export type PageAgentTurnEvent = { + role: string, + message: string, + usage?: { + inputTokens: number, + outputTokens: number, + }, +}; +export type PageAgentPerformParams = { + task: string, + maxTurns?: number, + maxTokens?: number, + cacheKey?: string, +}; +export type PageAgentPerformOptions = { + maxTurns?: number, + maxTokens?: number, + cacheKey?: string, +}; +export type PageAgentPerformResult = { + usage: AgentUsage, +}; +export type PageAgentExpectParams = { + expectation: string, + maxTurns?: number, + maxTokens?: number, + cacheKey?: string, +}; +export type PageAgentExpectOptions = { + maxTurns?: number, + maxTokens?: number, + cacheKey?: string, +}; +export type PageAgentExpectResult = { + usage: AgentUsage, +}; +export type PageAgentExtractParams = { + query: string, + schema: any, + maxTurns?: number, + maxTokens?: number, + cacheKey?: string, +}; +export type PageAgentExtractOptions = { + maxTurns?: number, + maxTokens?: number, + cacheKey?: string, +}; +export type PageAgentExtractResult = { + result: any, + usage: AgentUsage, +}; +export type PageAgentDisposeParams = {}; +export type PageAgentDisposeOptions = {}; +export type PageAgentDisposeResult = void; + +export interface PageAgentEvents { + 'turn': PageAgentTurnEvent; +} + export type AgentUsage = { turns: number, inputTokens: number, diff --git a/packages/protocol/src/protocol.yml b/packages/protocol/src/protocol.yml index 0500e0a2f9f54..8dcf286b59703 100644 --- a/packages/protocol/src/protocol.yml +++ b/packages/protocol/src/protocol.yml @@ -589,20 +589,6 @@ ContextOptions: type: array? items: SelectorEngine testIdAttributeName: string? - agent: - type: object? - properties: - api: string? - apiKey: string? - apiEndpoint: string? - model: string? - cacheFile: string? - cacheOutFile: string? - secrets: - type: array? - items: NameValue - maxTurns: int? - maxTokens: int? LocalUtils: type: interface @@ -2027,47 +2013,25 @@ Page: - requestFailed enabled: boolean - agentPerform: - internal: true - parameters: - task: string - $mixin: PageAgentOptions - - returns: - usage: AgentUsage - - agentExpect: - internal: true - parameters: - expectation: string - $mixin: PageAgentOptions - - returns: - usage: AgentUsage - - agentExtract: + agent: internal: true parameters: - query: string - schema: json - $mixin: PageAgentOptions - + api: string? + apiKey: string? + apiEndpoint: string? + model: string? + cacheFile: string? + cacheOutFile: string? + secrets: + type: array? + items: NameValue + maxTurns: int? + maxTokens: int? returns: - result: json - usage: AgentUsage + agent: PageAgent events: - agentTurn: - parameters: - role: string - message: string - usage: - type: object? - properties: - inputTokens: int - outputTokens: int - bindingCall: parameters: binding: BindingCall @@ -4345,12 +4309,63 @@ JsonPipe: parameters: reason: string? + +PageAgent: + type: interface + extends: EventTarget + + initializer: + page: Page + + commands: + perform: + internal: true + parameters: + task: string + $mixin: PageAgentOptions + + returns: + usage: AgentUsage + + expect: + internal: true + parameters: + expectation: string + $mixin: PageAgentOptions + + returns: + usage: AgentUsage + + extract: + internal: true + parameters: + query: string + schema: json + $mixin: PageAgentOptions + + returns: + result: json + usage: AgentUsage + + dispose: + internal: true + + events: + + turn: + parameters: + role: string + message: string + usage: + type: object? + properties: + inputTokens: int + outputTokens: int + + PageAgentOptions: type: mixin properties: - api: string? - apiEndpoint: string? - apiKey: string? maxTurns: int? maxTokens: int? cacheKey: string? diff --git a/tests/library/perform-task.spec.ts b/tests/library/perform-task.spec.ts index a2d6af8b67c2c..2f8b85da512bb 100644 --- a/tests/library/perform-task.spec.ts +++ b/tests/library/perform-task.spec.ts @@ -19,7 +19,7 @@ import z from 'zod'; import { browserTest as test, expect } from '../config/browserTest'; test.use({ - agent: { + agentOptions: { api: 'anthropic', apiKey: process.env.AZURE_SONNET_API_KEY!, apiEndpoint: process.env.AZURE_SONNET_ENDPOINT!, @@ -30,13 +30,13 @@ test.use({ } }); -test('page.perform', async ({ page, server }) => { +test('page.perform', async ({ page, agent, server }) => { await page.goto(server.PREFIX + '/evals/fill-form.html'); - page.on('agentturn', turn => { + agent.on('turn', turn => { // For debugging purposes it is on for now. console.log('agentturn', turn); }); - await page.agent.perform('Fill out the form with the following details:\n' + + await agent.perform('Fill out the form with the following details:\n' + 'Name: John Smith\n' + 'Address: 1045 La Avenida St, Mountain View, CA 94043\n' + 'Email: john.smith@at-microsoft.com'); @@ -50,18 +50,18 @@ test('page.perform', async ({ page, server }) => { `); }); -test('page.perform secret', async ({ page, server }) => { +test('page.perform secret', async ({ page, agent }) => { await page.setContent(''); - await page.agent.perform('Enter x-secret-email into the email field'); + await agent.perform('Enter x-secret-email into the email field'); await expect(page.locator('body')).toMatchAriaSnapshot(` - textbox "Email Address": secret-email@at-microsoft.com `); }); -test.skip('extract task', async ({ page }) => { +test.skip('extract task', async ({ page, agent }) => { await page.goto('https://demo.playwright.dev/todomvc'); - await page.agent.perform('Add "Buy groceries" todo'); - console.log(await page.agent.extract('List todos with their statuses', z.object({ + await agent.perform('Add "Buy groceries" todo'); + console.log(await agent.extract('List todos with their statuses', z.object({ items: z.object({ title: z.string(), completed: z.boolean() @@ -69,7 +69,7 @@ test.skip('extract task', async ({ page }) => { }))); }); -test('page.perform expect value', async ({ page, server }) => { +test('page.perform expect value', async ({ page, agent }) => { await page.setContent(`