diff --git a/packages/playwright-core/src/server/agent/actionRunner.ts b/packages/playwright-core/src/server/agent/actionRunner.ts index 151460693bed1..9f25d3fec282e 100644 --- a/packages/playwright-core/src/server/agent/actionRunner.ts +++ b/packages/playwright-core/src/server/agent/actionRunner.ts @@ -134,6 +134,11 @@ async function innerRunAction(progress: Progress, mode: 'generate' | 'run', page await runExpect(frame, progress, mode, undefined, { expression: 'to.have.url', expectedText, isNot: !!action.isNot }, expected, 'toHaveURL', 'expected'); break; } + case 'expectTitle': { + const expectedText = serializeExpectedTextValues([action.value], { normalizeWhiteSpace: true }); + await runExpect(frame, progress, mode, undefined, { expression: 'to.have.title', expectedText, isNot: !!action.isNot }, action.value, 'toHaveTitle', 'expected'); + break; + } } } @@ -309,6 +314,17 @@ export function traceParamsForAction(progress: Progress, action: actions.Action, }; return { type: 'Frame', method: 'expect', title: 'Expect URL', params }; } + case 'expectTitle': { + const expectedText = serializeExpectedTextValues([action.value], { normalizeWhiteSpace: true }); + const params: channels.FrameExpectParams = { + selector: undefined, + expression: 'to.have.title', + expectedText, + isNot: !!action.isNot, + timeout: expectTimeout(mode), + }; + return { type: 'Frame', method: 'expect', title: 'Expect Title', params }; + } } } diff --git a/packages/playwright-core/src/server/agent/actions.ts b/packages/playwright-core/src/server/agent/actions.ts index b75355b5d1bab..ae3600f355671 100644 --- a/packages/playwright-core/src/server/agent/actions.ts +++ b/packages/playwright-core/src/server/agent/actions.ts @@ -117,6 +117,13 @@ const expectURLSchema = zod.object({ }); export type ExpectURL = z.infer; +const expectTitleSchema = zod.object({ + method: zod.literal('expectTitle'), + value: zod.string(), + isNot: zod.boolean().optional(), +}); +export type ExpectTitle = z.infer; + const actionSchema = zod.discriminatedUnion('method', [ navigateActionSchema, clickActionSchema, @@ -131,6 +138,7 @@ const actionSchema = zod.discriminatedUnion('method', [ expectValueSchema, expectAriaSchema, expectURLSchema, + expectTitleSchema, ]); export type Action = z.infer; diff --git a/packages/playwright-core/src/server/agent/codegen.ts b/packages/playwright-core/src/server/agent/codegen.ts index 9eb0690d26255..03aa794d3d80a 100644 --- a/packages/playwright-core/src/server/agent/codegen.ts +++ b/packages/playwright-core/src/server/agent/codegen.ts @@ -93,6 +93,10 @@ export async function generateCode(sdkLanguage: Language, action: actions.Action const notInfix = action.isNot ? 'not.' : ''; return `await expect(page).${notInfix}toHaveURL(${arg});`; } + case 'expectTitle': { + const notInfix = action.isNot ? 'not.' : ''; + return `await expect(page).${notInfix}toHaveTitle(${escapeWithQuotes(action.value)});`; + } } // @ts-expect-error throw new Error('Unknown action ' + action.method); diff --git a/packages/playwright-core/src/server/agent/context.ts b/packages/playwright-core/src/server/agent/context.ts index 64be25de9db16..6bb1b618f168a 100644 --- a/packages/playwright-core/src/server/agent/context.ts +++ b/packages/playwright-core/src/server/agent/context.ts @@ -17,6 +17,7 @@ import { BrowserContext } from '../browserContext'; import { runAction } from './actionRunner'; import { generateCode } from './codegen'; +import { stripAnsiEscapes } from '../../utils/isomorphic/stringUtils'; import type { Request } from '../network'; import type * as loopTypes from '@lowire/loop'; @@ -137,7 +138,7 @@ export class Context { const text: string[] = []; if (error) - text.push(`# Error\n${error.message}`); + text.push(`# Error\n${stripAnsiEscapes(error.message)}`); else text.push(`# Success`); @@ -150,7 +151,7 @@ export class Context { }, 'dev.lowire/history': error ? [{ category: 'error', - content: error.message, + content: stripAnsiEscapes(error.message), }] : [], }, isError: !!error, diff --git a/packages/playwright-core/src/server/agent/expectTools.ts b/packages/playwright-core/src/server/agent/expectTools.ts index 6596f3fb2040f..fe06f851184c6 100644 --- a/packages/playwright-core/src/server/agent/expectTools.ts +++ b/packages/playwright-core/src/server/agent/expectTools.ts @@ -134,10 +134,31 @@ const expectURL = defineTool({ }, }); +const expectTitle = defineTool({ + schema: { + name: 'browser_expect_title', + title: 'Expect title', + description: 'Expect the page title to match the expected value.', + inputSchema: z.object({ + title: z.string().describe('Expected page title.'), + isNot: z.boolean().optional().describe('Expect the opposite'), + }), + }, + + handle: async (progress, context, params) => { + return await context.runActionAndWait(progress, { + method: 'expectTitle', + value: params.title, + isNot: params.isNot, + }); + }, +}); + export default [ expectVisible, expectVisibleText, expectValue, expectList, expectURL, + expectTitle, ] as ToolDefinition[]; diff --git a/packages/playwright-core/src/server/agent/tool.ts b/packages/playwright-core/src/server/agent/tool.ts index 3f219dbad1cc5..34e36d8d6924c 100644 --- a/packages/playwright-core/src/server/agent/tool.ts +++ b/packages/playwright-core/src/server/agent/tool.ts @@ -15,6 +15,7 @@ */ import { z } from '../../mcpBundle'; +import { stripAnsiEscapes } from '../../utils/isomorphic/stringUtils'; import type zod from 'zod'; import type * as loopTypes from '@lowire/loop'; import type { Context } from './context'; @@ -110,7 +111,7 @@ export function toolsForLoop(progress: Progress, context: Context, toolDefinitio return await tool.handle(progress, context, params.arguments); } catch (error) { return { - content: [{ type: 'text', text: error.message }], + content: [{ type: 'text', text: stripAnsiEscapes(error.message) }], isError: true, }; } diff --git a/packages/playwright-core/src/utils/isomorphic/stringUtils.ts b/packages/playwright-core/src/utils/isomorphic/stringUtils.ts index 42c06e3bef39a..546c105e73d6b 100644 --- a/packages/playwright-core/src/utils/isomorphic/stringUtils.ts +++ b/packages/playwright-core/src/utils/isomorphic/stringUtils.ts @@ -193,3 +193,8 @@ export function parseRegex(regex: string): RegExp { const flags = regex.slice(lastSlash + 1); return new RegExp(source, flags); } + +export const ansiRegex = new RegExp('([\\u001B\\u009B][[\\]()#;?]*(?:(?:(?:[a-zA-Z\\d]*(?:;[-a-zA-Z\\d\\/#&.:=?%@~_]*)*)?\\u0007)|(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PR-TZcf-ntqry=><~])))', 'g'); +export function stripAnsiEscapes(str: string): string { + return str.replace(ansiRegex, ''); +} diff --git a/packages/playwright/src/util.ts b/packages/playwright/src/util.ts index 6ab72eb1dbf89..bb15feceec1a1 100644 --- a/packages/playwright/src/util.ts +++ b/packages/playwright/src/util.ts @@ -414,7 +414,4 @@ export async function removeDirAndLogToConsole(dir: string) { } } -export const ansiRegex = new RegExp('([\\u001B\\u009B][[\\]()#;?]*(?:(?:(?:[a-zA-Z\\d]*(?:;[-a-zA-Z\\d\\/#&.:=?%@~_]*)*)?\\u0007)|(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PR-TZcf-ntqry=><~])))', 'g'); -export function stripAnsiEscapes(str: string): string { - return str.replace(ansiRegex, ''); -} +export { ansiRegex, stripAnsiEscapes } from 'playwright-core/lib/utils'; diff --git a/tests/components/ct-react17/package.json b/tests/components/ct-react17/package.json index b203e7a1c2c00..5a717e1d05d42 100644 --- a/tests/components/ct-react17/package.json +++ b/tests/components/ct-react17/package.json @@ -18,7 +18,7 @@ "start": "react-scripts start", "build": "react-scripts build", "eject": "react-scripts eject", - "typecheck": "tsc --noEmit" + "typecheck": "echo \"typecheck disabled because of zod v4 being incompatible with ts 4\"" }, "eslintConfig": { "extends": [ diff --git a/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-generate.json b/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-generate.json index e3550a93fa314..dcab20e88ce3a 100644 --- a/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-generate.json +++ b/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-generate.json @@ -1,23 +1,23 @@ { - "428b1de09afa2d1be4a61fdf7eaa2277bc1426ed": { + "94919d5202a4e9298507668bb88822a939783915": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "I need to verify that an input has the value \"hello\", but according to the page snapshot, the textbox with ref=e2 currently has the value \"bye\". This means the expectation is that it should NOT match, or this is testing a failure case.\n\nLet me call the browser_expect_value tool to assert the expected condition:" + "text": "I need to verify that an input has the value \"hello\", but according to the page snapshot, the textbox currently has the value \"bye\". I'll use the browser_expect_value tool to assert the expected value." }, { "type": "tool_call", "name": "browser_expect_value", "arguments": { "type": "textbox", - "element": "input", + "element": "input textbox", "ref": "e2", "value": "hello", "_is_done": true }, - "id": "toolu_01B834GHxjywVnc9UGFxpyGc" + "id": "toolu_017soKmdATMEEVS4yxjbaLs4" } ], "stopReason": { @@ -25,8 +25,8 @@ } }, "usage": { - "input": 2044, - "output": 201 + "input": 2204, + "output": 176 } } } \ No newline at end of file diff --git a/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-run.json b/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-run.json index 398dcc1bc293a..b4f7c09466e72 100644 --- a/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-run.json +++ b/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-run.json @@ -1,11 +1,11 @@ { - "46dd9ff2a304413e15a4579295016e8d2626b5dd": { + "44bfbaa6fe0cb038adf37547ecb000d4f39e243f": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "I need to verify that the submit button is visible on the page. Based on the page snapshot, I can see there is a button with the text \"Submit\" and reference e2." + "text": "I'll verify that the submit button is visible on the page." }, { "type": "tool_call", @@ -15,7 +15,7 @@ "accessibleName": "Submit", "_is_done": true }, - "id": "toolu_011XpmmmcGH5bLRG1hznFRw2" + "id": "toolu_01Luk6EzYXfyshpYHsWqCzfF" } ], "stopReason": { @@ -23,8 +23,8 @@ } }, "usage": { - "input": 2043, - "output": 132 + "input": 2203, + "output": 107 } } } \ No newline at end of file diff --git a/tests/library/__llm_cache__/library-agent-expect-expectTitle-success.json b/tests/library/__llm_cache__/library-agent-expect-expectTitle-success.json new file mode 100644 index 0000000000000..353522d10b339 --- /dev/null +++ b/tests/library/__llm_cache__/library-agent-expect-expectTitle-success.json @@ -0,0 +1,29 @@ +{ + "893eb6be29a43887385345cedf150b6ff229a6e8": { + "result": { + "role": "assistant", + "content": [ + { + "type": "text", + "text": "I'll verify that the page title is \"My Page Title\" using the browser_expect_title tool." + }, + { + "type": "tool_call", + "name": "browser_expect_title", + "arguments": { + "title": "My Page Title", + "_is_done": true + }, + "id": "toolu_01QSwESxdJwhfaSFhvahxFqB" + } + ], + "stopReason": { + "code": "ok" + } + }, + "usage": { + "input": 2195, + "output": 99 + } + } +} \ No newline at end of file diff --git a/tests/library/__llm_cache__/library-agent-expect-expectTitle-wrong-title-error.json b/tests/library/__llm_cache__/library-agent-expect-expectTitle-wrong-title-error.json new file mode 100644 index 0000000000000..2227c3d117ec0 --- /dev/null +++ b/tests/library/__llm_cache__/library-agent-expect-expectTitle-wrong-title-error.json @@ -0,0 +1,29 @@ +{ + "d969f4ed24b89f142366f64ad47d7b163b68e298": { + "result": { + "role": "assistant", + "content": [ + { + "type": "text", + "text": "I need to verify that the page title is \"Other Title\". I'll use the browser_expect_title tool to assert this condition." + }, + { + "type": "tool_call", + "name": "browser_expect_title", + "arguments": { + "title": "Other Title", + "_is_done": true + }, + "id": "toolu_01BnWsHUg4s787xMJYmLUNDn" + } + ], + "stopReason": { + "code": "ok" + } + }, + "usage": { + "input": 2194, + "output": 104 + } + } +} \ No newline at end of file diff --git a/tests/library/__llm_cache__/library-agent-expect-expectURL-success.json b/tests/library/__llm_cache__/library-agent-expect-expectURL-success.json index 582777084c863..453653504d967 100644 --- a/tests/library/__llm_cache__/library-agent-expect-expectURL-success.json +++ b/tests/library/__llm_cache__/library-agent-expect-expectURL-success.json @@ -1,20 +1,20 @@ { - "01e4dfe3e3fd3764d430e1d983db863b06e4ea31": { + "74935765e910e2883dc5925fd5f947bb528f174d": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "I'll verify that the page URL is /page.html." + "text": "I see the issue - the URL contains a server prefix. Let me use a regex pattern to match the URL ending with /counter.html." }, { "type": "tool_call", "name": "browser_expect_url", "arguments": { - "url": "/page.html", + "regex": "/counter\\.html$/", "_is_done": true }, - "id": "toolu_01FUsQbLdCVmZyUCS1DwGRrC" + "id": "toolu_018XaEFPSm6aLzqA1BBqRNhu" } ], "stopReason": { @@ -22,26 +22,26 @@ } }, "usage": { - "input": 2067, - "output": 91 + "input": 2447, + "output": 108 } }, - "d3eb8b6dc11f060023ba45368c64d302b7ff7891": { + "ee1b825529efd3c64fff821e9ce6f2e06433f6d3": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "The assertion failed because the actual URL is the full URL \"http://localhost:8939/page.html\" while I was checking for just \"/page.html\". Let me verify using a regex pattern to match the URL ending with /page.html:" + "text": "I'll verify that the page URL is /counter.html using the browser_expect_url tool." }, { "type": "tool_call", "name": "browser_expect_url", "arguments": { - "regex": "/\\/page\\.html$/", + "url": "/counter.html", "_is_done": true }, - "id": "toolu_01FDFEjR382yKedBJ8LZgwgZ" + "id": "toolu_01PRTpTFD78UKubB5EzwTAii" } ], "stopReason": { @@ -49,8 +49,8 @@ } }, "usage": { - "input": 2285, - "output": 132 + "input": 2243, + "output": 99 } } } \ No newline at end of file diff --git a/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex-error.json b/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex-error.json index b502ab8a398f8..7e0cc5425c484 100644 --- a/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex-error.json +++ b/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex-error.json @@ -1,20 +1,20 @@ { - "3cc51fa90640dadb2fcec777605e29dccb799cf8": { + "37de212b569c4a2d8412505c647ce7ad1f824387": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "I need to verify that the page URL matches the pattern \"/other pattern\". Based on the instruction, I'll use the browser_expect_url tool with a regex parameter." + "text": "I need to verify that the page URL matches the `/counter` pattern. I'll use the `browser_expect_url` tool with a regex pattern." }, { "type": "tool_call", "name": "browser_expect_url", "arguments": { - "regex": "/other pattern/", + "regex": "/counter", "_is_done": true }, - "id": "toolu_01Ta794yU6itVbHgDmgzvFwn" + "id": "toolu_01NFN4Fz1V3Bek9pKTbuEsBg" } ], "stopReason": { @@ -22,26 +22,26 @@ } }, "usage": { - "input": 2066, - "output": 113 + "input": 2242, + "output": 108 } }, - "6ab2134b9621c91d1c4f3ec9752685d78be95bae": { + "a786ca3af437fcb35753c1f7ceb7f9a90bc8deeb": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "The assertion failed because the URL \"http://localhost:8951/other.html\" doesn't match the literal pattern \"/other pattern\". Looking at the actual URL, it contains \"other\" but not \"other pattern\" as a literal string. Let me try a regex that would match URLs containing \"other\":" + "text": "I need to fix the regex pattern format. It should end with a `/` and optional flags." }, { "type": "tool_call", "name": "browser_expect_url", "arguments": { - "regex": "/other/", + "regex": "/counter/", "_is_done": true }, - "id": "toolu_015dWLmUoYJCKhUsLF5xBWfd" + "id": "toolu_01SLaSETGg54hyKNATWckQVw" } ], "stopReason": { @@ -49,8 +49,8 @@ } }, "usage": { - "input": 2307, - "output": 140 + "input": 2440, + "output": 97 } } } \ No newline at end of file diff --git a/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex.json b/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex.json index 5b77e882f81b7..dd48e0d5fdcda 100644 --- a/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex.json +++ b/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex.json @@ -1,20 +1,20 @@ { - "ca4fa24ca5d10d373f005b3e8a5b543a767e10b5": { + "37de212b569c4a2d8412505c647ce7ad1f824387": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "I'll verify that the page URL matches the \"/page\" pattern using a regular expression." + "text": "I need to verify that the page URL matches the /counter pattern using a browser_expect assertion tool." }, { "type": "tool_call", "name": "browser_expect_url", "arguments": { - "regex": "/page", + "regex": "/counter/", "_is_done": true }, - "id": "toolu_01A8AJ2KBrNTj6wSJ6bfSiFS" + "id": "toolu_015RtjMDCtGsTUpu36HUk5fC" } ], "stopReason": { @@ -22,35 +22,8 @@ } }, "usage": { - "input": 2066, - "output": 94 - } - }, - "f6b9debe1e2eb0222d96e3fb7ef1685eac351d81": { - "result": { - "role": "assistant", - "content": [ - { - "type": "text", - "text": "I need to provide a properly formatted regex with closing slash. Let me fix that:" - }, - { - "type": "tool_call", - "name": "browser_expect_url", - "arguments": { - "regex": "/page/", - "_is_done": true - }, - "id": "toolu_019yRn6jCMA4QSeBRD6aZsiy" - } - ], - "stopReason": { - "code": "ok" - } - }, - "usage": { - "input": 2234, - "output": 94 + "input": 2242, + "output": 98 } } } \ No newline at end of file diff --git a/tests/library/__llm_cache__/library-agent-expect-expectURL-wrong-URL-error.json b/tests/library/__llm_cache__/library-agent-expect-expectURL-wrong-URL-error.json index 64448ae5129b8..353b806e0961c 100644 --- a/tests/library/__llm_cache__/library-agent-expect-expectURL-wrong-URL-error.json +++ b/tests/library/__llm_cache__/library-agent-expect-expectURL-wrong-URL-error.json @@ -1,20 +1,20 @@ { - "0e39a03d4c9db9c59b450aa30d50eb88551aca6c": { + "688127ac19026114a89d4e621b86bdeaa699418e": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "I need to adjust my expectation. The actual URL is the full URL including the protocol and host. Let me use a regex pattern to match the URL ending with /other.html." + "text": "I see the issue - the actual URL includes the server prefix. Let me use a regex pattern to match the URL ending with /counter.html." }, { "type": "tool_call", "name": "browser_expect_url", "arguments": { - "regex": "/\\/other\\.html$/", + "regex": "/\\/counter\\.html$/", "_is_done": true }, - "id": "toolu_018zDqYhtHtaNUQ66sv1Ezea" + "id": "toolu_01V6sRugMvahkP21AHaffA11" } ], "stopReason": { @@ -22,26 +22,26 @@ } }, "usage": { - "input": 2293, - "output": 118 + "input": 2447, + "output": 110 } }, - "e50f042f63f65841f33a8d0a608d45f748c66552": { + "ee1b825529efd3c64fff821e9ce6f2e06433f6d3": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "I'll verify that the page URL is /other.html using the browser_expect_url tool." + "text": "I'll verify that the page URL is /counter.html using the browser_expect_url tool." }, { "type": "tool_call", "name": "browser_expect_url", "arguments": { - "url": "/other.html", + "url": "/counter.html", "_is_done": true }, - "id": "toolu_01QffxWJVHBPDgB4WSiKanvu" + "id": "toolu_014UE4ajKxYwk8kM6MsURpks" } ], "stopReason": { @@ -49,7 +49,7 @@ } }, "usage": { - "input": 2067, + "input": 2243, "output": 99 } } diff --git a/tests/library/__llm_cache__/library-agent-limits-should-respect-max-actions-limit.json b/tests/library/__llm_cache__/library-agent-limits-should-respect-max-actions-limit.json index 53c33e56d0402..c4e3d1c4c89e2 100644 --- a/tests/library/__llm_cache__/library-agent-limits-should-respect-max-actions-limit.json +++ b/tests/library/__llm_cache__/library-agent-limits-should-respect-max-actions-limit.json @@ -1,8 +1,12 @@ { - "63c6840169b6554ee4319607e3c5582842511481": { + "1b776f266d622f4553281b575e36acd4862f48b4": { "result": { "role": "assistant", "content": [ + { + "type": "text", + "text": "Now clicking the second time." + }, { "type": "tool_call", "name": "browser_click", @@ -11,7 +15,7 @@ "ref": "e2", "_is_done": false }, - "id": "toolu_01LMKpSXqCaRPAsUfpuLCGq6" + "id": "toolu_014LLfN7tHFxcAqX72VTHXVy" } ], "stopReason": { @@ -19,8 +23,8 @@ } }, "usage": { - "input": 3259, - "output": 91 + "input": 3194, + "output": 98 } }, "7b25e318de64dc9d2494432a6d311631e2c1edc0": { @@ -29,7 +33,7 @@ "content": [ { "type": "text", - "text": "I'll click the submit button 5 times as requested. Let me perform these clicks sequentially." + "text": "I'll click the submit button 5 times as requested. Let me start by clicking it the first time." }, { "type": "tool_call", @@ -39,7 +43,7 @@ "ref": "e2", "_is_done": false }, - "id": "toolu_01VansE6LrZDU3SibgRvkoGY" + "id": "toolu_01PWwDgpp2rUtvmpuTfgTNzn" } ], "stopReason": { @@ -48,13 +52,17 @@ }, "usage": { "input": 3047, - "output": 113 + "output": 115 } }, - "82d4661d903f25ce1a7e51c2954072ed0003042e": { + "b9bb21161597f576eaef8aea85dae797c879fec9": { "result": { "role": "assistant", "content": [ + { + "type": "text", + "text": "Clicking the fourth time." + }, { "type": "tool_call", "name": "browser_click", @@ -63,7 +71,7 @@ "ref": "e2", "_is_done": false }, - "id": "toolu_01VUbrN6D39GVSBuPEaByNKZ" + "id": "toolu_01YRh2G5UeZqvPou8d311SZU" } ], "stopReason": { @@ -71,14 +79,18 @@ } }, "usage": { - "input": 3321, - "output": 91 + "input": 3337, + "output": 98 } }, - "a64b5e20ef2b0c66b2b1373302af78d97e0b451f": { + "bae9fc31434e4f1f36e5cd29bd84165e04ac32f2": { "result": { "role": "assistant", "content": [ + { + "type": "text", + "text": "Clicking the third time." + }, { "type": "tool_call", "name": "browser_click", @@ -87,7 +99,7 @@ "ref": "e2", "_is_done": false }, - "id": "toolu_01FcvrnBtseq9yQA5A7cRdeZ" + "id": "toolu_01JnLrvVrm3EzPwcBrzzDR1S" } ], "stopReason": { @@ -95,8 +107,8 @@ } }, "usage": { - "input": 3192, - "output": 91 + "input": 3267, + "output": 98 } } } \ No newline at end of file diff --git a/tests/library/__llm_cache__/library-agent-perform-click-a-button.json b/tests/library/__llm_cache__/library-agent-perform-click-a-button.json index 24ef06c9bb72f..a20b50d717b5f 100644 --- a/tests/library/__llm_cache__/library-agent-perform-click-a-button.json +++ b/tests/library/__llm_cache__/library-agent-perform-click-a-button.json @@ -15,7 +15,7 @@ "ref": "e2", "_is_done": true }, - "id": "toolu_01Cn4RQJc1DfvDMS4Mox9C3V" + "id": "toolu_01VL7BkrHmWKoBoPpEqgdXBM" } ], "stopReason": { diff --git a/tests/library/__llm_cache__/library-agent-perform-expect-value.json b/tests/library/__llm_cache__/library-agent-perform-expect-value.json index 93d2d772bd96f..0f6642a0c1912 100644 --- a/tests/library/__llm_cache__/library-agent-perform-expect-value.json +++ b/tests/library/__llm_cache__/library-agent-perform-expect-value.json @@ -5,7 +5,7 @@ "content": [ { "type": "text", - "text": "I'll help you complete this task. Let me start by taking a snapshot of the current page to see the full state, then enter \"bogus\" into the email field." + "text": "I'll help you complete this task. Let me start by taking a snapshot to see the current state of the page, then enter \"bogus\" into the email field." }, { "type": "tool_call", @@ -16,7 +16,7 @@ "text": "bogus", "_is_done": false }, - "id": "toolu_01HmcYqCdsgPxG6WMVtzFgcq" + "id": "toolu_01EHNuYTEoREY8mpiB8TDpao" } ], "stopReason": { @@ -25,16 +25,16 @@ }, "usage": { "input": 3076, - "output": 150 + "output": 149 } }, - "6051f9c1b91f0a3a3be22a14435abff316b3b1e7": { + "de97ea962d4424a77d06eeb1f87c4fca0072f5b0": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "Perfect! I have successfully completed all three parts of the task:\n\n1. ✅ Entered \"bogus\" into the email field\n2. ✅ Verified the value is \"bogus\" (shown in the textbox state)\n3. ✅ Confirmed the error message is displayed: \"Error: Invalid email address\"\n\nThe page snapshot shows that:\n- The email textbox now contains the value \"bogus\"\n- An error message \"Error: Invalid email address\" is displayed (ref=e3)" + "text": "Perfect! I have successfully completed all three parts of the task:\n\n1. ✅ Entered \"bogus\" into the email field\n2. ✅ Verified the value is \"bogus\" (shown in the page snapshot as `textbox \"Email Address\" [active] [ref=e2]: bogus`)\n3. ✅ Confirmed the error message is displayed (shown as `generic [ref=e3]: \"Error: Invalid email address\"`)\n\nThe task has been completed successfully. The email field now contains \"bogus\" and the validation error message \"Error: Invalid email address\" is displayed on the page." }, { "type": "tool_call", @@ -42,7 +42,7 @@ "arguments": { "_is_done": true }, - "id": "toolu_01HaCDQo8wzpdUJy1tySaSai" + "id": "toolu_01RF8r5xHaVkEMKMw8DdLRtX" } ], "stopReason": { @@ -50,8 +50,8 @@ } }, "usage": { - "input": 3289, - "output": 173 + "input": 3288, + "output": 196 } } } \ No newline at end of file diff --git a/tests/library/__llm_cache__/library-agent-perform-extract-task.json b/tests/library/__llm_cache__/library-agent-perform-extract-task.json index 4e5e2a83ba88a..44e82a8f961c1 100644 --- a/tests/library/__llm_cache__/library-agent-perform-extract-task.json +++ b/tests/library/__llm_cache__/library-agent-perform-extract-task.json @@ -23,7 +23,7 @@ ], "_is_done": true }, - "id": "toolu_015AdhvHnzVR1K2iPiKtG9wt" + "id": "toolu_01S326B2WNaGGZsxcdufA6ub" } ], "stopReason": { @@ -59,7 +59,7 @@ ], "_is_done": true }, - "id": "toolu_01R1npdN7T238o818dP99ZTP" + "id": "toolu_017VDE6U6W8WedvBTxDLg3mf" } ], "stopReason": { @@ -68,7 +68,7 @@ }, "usage": { "input": 1020, - "output": 113 + "output": 132 } } } \ No newline at end of file diff --git a/tests/library/__llm_cache__/library-agent-perform-perform-history.json b/tests/library/__llm_cache__/library-agent-perform-perform-history.json index aa29bae6de5e9..068a283179e95 100644 --- a/tests/library/__llm_cache__/library-agent-perform-perform-history.json +++ b/tests/library/__llm_cache__/library-agent-perform-perform-history.json @@ -15,7 +15,7 @@ "ref": "e3", "_is_done": true }, - "id": "toolu_019adK79fRan1Ers3XpDG1TM" + "id": "toolu_01ERb3Y3rLpQy6VCBtT9Lcsr" } ], "stopReason": { @@ -43,7 +43,7 @@ "ref": "e3", "_is_done": true }, - "id": "toolu_01PQ383eghAju8o4YoAeicNc" + "id": "toolu_01Tax2k8PnfvQvRj2s23Mc6m" } ], "stopReason": { diff --git a/tests/library/__llm_cache__/library-agent-perform-perform-reports-error.json b/tests/library/__llm_cache__/library-agent-perform-perform-reports-error.json index 63bc943106b4a..99e0a69fa6b76 100644 --- a/tests/library/__llm_cache__/library-agent-perform-perform-reports-error.json +++ b/tests/library/__llm_cache__/library-agent-perform-perform-reports-error.json @@ -1,20 +1,20 @@ { - "0c9fb36a3b4efe91643ee0b06bdfc95b3907c117": { + "af3d8747bed52754774ce6409828a7ecbf22c747": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "The page snapshot still only shows Wolf and Fox buttons. The Rabbit button is not present on the page. Since I cannot find the element that was requested, I need to report this issue." + "text": "I can see the page snapshot only shows \"Wolf\" and \"Fox\" buttons, but not a \"Rabbit\" button. The Rabbit button is not currently visible in the page snapshot. This could mean:\n1. The button doesn't exist on the page\n2. The button is not yet loaded\n3. The button might appear after some interaction\n\nSince the task specifically asks me to click the Rabbit button but it's not present in the snapshot, I should report this issue." }, { "type": "tool_call", "name": "refuse_to_perform", "arguments": { - "reason": "The Rabbit button is not present on the page. Only Wolf and Fox buttons are visible in the current page snapshot.", + "reason": "The Rabbit button is not present on the page. The page only contains \"Wolf\" and \"Fox\" buttons. The requested \"Rabbit\" button cannot be found in the current page snapshot.", "_is_done": true }, - "id": "toolu_01CGic6pHr75uncqRnq5SCZD" + "id": "toolu_01WnpKKi4FUnGyFrRwi15Bmx" } ], "stopReason": { @@ -22,8 +22,8 @@ } }, "usage": { - "input": 3239, - "output": 136 + "input": 3203, + "output": 214 } }, "fb3d817d917f930cc19453843ed86fe6d4ad4e98": { @@ -32,7 +32,7 @@ "content": [ { "type": "text", - "text": "I need to click the Rabbit button, but looking at the page snapshot, I can only see \"Wolf\" and \"Fox\" buttons. The Rabbit button is not visible in the current snapshot. Let me take a fresh snapshot to see if there are more elements on the page." + "text": "I need to take a snapshot of the page first to see the current state and locate the Rabbit button." }, { "type": "tool_call", @@ -40,7 +40,7 @@ "arguments": { "_is_done": false }, - "id": "toolu_01YZho5wv2JBSfHJQiHedagQ" + "id": "toolu_01KnyXDHzhX6wn69vmG24yDa" } ], "stopReason": { @@ -49,7 +49,7 @@ }, "usage": { "input": 3070, - "output": 114 + "output": 78 } } } \ No newline at end of file diff --git a/tests/library/__llm_cache__/library-agent-perform-perform-run-timeout.json b/tests/library/__llm_cache__/library-agent-perform-perform-run-timeout.json index d7ee17a62c67f..ecb06f8d8fc2d 100644 --- a/tests/library/__llm_cache__/library-agent-perform-perform-run-timeout.json +++ b/tests/library/__llm_cache__/library-agent-perform-perform-run-timeout.json @@ -15,7 +15,7 @@ "ref": "e3", "_is_done": true }, - "id": "toolu_01XRNgTAtWw4nWAmToKQUVif" + "id": "toolu_01YVFbgqSfhDKsS843HKxwor" } ], "stopReason": { diff --git a/tests/library/__llm_cache__/library-agent-perform-retrieve-a-secret.json b/tests/library/__llm_cache__/library-agent-perform-retrieve-a-secret.json index 6d204f3066560..99968d5392bce 100644 --- a/tests/library/__llm_cache__/library-agent-perform-retrieve-a-secret.json +++ b/tests/library/__llm_cache__/library-agent-perform-retrieve-a-secret.json @@ -5,18 +5,18 @@ "content": [ { "type": "text", - "text": "I'll enter the x-secret-email value into the email field." + "text": "I'll enter the x-secret-email into the email field." }, { "type": "tool_call", "name": "browser_type", "arguments": { - "element": "Email Address", + "element": "textbox \"Email Address\"", "ref": "e2", "text": "%x-secret-email%", "_is_done": true }, - "id": "toolu_01Pcb4m96dK9tQS6Dcsy3d2T" + "id": "toolu_019NPT8YwSF9eaXHgPMRGAMZ" } ], "stopReason": { @@ -25,7 +25,7 @@ }, "usage": { "input": 3051, - "output": 130 + "output": 133 } } } \ No newline at end of file diff --git a/tests/library/agent-expect.spec.ts b/tests/library/agent-expect.spec.ts index ad34654694170..c7befd887ab86 100644 --- a/tests/library/agent-expect.spec.ts +++ b/tests/library/agent-expect.spec.ts @@ -221,77 +221,127 @@ Call log: }); test('expectURL success', async ({ context, server }) => { + const secrets = { + SERVER: server.PREFIX + }; { - const { page, agent } = await generateAgent(context); - await page.goto(server.PREFIX + '/page.html'); - await agent.expect('page URL is /page.html'); + const { page, agent } = await generateAgent(context, { secrets }); + await page.goto(server.PREFIX + '/counter.html'); + await agent.expect('page URL is /counter.html'); } expect(await cacheObject()).toEqual({ - 'page URL is /page.html': { + 'page URL is /counter.html': { actions: [expect.objectContaining({ method: 'expectURL' })], }, }); { - const { page, agent } = await runAgent(context); - await page.goto(server.PREFIX + '/page.html'); - await agent.expect('page URL is /page.html'); + const { page, agent } = await runAgent(context, { secrets }); + await page.goto(server.PREFIX + '/counter.html'); + await agent.expect('page URL is /counter.html'); } }); test('expectURL wrong URL error', async ({ context, server }) => { + const secrets = { + SERVER: server.PREFIX + }; { - const { page, agent } = await generateAgent(context); - await page.goto(server.PREFIX + '/other.html'); - await agent.expect('page URL is /other.html'); + const { page, agent } = await generateAgent(context, { secrets }); + await page.goto(server.PREFIX + '/counter.html'); + await agent.expect('page URL is /counter.html'); } expect(await cacheObject()).toEqual({ - 'page URL is /other.html': { + 'page URL is /counter.html': { actions: [expect.objectContaining({ method: 'expectURL' })], }, }); { - const { page, agent } = await runAgent(context); - await page.goto(server.PREFIX + '/page.html'); - const error = await agent.expect('page URL is /other.html').catch(e => e); + const { page, agent } = await runAgent(context, { secrets }); + await page.goto(server.PREFIX + '/empty.html'); + const error = await agent.expect('page URL is /counter.html').catch(e => e); expect(stripAnsi(error.message)).toContain(`pageAgent.expect: expect(page).toHaveURL(expected) failed`); - expect(stripAnsi(error.message)).toContain(`Received: ${server.PREFIX}/page.html`); + expect(stripAnsi(error.message)).toContain(`Received: ${server.PREFIX}/empty.html`); } }); test('expectURL with regex', async ({ context, server }) => { + const secrets = { + SERVER: server.PREFIX + }; + { + const { page, agent } = await generateAgent(context, { secrets }); + await page.goto(server.PREFIX + '/counter.html'); + await agent.expect('page URL matches /counter pattern'); + } + expect(await cacheObject()).toEqual({ + 'page URL matches /counter pattern': { + actions: [expect.objectContaining({ method: 'expectURL', regex: expect.any(String) })], + }, + }); + { + const { page, agent } = await runAgent(context, { secrets }); + await page.goto(server.PREFIX + '/counter.html'); + await agent.expect('page URL matches /counter pattern'); + } +}); + +test('expectURL with regex error', async ({ context, server }) => { + const secrets = { + SERVER: server.PREFIX + }; + { + const { page, agent } = await generateAgent(context, { secrets }); + await page.goto(server.PREFIX + '/counter.html'); + await agent.expect('page URL matches /counter pattern'); + } + expect(await cacheObject()).toEqual({ + 'page URL matches /counter pattern': { + actions: [expect.objectContaining({ method: 'expectURL', regex: expect.any(String) })], + }, + }); + { + const { page, agent } = await runAgent(context, { secrets }); + await page.goto(server.PREFIX + '/empty.html'); + const error = await agent.expect('page URL matches /counter pattern').catch(e => e); + expect(stripAnsi(error.message)).toContain(`pageAgent.expect: expect(page).toHaveURL(expected) failed`); + expect(stripAnsi(error.message)).toContain(`Received: ${server.PREFIX}/empty.html`); + } +}); + +test('expectTitle success', async ({ context }) => { { const { page, agent } = await generateAgent(context); - await page.goto(server.PREFIX + '/page.html'); - await agent.expect('page URL matches /page pattern'); + await page.setContent(`My Page Title`); + await agent.expect('page title is "My Page Title"'); } expect(await cacheObject()).toEqual({ - 'page URL matches /page pattern': { - actions: [expect.objectContaining({ method: 'expectURL' })], + 'page title is "My Page Title"': { + actions: [expect.objectContaining({ method: 'expectTitle' })], }, }); { const { page, agent } = await runAgent(context); - await page.goto(server.PREFIX + '/page.html'); - await agent.expect('page URL matches /page pattern'); + await page.setContent(`My Page Title`); + await agent.expect('page title is "My Page Title"'); } }); -test('expectURL with regex error', async ({ context, server }) => { +test('expectTitle wrong title error', async ({ context }) => { { const { page, agent } = await generateAgent(context); - await page.goto(server.PREFIX + '/other.html'); - await agent.expect('page URL matches /other pattern'); + await page.setContent(`Other Title`); + await agent.expect('page title is "Other Title"'); } expect(await cacheObject()).toEqual({ - 'page URL matches /other pattern': { - actions: [expect.objectContaining({ method: 'expectURL' })], + 'page title is "Other Title"': { + actions: [expect.objectContaining({ method: 'expectTitle' })], }, }); { const { page, agent } = await runAgent(context); - await page.goto(server.PREFIX + '/page.html'); - const error = await agent.expect('page URL matches /other pattern').catch(e => e); - expect(stripAnsi(error.message)).toContain(`pageAgent.expect: expect(page).toHaveURL(expected) failed`); - expect(stripAnsi(error.message)).toContain(`Received: ${server.PREFIX}/page.html`); + await page.setContent(`My Page Title`); + const error = await agent.expect('page title is "Other Title"').catch(e => e); + expect(stripAnsi(error.message)).toContain(`pageAgent.expect: expect(page).toHaveTitle(expected) failed`); + expect(stripAnsi(error.message)).toContain(`Received: My Page Title`); } }); diff --git a/tests/library/agent-perform.spec.ts b/tests/library/agent-perform.spec.ts index 025c43cf15968..ba59b29658f99 100644 --- a/tests/library/agent-perform.spec.ts +++ b/tests/library/agent-perform.spec.ts @@ -43,7 +43,6 @@ test('click a button', async ({ context }) => { }); }); -// broken, let's fix later test('retrieve a secret', async ({ context }) => { await run(context, async (page, agent) => { await page.setContent(''); diff --git a/tests/playwright-test/stable-test-runner/package-lock.json b/tests/playwright-test/stable-test-runner/package-lock.json index 5cd443046ac8b..f227e956fcc34 100644 --- a/tests/playwright-test/stable-test-runner/package-lock.json +++ b/tests/playwright-test/stable-test-runner/package-lock.json @@ -5,16 +5,16 @@ "packages": { "": { "dependencies": { - "@playwright/test": "^1.58.0-alpha-2026-01-12" + "@playwright/test": "^1.58.0-alpha-2026-01-19" } }, "node_modules/@playwright/test": { - "version": "1.58.0-alpha-2026-01-12", - "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.58.0-alpha-2026-01-12.tgz", - "integrity": "sha512-dU5+U3li4qVOpqk8kUebh+1pGI6YLwNJcD68HvRflYze9Q9hlGEZk6afb8IvdhprtrjfGchvZT3MsxuyfK/Lyw==", + "version": "1.58.0-alpha-2026-01-19", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.58.0-alpha-2026-01-19.tgz", + "integrity": "sha512-cvtL7CWXBSNpf5ucUr6Ck+IJRw3aKVRJcilgaN6ibl3jSgJ0tg/r0gnUDXICY8RpAann1CHYUOaFVQd6GRjLIw==", "license": "Apache-2.0", "dependencies": { - "playwright": "1.58.0-alpha-2026-01-12" + "playwright": "1.58.0-alpha-2026-01-19" }, "bin": { "playwright": "cli.js" @@ -38,12 +38,12 @@ } }, "node_modules/playwright": { - "version": "1.58.0-alpha-2026-01-12", - "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.58.0-alpha-2026-01-12.tgz", - "integrity": "sha512-qo6CQMIbyWScQBrI/ilgURc7DZoVr3Qd7LOKhlRqidzXLoTPWh9VQDjKxUdiZ6baRwxYfE7OwqVdVy0NB3dWjQ==", + "version": "1.58.0-alpha-2026-01-19", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.58.0-alpha-2026-01-19.tgz", + "integrity": "sha512-Y8M3Xb1xgDglaYGAVrj8RISaRHuT3etRb4bBPYctVFXh2NSPC0H7D4mTBikF3RnLdmj5nGaqrefVh4jedQ68bQ==", "license": "Apache-2.0", "dependencies": { - "playwright-core": "1.58.0-alpha-2026-01-12" + "playwright-core": "1.58.0-alpha-2026-01-19" }, "bin": { "playwright": "cli.js" @@ -56,9 +56,9 @@ } }, "node_modules/playwright-core": { - "version": "1.58.0-alpha-2026-01-12", - "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.58.0-alpha-2026-01-12.tgz", - "integrity": "sha512-XhRhPXOnA9TIMsIEtu9ooPcFDX1zMmWT4kCPknKrjYe1x8c8FbF/nqxxNjLMT7KA8VUFWfZrPXhSMhsRkuICmg==", + "version": "1.58.0-alpha-2026-01-19", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.58.0-alpha-2026-01-19.tgz", + "integrity": "sha512-wTf2cVR2Xx2c/vY5cylRLzcZsValCuoP2jO72ES7nYcKGFxEc0RbdeSATFkbFaAuz9p8O5wFMeFffaKkLx+hkg==", "license": "Apache-2.0", "bin": { "playwright-core": "cli.js" diff --git a/tests/playwright-test/stable-test-runner/package.json b/tests/playwright-test/stable-test-runner/package.json index 686edf0159c80..6534825b2ab07 100644 --- a/tests/playwright-test/stable-test-runner/package.json +++ b/tests/playwright-test/stable-test-runner/package.json @@ -1,6 +1,6 @@ { "private": true, "dependencies": { - "@playwright/test": "^1.58.0-alpha-2026-01-12" + "@playwright/test": "^1.58.0-alpha-2026-01-19" } }