diff --git a/packages/playwright-core/src/server/agent/context.ts b/packages/playwright-core/src/server/agent/context.ts index 41853f073d376..9b84009f67279 100644 --- a/packages/playwright-core/src/server/agent/context.ts +++ b/packages/playwright-core/src/server/agent/context.ts @@ -150,15 +150,6 @@ export class Context { text.push(`# Page snapshot\n${snapshot}`); return { - _meta: { - 'dev.lowire/state': { - 'Page snapshot': snapshot - }, - 'dev.lowire/history': error ? [{ - category: 'error', - content: stripAnsiEscapes(error.message), - }] : [], - }, isError: !!error, content: [{ type: 'text', text: text.join('\n\n') }], }; diff --git a/packages/playwright/src/mcp/browser/actions.d.ts b/packages/playwright/src/mcp/browser/actions.d.ts deleted file mode 100644 index e52b420b9041f..0000000000000 --- a/packages/playwright/src/mcp/browser/actions.d.ts +++ /dev/null @@ -1,172 +0,0 @@ -/** - * Copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -type Point = { x: number, y: number }; - -export type ActionName = - 'check' | - 'click' | - 'closePage' | - 'fill' | - 'navigate' | - 'openPage' | - 'press' | - 'select' | - 'uncheck' | - 'setInputFiles' | - 'assertText' | - 'assertValue' | - 'assertChecked' | - 'assertVisible' | - 'assertSnapshot'; - -export type ActionBase = { - name: ActionName, - signals: Signal[], - ariaSnapshot?: string, -}; - -export type ActionWithSelector = ActionBase & { - selector: string, - ref?: string, -}; - -export type ClickAction = ActionWithSelector & { - name: 'click', - button: 'left' | 'middle' | 'right', - modifiers: number, - clickCount: number, - position?: Point, -}; - -export type CheckAction = ActionWithSelector & { - name: 'check', -}; - -export type UncheckAction = ActionWithSelector & { - name: 'uncheck', -}; - -export type FillAction = ActionWithSelector & { - name: 'fill', - text: string, -}; - -export type NavigateAction = ActionBase & { - name: 'navigate', - url: string, -}; - -export type OpenPageAction = ActionBase & { - name: 'openPage', - url: string, -}; - -export type ClosesPageAction = ActionBase & { - name: 'closePage', -}; - -export type PressAction = ActionWithSelector & { - name: 'press', - key: string, - modifiers: number, -}; - -export type SelectAction = ActionWithSelector & { - name: 'select', - options: string[], -}; - -export type SetInputFilesAction = ActionWithSelector & { - name: 'setInputFiles', - files: string[], -}; - -export type AssertTextAction = ActionWithSelector & { - name: 'assertText', - text: string, - substring: boolean, -}; - -export type AssertValueAction = ActionWithSelector & { - name: 'assertValue', - value: string, -}; - -export type AssertCheckedAction = ActionWithSelector & { - name: 'assertChecked', - checked: boolean, -}; - -export type AssertVisibleAction = ActionWithSelector & { - name: 'assertVisible', -}; - -export type AssertSnapshotAction = ActionWithSelector & { - name: 'assertSnapshot', - ariaSnapshot: string, -}; - -export type Action = ClickAction | CheckAction | ClosesPageAction | OpenPageAction | UncheckAction | FillAction | NavigateAction | PressAction | SelectAction | SetInputFilesAction | AssertTextAction | AssertValueAction | AssertCheckedAction | AssertVisibleAction | AssertSnapshotAction; -export type AssertAction = AssertCheckedAction | AssertValueAction | AssertTextAction | AssertVisibleAction | AssertSnapshotAction; -export type PerformOnRecordAction = ClickAction | CheckAction | UncheckAction | PressAction | SelectAction; - -// Signals. - -export type BaseSignal = { -}; - -export type NavigationSignal = BaseSignal & { - name: 'navigation', - url: string, -}; - -export type PopupSignal = BaseSignal & { - name: 'popup', - popupAlias: string, -}; - -export type DownloadSignal = BaseSignal & { - name: 'download', - downloadAlias: string, -}; - -export type DialogSignal = BaseSignal & { - name: 'dialog', - dialogAlias: string, -}; - -export type Signal = NavigationSignal | PopupSignal | DownloadSignal | DialogSignal; - -export type FrameDescription = { - pageGuid: string; - pageAlias: string; - framePath: string[]; -}; - -export type ActionInContext = { - frame: FrameDescription; - description?: string; - action: Action; - startTime: number; - endTime?: number; -}; - -export type SignalInContext = { - frame: FrameDescription; - signal: Signal; - timestamp: number; -}; diff --git a/packages/playwright/src/mcp/browser/browserContextFactory.ts b/packages/playwright/src/mcp/browser/browserContextFactory.ts index d2428421ab310..abec50f070926 100644 --- a/packages/playwright/src/mcp/browser/browserContextFactory.ts +++ b/packages/playwright/src/mcp/browser/browserContextFactory.ts @@ -350,13 +350,13 @@ export class SharedContextFactory implements BrowserContextFactory { async function computeTracesDir(config: FullConfig, clientInfo: ClientInfo): Promise { if (!config.saveTrace && !config.capabilities?.includes('tracing')) return; - return await outputFile(config, clientInfo, `traces`, { origin: 'code', reason: 'Collecting trace' }); + return await outputFile(config, clientInfo, `traces`, { origin: 'code', title: 'Collecting trace' }); } async function browserContextOptionsFromConfig(config: FullConfig, clientInfo: ClientInfo): Promise { const result = { ...config.browser.contextOptions }; if (config.saveVideo) { - const dir = await outputFile(config, clientInfo, `videos`, { origin: 'code', reason: 'Saving video' }); + const dir = await outputFile(config, clientInfo, `videos`, { origin: 'code', title: 'Saving video' }); result.recordVideo = { dir, size: config.saveVideo, diff --git a/packages/playwright/src/mcp/browser/browserServerBackend.ts b/packages/playwright/src/mcp/browser/browserServerBackend.ts index 83b5a29dd652f..67dc800fe6d7a 100644 --- a/packages/playwright/src/mcp/browser/browserServerBackend.ts +++ b/packages/playwright/src/mcp/browser/browserServerBackend.ts @@ -56,25 +56,30 @@ export class BrowserServerBackend implements ServerBackend { async callTool(name: string, rawArguments: mcpServer.CallToolRequest['params']['arguments']) { const tool = this._tools.find(tool => tool.schema.name === name)!; - if (!tool) - throw new Error(`Tool "${name}" not found`); + if (!tool) { + return { + content: [{ type: 'text' as const, text: `### Error\nTool "${name}" not found` }], + isError: true, + }; + } const parsedArguments = tool.schema.inputSchema.parse(rawArguments || {}) as any; const context = this._context!; - const response = new Response(context, name, parsedArguments); - response.logBegin(); + const response = Response.create(context, name, parsedArguments); context.setRunningTool(name); + let responseObject: mcpServer.CallToolResult; try { await tool.handle(context, parsedArguments, response); - await response.finish(); - this._sessionLog?.logResponse(response); + responseObject = await response.build(); + this._sessionLog?.logResponse(name, parsedArguments, responseObject); } catch (error: any) { - response.addError(String(error)); + return { + content: [{ type: 'text' as const, text: `### Error\n${String(error)}` }], + isError: true, + }; } finally { context.setRunningTool(undefined); } - response.logEnd(); - const _meta = rawArguments?._meta as object | undefined; - return response.serialize({ _meta }); + return responseObject; } serverClosed() { diff --git a/packages/playwright/src/mcp/browser/config.ts b/packages/playwright/src/mcp/browser/config.ts index bed7edea6ff65..953b98989fde4 100644 --- a/packages/playwright/src/mcp/browser/config.ts +++ b/packages/playwright/src/mcp/browser/config.ts @@ -54,6 +54,7 @@ export type CLIOptions = { imageResponses?: 'allow' | 'omit'; sandbox?: boolean; outputDir?: string; + outputMode?: 'file' | 'stdout'; port?: number; proxyBypass?: string; proxyServer?: string; @@ -95,6 +96,7 @@ export const defaultConfig: FullConfig = { saveTrace: false, snapshot: { mode: 'incremental', + output: 'stdout', }, timeouts: { action: 5000, @@ -118,6 +120,7 @@ export type FullConfig = Config & { server: NonNullable, snapshot: { mode: 'incremental' | 'full' | 'none'; + output: 'stdout' | 'file'; }, timeouts: { action: number; @@ -257,6 +260,7 @@ export function configFromCLIOptions(cliOptions: CLIOptions): Config { secrets: cliOptions.secrets, sharedBrowserContext: cliOptions.sharedBrowserContext, snapshot: cliOptions.snapshotMode ? { mode: cliOptions.snapshotMode } : undefined, + outputMode: cliOptions.outputMode, outputDir: cliOptions.outputDir, imageResponses: cliOptions.imageResponses, testIdAttribute: cliOptions.testIdAttribute, @@ -338,10 +342,10 @@ export function outputDir(config: FullConfig, clientInfo: ClientInfo): string { ?? path.join(tmpDir(), String(clientInfo.timestamp)); } -export async function outputFile(config: FullConfig, clientInfo: ClientInfo, fileName: string, options: { origin: 'code' | 'llm' | 'web', reason: string }): Promise { +export async function outputFile(config: FullConfig, clientInfo: ClientInfo, fileName: string, options: { origin: 'code' | 'llm' | 'web', title: string }): Promise { const file = await resolveFile(config, clientInfo, fileName, options); await fs.promises.mkdir(path.dirname(file), { recursive: true }); - debug('pw:mcp:file')(options.reason, file); + debug('pw:mcp:file')(options.title, file); return file; } diff --git a/packages/playwright/src/mcp/browser/context.ts b/packages/playwright/src/mcp/browser/context.ts index f916051314c8e..674ca1df083b9 100644 --- a/packages/playwright/src/mcp/browser/context.ts +++ b/packages/playwright/src/mcp/browser/context.ts @@ -27,7 +27,6 @@ import { outputFile } from './config'; import type * as playwright from '../../../types/test'; import type { FullConfig } from './config'; import type { BrowserContextFactory, BrowserContextFactoryResult } from './browserContextFactory'; -import type * as actions from './actions'; import type { SessionLog } from './sessionLog'; import type { Tracing } from '../../../../playwright-core/src/client/tracing'; import type { ClientInfo } from '../sdk/server'; @@ -116,7 +115,7 @@ export class Context { return url; } - async outputFile(fileName: string, options: { origin: 'code' | 'llm' | 'web', reason: string }): Promise { + async outputFile(fileName: string, options: { origin: 'code' | 'llm' | 'web', title: string }): Promise { return outputFile(this.config, this._clientInfo, fileName, options); } @@ -219,8 +218,6 @@ export class Context { (browserContext as any)._setAllowedDirectories(allRootPaths(this._clientInfo)); } await this._setupRequestInterception(browserContext); - if (this.sessionLog) - await InputRecorder.create(this, browserContext); for (const page of browserContext.pages()) this._onPageCreated(page); browserContext.on('page', page => this._onPageCreated(page)); @@ -243,6 +240,10 @@ export class Context { code: `process.env['${secretName}']`, }; } + + firstRootPath(): string | undefined { + return allRootPaths(this._clientInfo)[0]; + } } function allRootPaths(clientInfo: ClientInfo): string[] { @@ -278,56 +279,3 @@ function originOrHostGlob(originOrHost: string) { // Support for legacy host-only mode. return `*://${originOrHost}/**`; } - -export class InputRecorder { - private _context: Context; - private _browserContext: playwright.BrowserContext; - - private constructor(context: Context, browserContext: playwright.BrowserContext) { - this._context = context; - this._browserContext = browserContext; - } - - static async create(context: Context, browserContext: playwright.BrowserContext) { - const recorder = new InputRecorder(context, browserContext); - await recorder._initialize(); - return recorder; - } - - private async _initialize() { - const sessionLog = this._context.sessionLog!; - await (this._browserContext as any)._enableRecorder({ - mode: 'recording', - recorderMode: 'api', - }, { - actionAdded: (page: playwright.Page, data: actions.ActionInContext, code: string) => { - if (this._context.isRunningTool()) - return; - const tab = Tab.forPage(page); - if (tab) - sessionLog.logUserAction(data.action, tab, code, false); - }, - actionUpdated: (page: playwright.Page, data: actions.ActionInContext, code: string) => { - if (this._context.isRunningTool()) - return; - const tab = Tab.forPage(page); - if (tab) - sessionLog.logUserAction(data.action, tab, code, true); - }, - signalAdded: (page: playwright.Page, data: actions.SignalInContext) => { - if (this._context.isRunningTool()) - return; - if (data.signal.name !== 'navigation') - return; - const tab = Tab.forPage(page); - const navigateAction: actions.Action = { - name: 'navigate', - url: data.signal.url, - signals: [], - }; - if (tab) - sessionLog.logUserAction(navigateAction, tab, `await page.goto('${data.signal.url}');`, false); - }, - }); - } -} diff --git a/packages/playwright/src/mcp/browser/response.ts b/packages/playwright/src/mcp/browser/response.ts index 81435ea3425b8..dcbda0ab69dba 100644 --- a/packages/playwright/src/mcp/browser/response.ts +++ b/packages/playwright/src/mcp/browser/response.ts @@ -14,188 +14,202 @@ * limitations under the License. */ +import fs from 'fs'; import path from 'path'; import { debug } from 'playwright-core/lib/utilsBundle'; -import { renderModalStates } from './tab'; -import { firstRootPath } from '../sdk/server'; +import { renderModalStates, shouldIncludeMessage } from './tab'; +import { dateAsFileName } from './tools/utils'; -import type { Tab, TabSnapshot } from './tab'; +import type { TabHeader } from './tab'; import type { CallToolResult, ImageContent, TextContent } from '@modelcontextprotocol/sdk/types.js'; import type { Context } from './context'; -import type { ModalState } from './tools/tool'; export const requestDebug = debug('pw:mcp:request'); +type Result = { + text?: string; + data?: Buffer; + title?: string; + filename?: string; +}; + export class Response { - private _result: string[] = []; + private _ordinal: number; + private _results: Result[] = []; + private _errors: string[] = []; private _code: string[] = []; private _images: { contentType: string, data: Buffer }[] = []; - private _files: { fileName: string, title: string }[] = []; private _context: Context; private _includeSnapshot: 'none' | 'full' | 'incremental' = 'none'; - private _includeTabs = false; - private _includeModalStates: ModalState[] | undefined; - private _includeMetaOnly: boolean = false; - private _tabSnapshot: TabSnapshot | undefined; + private _includeSnapshotFileName: string | undefined; readonly toolName: string; readonly toolArgs: Record; - private _isError: boolean | undefined; - constructor(context: Context, toolName: string, toolArgs: Record) { + private constructor(ordinal: number, context: Context, toolName: string, toolArgs: Record) { + this._ordinal = ordinal; this._context = context; this.toolName = toolName; this.toolArgs = toolArgs; } - addResult(result: string) { - this._result.push(result); + static _ordinal = 0; + + static create(context: Context, toolName: string, toolArgs: Record) { + return new Response(++Response._ordinal, context, toolName, toolArgs); } - addError(error: string) { - this._result.push(error); - this._isError = true; + addTextResult(result: string) { + this._results.push({ text: result }); } - isError() { - return this._isError; + async addResult(result: { text?: string, data?: Buffer, title?: string, suggestedFilename?: string, ext?: string }): Promise<{ fileName?: string }> { + // Binary always goes into a file. + if (result.data && !result.suggestedFilename) + result.suggestedFilename = dateAsFileName(result.ext ?? 'bin'); + + // What can go into a file goes into a file in outputMode === file. + if (this._context.config.outputMode === 'file') { + if (!result.suggestedFilename) + result.suggestedFilename = dateAsFileName(result.ext ?? (result.text ? 'txt' : 'bin')); + } + + const entry: Result = { text: result.text, data: result.data, title: result.title }; + if (result.suggestedFilename) + entry.filename = await this._context.outputFile(result.suggestedFilename, { origin: 'llm', title: result.title ?? 'Saved result' }); + + this._results.push(entry); + return { fileName: entry.filename }; } - result() { - return this._result.join('\n'); + addError(error: string) { + this._errors.push(error); } addCode(code: string) { this._code.push(code); } - code() { - return this._code.join('\n'); - } - addImage(image: { contentType: string, data: Buffer }) { this._images.push(image); } - images() { - return this._images; - } - - async addFile(fileName: string, options: { origin: 'code' | 'llm' | 'web', reason: string }) { - const resolvedFile = await this._context.outputFile(fileName, options); - this._files.push({ fileName: resolvedFile, title: options.reason }); - return resolvedFile; - } - setIncludeSnapshot() { this._includeSnapshot = this._context.config.snapshot.mode; } - setIncludeFullSnapshot() { + setIncludeFullSnapshot(includeSnapshotFileName?: string) { this._includeSnapshot = 'full'; + this._includeSnapshotFileName = includeSnapshotFileName; } - setIncludeTabs() { - this._includeTabs = true; - } - - setIncludeModalStates(modalStates: ModalState[]) { - this._includeModalStates = modalStates; - } - - setIncludeMetaOnly() { - this._includeMetaOnly = true; - } - - async finish() { - if (this._tabSnapshot) - return; - - // All the async snapshotting post-action is happening here. - // Everything below should race against modal states. - if (this._context.currentTab()) - this._tabSnapshot = await this._context.currentTabOrDie().captureSnapshot(this._includeSnapshot !== 'none'); - for (const tab of this._context.tabs()) - await tab.updateTitle(); - } - - tabSnapshot(): TabSnapshot | undefined { - return this._tabSnapshot; - } - - logBegin() { - if (requestDebug.enabled) - requestDebug(this.toolName, this.toolArgs); - } - - logEnd() { - if (requestDebug.enabled) - requestDebug(this.serialize()); - } + async build(): Promise<{ content: (TextContent | ImageContent)[], isError?: boolean }> { + const rootPath = this._context.firstRootPath(); + const text: string[] = []; - render(): RenderedResponse{ - const renderedResponse = new RenderedResponse(); + if (this._errors.length) { + text.push('### Error'); + text.push(this._errors.join('\n')); + text.push(''); + } - if (this._result.length) - renderedResponse.results.push(...this._result); + // Results + if (this._results.length) { + text.push('### Result'); + for (const result of this._results) { + if (result.filename) { + text.push(`- [${result.title}](${rootPath ? path.relative(rootPath, result.filename) : result.filename})`); + if (result.data) + await fs.promises.writeFile(result.filename, result.data); + else if (result.text) + await fs.promises.writeFile(result.filename, this._redactText(result.text)); + } else if (result.text) { + text.push(result.text); + } + } + text.push(''); + } - // Add code if it exists. - if (this._code.length && this._context.config.codegen !== 'none') - renderedResponse.code.push(...this._code); + // Code + if (this._context.config.codegen !== 'none' && this._code.length) { + text.push('### Ran Playwright code'); + text.push(...this._code); + text.push(''); + } - // List browser tabs. - if (this._includeSnapshot !== 'none' || this._includeTabs) { - const tabsMarkdown = renderTabsMarkdown(this._context.tabs(), this._includeTabs); - if (tabsMarkdown.length) - renderedResponse.states.tabs = tabsMarkdown.join('\n'); + // Render tab titles upon changes or when more than one tab. + const tabSnapshot = this._context.currentTab() ? await this._context.currentTabOrDie().captureSnapshot() : undefined; + const tabHeaders = await Promise.all(this._context.tabs().map(tab => tab.headerSnapshot())); + if (tabHeaders.some(header => header.changed)) { + if (tabHeaders.length !== 1) + text.push('### Open tabs'); + else if (this._context.config.outputMode !== 'file') + text.push('### Page'); + text.push(...renderTabsMarkdown(tabHeaders)); + text.push(''); } - // Add snapshot if provided. - if (this._tabSnapshot?.modalStates.length) { - const modalStatesMarkdown = renderModalStates(this._tabSnapshot.modalStates); - renderedResponse.states.modal = modalStatesMarkdown.join('\n'); - } else if (this._includeModalStates) { - const modalStatesMarkdown = renderModalStates(this._includeModalStates); - renderedResponse.states.modal = modalStatesMarkdown.join('\n'); - } else if (this._tabSnapshot) { - renderTabSnapshot(this._tabSnapshot, this._includeSnapshot, renderedResponse); + // Handle modal states. + if (tabSnapshot?.modalStates.length) { + text.push('### Modal state'); + text.push(...renderModalStates(tabSnapshot.modalStates)); + text.push(''); } - if (this._files.length) { - const lines: string[] = []; - for (const file of this._files) { - const filePath = this._pathRelativeToRoot(file.fileName); - lines.push(`- [${file.title}](${filePath})`); + // Handle tab log + if (tabSnapshot?.events.filter(event => event.type !== 'request').length) { + text.push('### Events'); + for (const event of tabSnapshot.events) { + if (event.type === 'console') { + if (shouldIncludeMessage(this._context.config.console.level, event.message.type)) + text.push(`- ${trimMiddle(event.message.toString(), 100)}`); + } else if (event.type === 'download-start') { + text.push(`- Downloading file ${event.download.download.suggestedFilename()} ...`); + } else if (event.type === 'download-finish') { + text.push(`- Downloaded file ${event.download.download.suggestedFilename()} to "${rootPath ? path.relative(rootPath, event.download.outputFile) : event.download.outputFile}"`); + } } - renderedResponse.updates.push({ category: 'files', content: lines.join('\n') }); + text.push(''); } - return this._context.config.secrets ? renderedResponse.redact(this._context.config.secrets) : renderedResponse; - } - - private _pathRelativeToRoot(filePath: string): string { - const root = firstRootPath(this._context.options.clientInfo); - if (!root) - return filePath; - return path.relative(root, filePath); - } + // Handle tab snapshot + if (tabSnapshot && this._includeSnapshot === 'full') { + let fileName: string | undefined; + if (this._includeSnapshotFileName) + fileName = await this._context.outputFile(this._includeSnapshotFileName, { origin: 'llm', title: 'Saved snapshot' }); + else if (this._context.config.outputMode === 'file') + fileName = await this._context.outputFile(`snapshot-${this._ordinal}.yml`, { origin: 'code', title: 'Saved snapshot' }); + if (fileName) { + await fs.promises.writeFile(fileName, tabSnapshot.ariaSnapshot); + text.push(`- [Page snapshot](${rootPath ? path.relative(rootPath, fileName) : fileName})`); + } else { + text.push(`### Snapshot`); + text.push('```yaml'); + text.push(tabSnapshot.ariaSnapshot); + text.push('```'); + text.push(''); + } + } - serialize(options: { _meta?: Record } = {}): { content: (TextContent | ImageContent)[], isError?: boolean, _meta?: Record } { - const renderedResponse = this.render(); - const includeMeta = options._meta && 'dev.lowire/history' in options._meta && 'dev.lowire/state' in options._meta; - const _meta: any = includeMeta ? renderedResponse.asMeta() : undefined; + if (tabSnapshot && this._includeSnapshot === 'incremental') { + text.push(`### Snapshot`); + text.push('```yaml'); + if (tabSnapshot.ariaSnapshotDiff !== undefined) + text.push(tabSnapshot.ariaSnapshotDiff); + else + text.push(tabSnapshot.ariaSnapshot); + text.push('```'); + text.push(''); + } const content: (TextContent | ImageContent)[] = [ { type: 'text', - text: renderedResponse.asText(this._includeMetaOnly ? { categories: ['files'] } : undefined) + text: this._redactText(text.join('\n')) }, ]; - if (this._includeMetaOnly) - return { _meta, content, isError: this._isError }; - // Image attachments. if (this._context.config.imageResponses !== 'omit') { for (const image of this._images) @@ -203,65 +217,35 @@ export class Response { } return { - _meta, content, - isError: this._isError + ...this._errors.length > 0 ? { isError: true } : {}, }; } -} - -function renderTabSnapshot(tabSnapshot: TabSnapshot, includeSnapshot: 'none' | 'full' | 'incremental', response: RenderedResponse) { - if (tabSnapshot.consoleMessages.length) { - const lines: string[] = []; - for (const message of tabSnapshot.consoleMessages) - lines.push(`- ${trimMiddle(message.toString(), 100)}`); - response.updates.push({ category: 'console', content: lines.join('\n') }); - } - - if (tabSnapshot.downloads.length) { - const lines: string[] = []; - for (const entry of tabSnapshot.downloads) { - if (entry.finished) - lines.push(`- Downloaded file ${entry.download.suggestedFilename()} to ${entry.outputFile}`); - else - lines.push(`- Downloading file ${entry.download.suggestedFilename()} ...`); - } - response.updates.push({ category: 'downloads', content: lines.join('\n') }); - } - if (includeSnapshot === 'incremental' && tabSnapshot.ariaSnapshotDiff === '') { - // When incremental snapshot is present, but empty, do not render page state altogether. - return; - } - - const lines: string[] = []; - lines.push(`- Page URL: ${tabSnapshot.url}`); - lines.push(`- Page Title: ${tabSnapshot.title}`); - - if (includeSnapshot !== 'none') { - lines.push(`- Page Snapshot:`); - lines.push('```yaml'); - if (includeSnapshot === 'incremental' && tabSnapshot.ariaSnapshotDiff !== undefined) - lines.push(tabSnapshot.ariaSnapshotDiff); - else - lines.push(tabSnapshot.ariaSnapshot); - lines.push('```'); + private _redactText(text: string): string { + for (const [secretName, secretValue] of Object.entries(this._context.config.secrets ?? {})) + text = text.replaceAll(secretValue, `${secretName}`); + return text; } - response.states.page = lines.join('\n'); } -function renderTabsMarkdown(tabs: Tab[], force: boolean = false): string[] { - if (tabs.length === 1 && !force) - return []; - +export function renderTabsMarkdown(tabs: TabHeader[]): string[] { if (!tabs.length) return ['No open tabs. Use the "browser_navigate" tool to navigate to a page first.']; const lines: string[] = []; + + if (tabs.length === 1) { + lines.push(`- Page URL: ${tabs[0].url}`); + if (tabs[0].title) + lines.push(`- Page Title: ${tabs[0].title}`); + return lines; + } + for (let i = 0; i < tabs.length; i++) { const tab = tabs[i]; - const current = tab.isCurrentTab() ? ' (current)' : ''; - lines.push(`- ${i}:${current} [${tab.lastTitle()}] (${tab.page.url()})`); + const current = tab.current ? ' (current)' : ''; + lines.push(`- ${i}:${current} [${tab.title}](${tab.url})`); } return lines; } @@ -272,93 +256,6 @@ function trimMiddle(text: string, maxLength: number) { return text.slice(0, Math.floor(maxLength / 2)) + '...' + text.slice(- 3 - Math.floor(maxLength / 2)); } -export class RenderedResponse { - readonly states: Partial> = {}; - readonly updates: { category: 'console' | 'downloads' | 'files', content: string }[] = []; - readonly results: string[] = []; - readonly code: string[] = []; - - constructor(copy?: { states: Partial>, updates: { category: 'console' | 'downloads' | 'files', content: string }[], results: string[], code: string[] }) { - if (copy) { - this.states = copy.states; - this.updates = copy.updates; - this.results = copy.results; - this.code = copy.code; - } - } - - asText(filter?: { categories: string[] }): string { - const text: string[] = []; - if (this.results.length) - text.push(`### Result\n${this.results.join('\n')}\n`); - if (this.code.length) - text.push(`### Ran Playwright code\n${this.code.join('\n')}\n`); - - for (const { category, content } of this.updates) { - if (filter && !filter.categories.includes(category)) - continue; - if (!content.trim()) - continue; - - switch (category) { - case 'console': - text.push(`### New console messages\n${content}\n`); - break; - case 'downloads': - text.push(`### Downloads\n${content}\n`); - break; - case 'files': - text.push(`### Files\n${content}\n`); - break; - } - } - - for (const [category, value] of Object.entries(this.states)) { - if (filter && !filter.categories.includes(category)) - continue; - if (!value.trim()) - continue; - - switch (category) { - case 'page': - text.push(`### Page state\n${value}\n`); - break; - case 'tabs': - text.push(`### Open tabs\n${value}\n`); - break; - case 'modal': - text.push(`### Modal state\n${value}\n`); - break; - } - } - return text.join('\n'); - } - - asMeta() { - const codeUpdate = this.code.length ? { category: 'code', content: this.code.join('\n') } : undefined; - const resultUpdate = this.results.length ? { category: 'result', content: this.results.join('\n') } : undefined; - const updates = [resultUpdate, codeUpdate, ...this.updates].filter(Boolean); - return { - 'dev.lowire/history': updates, - 'dev.lowire/state': { ...this.states }, - }; - } - - redact(secrets: Record): RenderedResponse { - const redactText = (text: string) => { - for (const [secretName, secretValue] of Object.entries(secrets)) - text = text.replaceAll(secretValue, `${secretName}`); - return text; - }; - - const updates = this.updates.map(update => ({ ...update, content: redactText(update.content) })); - const results = this.results.map(result => redactText(result)); - const code = this.code.map(code => redactText(code)); - const states = Object.fromEntries(Object.entries(this.states).map(([key, value]) => [key, redactText(value)])); - return new RenderedResponse({ states, updates, results, code }); - } -} - function parseSections(text: string): Map { const sections = new Map(); const sectionHeaders = text.split(/^### /m).slice(1); // Remove empty first element @@ -382,29 +279,29 @@ export function parseResponse(response: CallToolResult) { const text = response.content[0].text; const sections = parseSections(text); + const error = sections.get('Error'); const result = sections.get('Result'); const code = sections.get('Ran Playwright code'); const tabs = sections.get('Open tabs'); - const pageState = sections.get('Page state'); - const consoleMessages = sections.get('New console messages'); + const page = sections.get('Page'); + const snapshot = sections.get('Snapshot'); + const events = sections.get('Events'); const modalState = sections.get('Modal state'); - const downloads = sections.get('Downloads'); - const files = sections.get('Files'); const codeNoFrame = code?.replace(/^```js\n/, '').replace(/\n```$/, ''); const isError = response.isError; - const attachments = response.content.slice(1); + const attachments = response.content.length > 1 ? response.content.slice(1) : undefined; return { result, + error, code: codeNoFrame, tabs, - pageState, - consoleMessages, + page, + snapshot, + events, modalState, - downloads, - files, isError, attachments, - _meta: response._meta, + text, }; } diff --git a/packages/playwright/src/mcp/browser/sessionLog.ts b/packages/playwright/src/mcp/browser/sessionLog.ts index e0558b866c435..2abae731f5933 100644 --- a/packages/playwright/src/mcp/browser/sessionLog.ts +++ b/packages/playwright/src/mcp/browser/sessionLog.ts @@ -17,35 +17,16 @@ import fs from 'fs'; import path from 'path'; -import { Response } from './response'; -import { logUnhandledError } from '../log'; import { outputFile } from './config'; +import { parseResponse } from './response'; import type { FullConfig } from './config'; -import type * as actions from './actions'; -import type { Tab, TabSnapshot } from './tab'; import type * as mcpServer from '../sdk/server'; -type LogEntry = { - timestamp: number; - toolCall?: { - toolName: string; - toolArgs: Record; - result: string; - isError?: boolean; - }; - userAction?: actions.Action; - code: string; - tabSnapshot?: TabSnapshot; -}; - export class SessionLog { private _folder: string; private _file: string; - private _ordinal = 0; - private _pendingEntries: LogEntry[] = []; private _sessionFileQueue = Promise.resolve(); - private _flushEntriesTimeout: NodeJS.Timeout | undefined; constructor(sessionFolder: string) { this._folder = sessionFolder; @@ -53,125 +34,34 @@ export class SessionLog { } static async create(config: FullConfig, clientInfo: mcpServer.ClientInfo): Promise { - const sessionFolder = await outputFile(config, clientInfo, `session-${Date.now()}`, { origin: 'code', reason: 'Saving session' }); + const sessionFolder = await outputFile(config, clientInfo, `session-${Date.now()}`, { origin: 'code', title: 'Saving session' }); await fs.promises.mkdir(sessionFolder, { recursive: true }); // eslint-disable-next-line no-console console.error(`Session: ${sessionFolder}`); return new SessionLog(sessionFolder); } - logResponse(response: Response) { - const entry: LogEntry = { - timestamp: performance.now(), - toolCall: { - toolName: response.toolName, - toolArgs: response.toolArgs, - result: response.result(), - isError: response.isError(), - }, - code: response.code(), - tabSnapshot: response.tabSnapshot(), - }; - this._appendEntry(entry); - } - - logUserAction(action: actions.Action, tab: Tab, code: string, isUpdate: boolean) { - code = code.trim(); - if (isUpdate) { - const lastEntry = this._pendingEntries[this._pendingEntries.length - 1]; - if (lastEntry?.userAction?.name === action.name) { - lastEntry.userAction = action; - lastEntry.code = code; - return; - } - } - if (action.name === 'navigate') { - // Already logged at this location. - const lastEntry = this._pendingEntries[this._pendingEntries.length - 1]; - if (lastEntry?.tabSnapshot?.url === action.url) - return; - } - const entry: LogEntry = { - timestamp: performance.now(), - userAction: action, - code, - tabSnapshot: { - url: tab.page.url(), - title: '', - ariaSnapshot: action.ariaSnapshot || '', - modalStates: [], - consoleMessages: [], - downloads: [], - }, - }; - this._appendEntry(entry); - } - - private _appendEntry(entry: LogEntry) { - this._pendingEntries.push(entry); - if (this._flushEntriesTimeout) - clearTimeout(this._flushEntriesTimeout); - this._flushEntriesTimeout = setTimeout(() => this._flushEntries(), 1000); - } + logResponse(toolName: string, toolArgs: Record, responseObject: any) { + const parsed = parseResponse(responseObject) as any; + if (parsed) + delete parsed.text; - private async _flushEntries() { - clearTimeout(this._flushEntriesTimeout); - const entries = this._pendingEntries; - this._pendingEntries = []; const lines: string[] = ['']; - - for (const entry of entries) { - const ordinal = (++this._ordinal).toString().padStart(3, '0'); - if (entry.toolCall) { - lines.push( - `### Tool call: ${entry.toolCall.toolName}`, - `- Args`, - '```json', - JSON.stringify(entry.toolCall.toolArgs, null, 2), - '```', - ); - if (entry.toolCall.result) { - lines.push( - entry.toolCall.isError ? `- Error` : `- Result`, - '```', - entry.toolCall.result, - '```', - ); - } - } - - if (entry.userAction) { - const actionData = { ...entry.userAction } as any; - delete actionData.ariaSnapshot; - delete actionData.selector; - delete actionData.signals; - - lines.push( - `### User action: ${entry.userAction.name}`, - `- Args`, - '```json', - JSON.stringify(actionData, null, 2), - '```', - ); - } - - if (entry.code) { - lines.push( - `- Code`, - '```js', - entry.code, - '```'); - } - - if (entry.tabSnapshot) { - const fileName = `${ordinal}.snapshot.yml`; - fs.promises.writeFile(path.join(this._folder, fileName), entry.tabSnapshot.ariaSnapshot).catch(logUnhandledError); - lines.push(`- Snapshot: ${fileName}`); - } - - lines.push('', ''); + lines.push( + `### Tool call: ${toolName}`, + `- Args`, + '```json', + JSON.stringify(toolArgs, null, 2), + '```', + ); + if (parsed) { + lines.push(`- Result`); + lines.push('```json'); + lines.push(JSON.stringify(parsed, null, 2)); + lines.push('```'); } + lines.push(''); this._sessionFileQueue = this._sessionFileQueue.then(() => fs.promises.appendFile(this._file, lines.join('\n'))); } } diff --git a/packages/playwright/src/mcp/browser/tab.ts b/packages/playwright/src/mcp/browser/tab.ts index 958a9c27a2938..a346e6f628b18 100644 --- a/packages/playwright/src/mcp/browser/tab.ts +++ b/packages/playwright/src/mcp/browser/tab.ts @@ -37,29 +37,65 @@ export type TabEventsInterface = { [TabEvents.modalState]: [modalState: ModalState]; }; -export type TabSnapshot = { - url: string; +type Download = { + download: playwright.Download; + finished: boolean; + outputFile: string; +}; + +type ConsoleLogEntry = { + type: 'console'; + wallTime: number; + message: ConsoleMessage; +}; + +type DownloadStartLogEntry = { + type: 'download-start'; + wallTime: number; + download: Download; +}; + +type DownloadFinishLogEntry = { + type: 'download-finish'; + wallTime: number; + download: Download; +}; + +type RequestLogEntry = { + type: 'request'; + wallTime: number; + request: playwright.Request; +}; + +type EventEntry = ConsoleLogEntry | DownloadStartLogEntry | DownloadFinishLogEntry | RequestLogEntry; + + +export type TabHeader = { title: string; + url: string; + current: boolean; +}; + +export type TabSnapshot = { ariaSnapshot: string; ariaSnapshotDiff?: string; modalStates: ModalState[]; - consoleMessages: ConsoleMessage[]; - downloads: { download: playwright.Download, finished: boolean, outputFile: string }[]; + events: EventEntry[]; }; export class Tab extends EventEmitter { readonly context: Context; readonly page: Page; - private _lastTitle = 'about:blank'; + private _lastHeader: TabHeader = { title: 'about:blank', url: 'about:blank', current: false }; private _consoleMessages: ConsoleMessage[] = []; - private _recentConsoleMessages: ConsoleMessage[] = []; + private _downloads: Download[] = []; private _requests: Set = new Set(); private _onPageClose: (tab: Tab) => void; private _modalStates: ModalState[] = []; - private _downloads: { download: playwright.Download, finished: boolean, outputFile: string }[] = []; - // TODO: split into Tab and TabHeader private _initializedPromise: Promise; private _needsFullSnapshot = false; + private _eventEntries: EventEntry[] = []; + private _recentEventEntries: EventEntry[] = []; constructor(context: Context, page: playwright.Page, onPageClose: (tab: Tab) => void) { super(); @@ -68,7 +104,7 @@ export class Tab extends EventEmitter { this._onPageClose = onPageClose; page.on('console', event => this._handleConsoleMessage(messageToConsoleMessage(event))); page.on('pageerror', error => this._handleConsoleMessage(pageErrorToConsoleMessage(error))); - page.on('request', request => this._requests.add(request)); + page.on('request', request => this._handleRequest(request)); page.on('close', () => this._onClose()); page.on('filechooser', chooser => { this.setModalState({ @@ -145,22 +181,36 @@ export class Tab extends EventEmitter { const entry = { download, finished: false, - outputFile: await this.context.outputFile(download.suggestedFilename(), { origin: 'web', reason: 'Saving download' }) + outputFile: await this.context.outputFile(download.suggestedFilename(), { origin: 'web', title: 'Saving download' }) }; this._downloads.push(entry); + this._addLogEntry({ type: 'download-start', wallTime: Date.now(), download: entry }); await download.saveAs(entry.outputFile); entry.finished = true; + this._addLogEntry({ type: 'download-finish', wallTime: Date.now(), download: entry }); } private _clearCollectedArtifacts() { this._consoleMessages.length = 0; - this._recentConsoleMessages.length = 0; + this._downloads.length = 0; this._requests.clear(); + this._eventEntries.length = 0; + this._recentEventEntries.length = 0; + } + + private _handleRequest(request: playwright.Request) { + this._requests.add(request); + this._addLogEntry({ type: 'request', wallTime: Date.now(), request }); } private _handleConsoleMessage(message: ConsoleMessage) { this._consoleMessages.push(message); - this._recentConsoleMessages.push(message); + this._addLogEntry({ type: 'console', wallTime: Date.now(), message }); + } + + private _addLogEntry(entry: EventEntry) { + this._eventEntries.push(entry); + this._recentEventEntries.push(entry); } private _onClose() { @@ -168,15 +218,18 @@ export class Tab extends EventEmitter { this._onPageClose(this); } - async updateTitle() { + async headerSnapshot(): Promise { + let title: string | undefined; await this._raceAgainstModalStates(async () => { - this._lastTitle = await callOnPageNoTrace(this.page, page => page.title()); + title = await callOnPageNoTrace(this.page, page => page.title()); }); + if (this._lastHeader.title !== title || this._lastHeader.url !== this.page.url() || this._lastHeader.current !== this.isCurrentTab()) { + this._lastHeader = { title: title ?? '', url: this.page.url(), current: this.isCurrentTab() }; + return { ...this._lastHeader, changed: true }; + } + return { ...this._lastHeader, changed: false }; } - lastTitle(): string { - return this._lastTitle; - } isCurrentTab(): boolean { return this === this.context.currentTab(); @@ -225,36 +278,31 @@ export class Tab extends EventEmitter { return this._requests; } - async captureSnapshot(includePageSnapshot: boolean): Promise { + async captureSnapshot(): Promise { await this._initializedPromise; let tabSnapshot: TabSnapshot | undefined; const modalStates = await this._raceAgainstModalStates(async () => { - const snapshot = includePageSnapshot ? await this.page._snapshotForAI({ track: 'response' }) : { full: '', incremental: '' }; + const snapshot = await this.page._snapshotForAI({ track: 'response' }); tabSnapshot = { - url: this.page.url(), - title: await this.page.title(), ariaSnapshot: snapshot.full, ariaSnapshotDiff: this._needsFullSnapshot ? undefined : snapshot.incremental, modalStates: [], - consoleMessages: [], - downloads: this._downloads, + events: [] }; }); if (tabSnapshot) { - // Assign console message late so that we did not lose any to modal state. - tabSnapshot.consoleMessages = this._recentConsoleMessages.filter(message => shouldIncludeMessage(this.context.config.console.level, message.type)); - this._recentConsoleMessages = []; + tabSnapshot.events = this._recentEventEntries; + this._recentEventEntries = []; } + // If we failed to capture a snapshot this time, make sure we do a full one next time, // to avoid reporting deltas against un-reported snapshot. this._needsFullSnapshot = !tabSnapshot; return tabSnapshot ?? { - url: this.page.url(), - title: '', ariaSnapshot: '', + ariaSnapshotDiff: '', modalStates, - consoleMessages: [], - downloads: [], + events: [], }; } @@ -358,7 +406,7 @@ type ConsoleMessageType = ReturnType; type ConsoleMessageLevel = 'error' | 'warning' | 'info' | 'debug'; const consoleMessageLevels: ConsoleMessageLevel[] = ['error', 'warning', 'info', 'debug']; -function shouldIncludeMessage(thresholdLevel: ConsoleMessageLevel, type: ConsoleMessageType): boolean { +export function shouldIncludeMessage(thresholdLevel: ConsoleMessageLevel, type: ConsoleMessageType): boolean { const messageLevel = consoleLevelForMessageType(type); return consoleMessageLevels.indexOf(messageLevel) <= consoleMessageLevels.indexOf(thresholdLevel); } diff --git a/packages/playwright/src/mcp/browser/tools/common.ts b/packages/playwright/src/mcp/browser/tools/common.ts index 9c42dad9bef5b..a834e10ebd41d 100644 --- a/packages/playwright/src/mcp/browser/tools/common.ts +++ b/packages/playwright/src/mcp/browser/tools/common.ts @@ -16,6 +16,7 @@ import { z } from 'playwright-core/lib/mcpBundle'; import { defineTabTool, defineTool } from './tool'; +import { renderTabsMarkdown } from '../response'; const close = defineTool({ capability: 'core', @@ -30,7 +31,8 @@ const close = defineTool({ handle: async (context, params, response) => { await context.closeBrowserContext(); - response.setIncludeTabs(); + const result = renderTabsMarkdown([]); + response.addTextResult(result.join('\n')); response.addCode(`await page.close()`); }, }); diff --git a/packages/playwright/src/mcp/browser/tools/console.ts b/packages/playwright/src/mcp/browser/tools/console.ts index 6643da31771b8..cc9fe13e6ef81 100644 --- a/packages/playwright/src/mcp/browser/tools/console.ts +++ b/packages/playwright/src/mcp/browser/tools/console.ts @@ -25,12 +25,14 @@ const console = defineTabTool({ description: 'Returns all console messages', inputSchema: z.object({ level: z.enum(['error', 'warning', 'info', 'debug']).default('info').describe('Level of the console messages to return. Each level includes the messages of more severe levels. Defaults to "info".'), + filename: z.string().optional().describe('Filename to save the console messages to. If not provided, messages are returned as text.'), }), type: 'readOnly', }, handle: async (tab, params, response) => { const messages = await tab.consoleMessages(params.level); - messages.map(message => response.addResult(message.toString())); + const text = messages.map(message => message.toString()).join('\n'); + await response.addResult({ text, suggestedFilename: params.filename }); }, }); diff --git a/packages/playwright/src/mcp/browser/tools/evaluate.ts b/packages/playwright/src/mcp/browser/tools/evaluate.ts index 4fcde54056873..e85f03582f325 100644 --- a/packages/playwright/src/mcp/browser/tools/evaluate.ts +++ b/packages/playwright/src/mcp/browser/tools/evaluate.ts @@ -25,6 +25,7 @@ const evaluateSchema = z.object({ function: z.string().describe('() => { /* code */ } or (element) => { /* code */ } when element is provided'), element: z.string().optional().describe('Human-readable element description used to obtain permission to interact with the element'), ref: z.string().optional().describe('Exact target element reference from the page snapshot'), + filename: z.string().optional().describe('Filename to save the result to. If not provided, result is returned as JSON string.'), }); const evaluate = defineTabTool({ @@ -51,7 +52,8 @@ const evaluate = defineTabTool({ await tab.waitForCompletion(async () => { const receiver = locator?.locator ?? tab.page; const result = await receiver._evaluateFunction(params.function); - response.addResult(JSON.stringify(result, null, 2) || 'undefined'); + const text = JSON.stringify(result, null, 2) || 'undefined'; + await response.addResult({ text, suggestedFilename: params.filename }); }); }, }); diff --git a/packages/playwright/src/mcp/browser/tools/install.ts b/packages/playwright/src/mcp/browser/tools/install.ts index 8cbc7ea44612b..23277b0f886ed 100644 --- a/packages/playwright/src/mcp/browser/tools/install.ts +++ b/packages/playwright/src/mcp/browser/tools/install.ts @@ -19,6 +19,7 @@ import path from 'path'; import { z } from 'playwright-core/lib/mcpBundle'; import { defineTool } from './tool'; +import { renderTabsMarkdown } from '../response'; const install = defineTool({ capability: 'core-install', @@ -47,7 +48,9 @@ const install = defineTool({ reject(new Error(`Failed to install browser: ${output.join('')}`)); }); }); - response.setIncludeTabs(); + const tabHeaders = await Promise.all(context.tabs().map(tab => tab.headerSnapshot())); + const result = renderTabsMarkdown(tabHeaders); + response.addTextResult(result.join('\n')); }, }); diff --git a/packages/playwright/src/mcp/browser/tools/network.ts b/packages/playwright/src/mcp/browser/tools/network.ts index 13e5fb3a5e9c7..cf6fb952fa339 100644 --- a/packages/playwright/src/mcp/browser/tools/network.ts +++ b/packages/playwright/src/mcp/browser/tools/network.ts @@ -29,17 +29,20 @@ const requests = defineTabTool({ description: 'Returns all network requests since loading the page', inputSchema: z.object({ includeStatic: z.boolean().default(false).describe('Whether to include successful static resources like images, fonts, scripts, etc. Defaults to false.'), + filename: z.string().optional().describe('Filename to save the network requests to. If not provided, requests are returned as text.'), }), type: 'readOnly', }, handle: async (tab, params, response) => { const requests = await tab.requests(); + const text: string[] = []; for (const request of requests) { const rendered = await renderRequest(request, params.includeStatic); if (rendered) - response.addResult(rendered); + text.push(rendered); } + await response.addResult({ text: text.join('\n'), suggestedFilename: params.filename }); }, }); diff --git a/packages/playwright/src/mcp/browser/tools/pdf.ts b/packages/playwright/src/mcp/browser/tools/pdf.ts index 1afd9a36f8f10..1cfa79f02302f 100644 --- a/packages/playwright/src/mcp/browser/tools/pdf.ts +++ b/packages/playwright/src/mcp/browser/tools/pdf.ts @@ -36,9 +36,10 @@ const pdf = defineTabTool({ }, handle: async (tab, params, response) => { - const fileName = await response.addFile(params.filename ?? dateAsFileName('pdf'), { origin: 'llm', reason: 'Page saved as PDF' }); - response.addCode(`await page.pdf(${formatObject({ path: fileName })});`); - await tab.page.pdf({ path: fileName }); + const data = await tab.page.pdf(); + const suggestedFilename = params.filename ?? dateAsFileName('pdf'); + await response.addResult({ data, title: 'Page as pdf', suggestedFilename }); + response.addCode(`await page.pdf(${formatObject({ path: suggestedFilename })});`); }, }); diff --git a/packages/playwright/src/mcp/browser/tools/runCode.ts b/packages/playwright/src/mcp/browser/tools/runCode.ts index 537ffdb06444c..75ea8b584400c 100644 --- a/packages/playwright/src/mcp/browser/tools/runCode.ts +++ b/packages/playwright/src/mcp/browser/tools/runCode.ts @@ -23,6 +23,7 @@ import { defineTabTool } from './tool'; const codeSchema = z.object({ code: z.string().describe(`A JavaScript function containing Playwright code to execute. It will be invoked with a single argument, page, which you can use for any page interaction. For example: \`async (page) => { await page.getByRole('button', { name: 'Submit' }).click(); return await page.title(); }\``), + filename: z.string().optional().describe('Filename to save the result to. If not provided, result is returned as JSON string.'), }); const runCode = defineTabTool({ @@ -56,7 +57,7 @@ const runCode = defineTabTool({ await vm.runInContext(snippet, context); const result = await __end__; if (typeof result === 'string') - response.addResult(result); + await response.addResult({ text: result, suggestedFilename: params.filename }); }); }, }); diff --git a/packages/playwright/src/mcp/browser/tools/screenshot.ts b/packages/playwright/src/mcp/browser/tools/screenshot.ts index c85a6ec4fd34d..2403831f5d7d2 100644 --- a/packages/playwright/src/mcp/browser/tools/screenshot.ts +++ b/packages/playwright/src/mcp/browser/tools/screenshot.ts @@ -14,9 +14,7 @@ * limitations under the License. */ -import fs from 'fs'; - -import { mkdirIfNeeded, scaleImageToSize } from 'playwright-core/lib/utils'; +import { scaleImageToSize } from 'playwright-core/lib/utils'; import { jpegjs, PNG } from 'playwright-core/lib/utilsBundle'; import { formatObject } from 'playwright-core/lib/utils'; @@ -60,25 +58,22 @@ const screenshot = defineTabTool({ const isElementScreenshot = params.element && params.ref; const screenshotTarget = isElementScreenshot ? params.element : (params.fullPage ? 'full page' : 'viewport'); - const fileName = await response.addFile(params.filename || dateAsFileName(fileType), { origin: 'llm', reason: `Screenshot of ${screenshotTarget}` }); - response.addCode(`// Screenshot ${screenshotTarget} and save it as ${fileName}`); - - // Only get snapshot when element screenshot is needed const ref = params.ref ? await tab.refLocator({ element: params.element || '', ref: params.ref }) : null; + const data = ref ? await ref.locator.screenshot(options) : await tab.page.screenshot(options); + const fileName = params.filename || dateAsFileName(fileType); + + response.addCode(`// Screenshot ${screenshotTarget} and save it as ${fileName}`); if (ref) - response.addCode(`await page.${ref.resolved}.screenshot(${formatObject(options)});`); + response.addCode(`await page.${ref.resolved}.screenshot(${formatObject({ ...options, path: fileName })});`); else - response.addCode(`await page.screenshot(${formatObject(options)});`); - - const buffer = ref ? await ref.locator.screenshot(options) : await tab.page.screenshot(options); + response.addCode(`await page.screenshot(${formatObject({ ...options, path: fileName })});`); - await mkdirIfNeeded(fileName); - await fs.promises.writeFile(fileName, buffer); + await response.addResult({ data, title: `Screenshot of ${screenshotTarget}`, suggestedFilename: fileName }); response.addImage({ contentType: fileType === 'png' ? 'image/png' : 'image/jpeg', - data: scaleImageToFitMessage(buffer, fileType) + data: scaleImageToFitMessage(data, fileType) }); } }); diff --git a/packages/playwright/src/mcp/browser/tools/snapshot.ts b/packages/playwright/src/mcp/browser/tools/snapshot.ts index ffed6cd20ae14..9aa57388bba28 100644 --- a/packages/playwright/src/mcp/browser/tools/snapshot.ts +++ b/packages/playwright/src/mcp/browser/tools/snapshot.ts @@ -14,8 +14,6 @@ * limitations under the License. */ -import fs from 'fs'; - import { z } from 'playwright-core/lib/mcpBundle'; import { formatObject } from 'playwright-core/lib/utils'; @@ -35,14 +33,7 @@ const snapshot = defineTool({ handle: async (context, params, response) => { await context.ensureTab(); - response.setIncludeFullSnapshot(); - if (params.filename) { - await response.finish(); - const renderedResponse = response.render(); - const fileName = await response.addFile(params.filename, { origin: 'llm', reason: 'Saved snapshot' }); - await fs.promises.writeFile(fileName, renderedResponse.asText()); - response.setIncludeMetaOnly(); - } + response.setIncludeFullSnapshot(params.filename); }, }); @@ -183,7 +174,7 @@ const pickLocator = defineTabTool({ handle: async (tab, params, response) => { const { resolved } = await tab.refLocator(params); - response.addResult(resolved); + response.addTextResult(resolved); }, }); diff --git a/packages/playwright/src/mcp/browser/tools/tabs.ts b/packages/playwright/src/mcp/browser/tools/tabs.ts index 6e8cc1e30e911..ddde4f243b77b 100644 --- a/packages/playwright/src/mcp/browser/tools/tabs.ts +++ b/packages/playwright/src/mcp/browser/tools/tabs.ts @@ -16,6 +16,7 @@ import { z } from 'playwright-core/lib/mcpBundle'; import { defineTool } from './tool'; +import { renderTabsMarkdown } from '../response'; const browserTabs = defineTool({ capability: 'core-tabs', @@ -35,27 +36,26 @@ const browserTabs = defineTool({ switch (params.action) { case 'list': { await context.ensureTab(); - response.setIncludeTabs(); - return; + break; } case 'new': { await context.newTab(); - response.setIncludeTabs(); - return; + break; } case 'close': { await context.closeTab(params.index); - response.setIncludeFullSnapshot(); - return; + break; } case 'select': { if (params.index === undefined) throw new Error('Tab index is required'); await context.selectTab(params.index); - response.setIncludeFullSnapshot(); - return; + break; } } + const tabHeaders = await Promise.all(context.tabs().map(tab => tab.headerSnapshot())); + const result = renderTabsMarkdown(tabHeaders); + response.addTextResult(result.join('\n')); }, }); diff --git a/packages/playwright/src/mcp/browser/tools/tool.ts b/packages/playwright/src/mcp/browser/tools/tool.ts index a8452b882cc61..306b711f2bacc 100644 --- a/packages/playwright/src/mcp/browser/tools/tool.ts +++ b/packages/playwright/src/mcp/browser/tools/tool.ts @@ -61,15 +61,12 @@ export function defineTabTool(tool: TabTool): Too handle: async (context, params, response) => { const tab = await context.ensureTab(); const modalStates = tab.modalStates().map(state => state.type); - if (tool.clearsModalState && !modalStates.includes(tool.clearsModalState)) { - response.setIncludeModalStates(tab.modalStates()); + if (tool.clearsModalState && !modalStates.includes(tool.clearsModalState)) response.addError(`Error: The tool "${tool.schema.name}" can only be used when there is related modal state present.`); - } else if (!tool.clearsModalState && modalStates.length) { - response.setIncludeModalStates(tab.modalStates()); + else if (!tool.clearsModalState && modalStates.length) response.addError(`Error: Tool "${tool.schema.name}" does not handle the modal state.`); - } else { + else return tool.handle(tab, params, response); - } }, }; } diff --git a/packages/playwright/src/mcp/browser/tools/tracing.ts b/packages/playwright/src/mcp/browser/tools/tracing.ts index 27a82d0974e3d..041f950c90125 100644 --- a/packages/playwright/src/mcp/browser/tools/tracing.ts +++ b/packages/playwright/src/mcp/browser/tools/tracing.ts @@ -32,7 +32,7 @@ const tracingStart = defineTool({ handle: async (context, params, response) => { const browserContext = await context.ensureBrowserContext(); - const tracesDir = await context.outputFile(`traces`, { origin: 'code', reason: 'Collecting trace' }); + const tracesDir = await context.outputFile(`traces`, { origin: 'code', title: 'Collecting trace' }); const name = 'trace-' + Date.now(); await (browserContext.tracing as Tracing).start({ name, @@ -44,7 +44,7 @@ const tracingStart = defineTool({ - Network log: ${tracesDir}/${name}.network - Resources with content by sha1: ${tracesDir}/resources`; - response.addResult(`Tracing started, saving to ${tracesDir}.\n${traceLegend}`); + response.addTextResult(`Tracing started, saving to ${tracesDir}.\n${traceLegend}`); (browserContext.tracing as any)[traceLegendSymbol] = traceLegend; }, }); @@ -64,7 +64,7 @@ const tracingStop = defineTool({ const browserContext = await context.ensureBrowserContext(); await browserContext.tracing.stop(); const traceLegend = (browserContext.tracing as any)[traceLegendSymbol]; - response.addResult(`Tracing stopped.\n${traceLegend}`); + response.addTextResult(`Tracing stopped.\n${traceLegend}`); }, }); diff --git a/packages/playwright/src/mcp/browser/tools/verify.ts b/packages/playwright/src/mcp/browser/tools/verify.ts index f65a9119fc8cc..381aecb864b4a 100644 --- a/packages/playwright/src/mcp/browser/tools/verify.ts +++ b/packages/playwright/src/mcp/browser/tools/verify.ts @@ -40,7 +40,7 @@ const verifyElement = defineTabTool({ } response.addCode(`await expect(page.getByRole(${escapeWithQuotes(params.role)}, { name: ${escapeWithQuotes(params.accessibleName)} })).toBeVisible();`); - response.addResult('Done'); + response.addTextResult('Done'); }, }); @@ -64,7 +64,7 @@ const verifyText = defineTabTool({ } response.addCode(`await expect(page.getByText(${escapeWithQuotes(params.text)})).toBeVisible();`); - response.addResult('Done'); + response.addTextResult('Done'); }, }); @@ -98,7 +98,7 @@ const verifyList = defineTabTool({ ${itemTexts.map(t => ` - listitem: ${escapeWithQuotes(t, '"')}`).join('\n')} \``; response.addCode(`await expect(page.locator('body')).toMatchAriaSnapshot(${ariaSnapshot});`); - response.addResult('Done'); + response.addTextResult('Done'); }, }); @@ -136,7 +136,7 @@ const verifyValue = defineTabTool({ const matcher = value ? 'toBeChecked' : 'not.toBeChecked'; response.addCode(`await expect(${locatorSource}).${matcher}();`); } - response.addResult('Done'); + response.addTextResult('Done'); }, }); diff --git a/packages/playwright/src/mcp/browser/tools/wait.ts b/packages/playwright/src/mcp/browser/tools/wait.ts index 4a4e78a451b2f..41854ccd23010 100644 --- a/packages/playwright/src/mcp/browser/tools/wait.ts +++ b/packages/playwright/src/mcp/browser/tools/wait.ts @@ -55,7 +55,7 @@ const wait = defineTool({ await locator.waitFor({ state: 'visible' }); } - response.addResult(`Waited for ${params.text || params.textGone || params.time}`); + response.addTextResult(`Waited for ${params.text || params.textGone || params.time}`); response.setIncludeSnapshot(); }, }); diff --git a/packages/playwright/src/mcp/config.d.ts b/packages/playwright/src/mcp/config.d.ts index 823ccfb985f58..513d12dc30cf6 100644 --- a/packages/playwright/src/mcp/config.d.ts +++ b/packages/playwright/src/mcp/config.d.ts @@ -147,6 +147,11 @@ export type Config = { */ outputDir?: string; + /** + * Whether to save snapshots, console messages, network logs and other session logs to a file or to the standard output. Defaults to "stdout". + */ + outputMode?: 'file' | 'stdout'; + console?: { /** * The level of console messages to return. Each level includes the messages of more severe levels. Defaults to "info". @@ -193,7 +198,7 @@ export type Config = { * When taking snapshots for responses, specifies the mode to use. */ mode?: 'incremental' | 'full' | 'none'; - } + }; /** * Whether to allow file uploads from anywhere on the file system. diff --git a/packages/playwright/src/mcp/program.ts b/packages/playwright/src/mcp/program.ts index 77aee4edef8ce..452b6d6d18ec5 100644 --- a/packages/playwright/src/mcp/program.ts +++ b/packages/playwright/src/mcp/program.ts @@ -17,6 +17,7 @@ /* eslint-disable no-console */ import fs from 'fs'; +import path from 'path'; import { colors, ProgramOption } from 'playwright-core/lib/utilsBundle'; import { registry } from 'playwright-core/lib/server'; @@ -57,6 +58,7 @@ export function decorateCommand(command: Command, version: string) { .option('--image-responses ', 'whether to send image responses to the client. Can be "allow" or "omit", Defaults to "allow".', enumParser.bind(null, '--image-responses', ['allow', 'omit'])) .option('--no-sandbox', 'disable the sandbox for all process types that are normally sandboxed.') .option('--output-dir ', 'path to the directory for output files.') + .option('--output-mode ', 'whether to save snapshots, console messages, network logs to a file or to the standard output. Can be "file" or "stdout". Default is "stdout".', enumParser.bind(null, '--output-mode', ['file', 'stdout'])) .option('--port ', 'port to listen on for SSE transport.') .option('--proxy-bypass ', 'comma-separated domains to bypass proxy, for example ".com,chromium.org,.domain.com"') .option('--proxy-server ', 'specify proxy server, for example "http://myproxy:3128" or "socks5://myproxy:8080"') @@ -110,8 +112,11 @@ export function decorateCommand(command: Command, version: string) { } if (options.daemon) { - config.snapshot.mode = 'none'; + config.outputDir = path.join(process.cwd(), '.playwright-cli'); + config.outputMode = 'file'; config.codegen = 'none'; + config.snapshot.mode = 'full'; + const serverBackendFactory: mcpServer.ServerBackendFactory = { name: 'Playwright', nameInConfig: 'playwright-daemon', diff --git a/tests/library/__llm_cache__/library-agent-perform-perform-reports-error.json b/tests/library/__llm_cache__/library-agent-perform-perform-reports-error.json index 33bc8a034197e..c98268e020605 100644 --- a/tests/library/__llm_cache__/library-agent-perform-perform-reports-error.json +++ b/tests/library/__llm_cache__/library-agent-perform-perform-reports-error.json @@ -25,6 +25,33 @@ "output": 72 } }, + "8216d5c7cc9689d794987b980fb1c8068fdbcf52": { + "result": { + "role": "assistant", + "content": [ + { + "type": "text", + "text": "I can see the page snapshot shows only \"Wolf\" and \"Fox\" buttons, but no \"Rabbit\" button is visible. Let me check if there might be more content on the page by scrolling or if the Rabbit button appears in a different location." + }, + { + "type": "tool_call", + "name": "refuse_to_perform", + "arguments": { + "reason": "The Rabbit button is not present on the page. The current page only shows Wolf and Fox buttons. The requested Rabbit button cannot be found to click.", + "_is_done": true + }, + "id": "toolu_012R5KoZ277NbYLkeFaCsC95" + } + ], + "stopReason": { + "code": "ok" + } + }, + "usage": { + "input": 3191, + "output": 159 + } + }, "889157cd583283667b9fd5d9add230316106c3a4": { "result": { "role": "assistant", diff --git a/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat0.json b/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat0.json deleted file mode 100644 index 45ca393ea53dc..0000000000000 --- a/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat0.json +++ /dev/null @@ -1,518 +0,0 @@ -{ - "c6e2436f30fc74b09258d515ca66258219b1d296": { - "result": { - "role": "assistant", - "content": [ - { - "type": "text", - "text": "I'll help you navigate to the page, fill out the order form, and report back the total price and delivery information." - } - ] - }, - "usage": { - "input": 4515, - "output": 119 - } - }, - "208d528f3788e85bbab2a1bf6914985fac1d835a": { - "result": { - "role": "assistant", - "content": [ - { - "type": "text", - "text": "Navigate to the form page to start ordering items", - "copilotToolCallId": "toolu_vrtx_01QJb24SaGmdDk2LSYhVwKFq" - }, - { - "type": "tool_call", - "name": "browser_navigate", - "arguments": { - "url": "http://localhost:/evals/fill-form.html" - }, - "id": "toolu_vrtx_01QJb24SaGmdDk2LSYhVwKFq" - } - ] - }, - "usage": { - "input": 4571, - "output": 92 - } - }, - "f92997bbe28f81024382efcd1b175fabfb9a348b": { - "result": { - "role": "assistant", - "content": [ - { - "type": "text", - "text": "Select Table as the first product", - "copilotToolCallId": "toolu_vrtx_01Wy78XiFT1hA7s364vd74ML" - }, - { - "type": "tool_call", - "name": "browser_select_option", - "arguments": { - "element": "Product Type dropdown", - "ref": "e7", - "values": [ - "Table" - ] - }, - "id": "toolu_vrtx_01Wy78XiFT1hA7s364vd74ML" - } - ] - }, - "usage": { - "input": 5284, - "output": 116 - } - }, - "224aef3f51ccb59d9003a65fe28f8abbac540d51": { - "result": { - "role": "assistant", - "content": [ - { - "type": "text", - "text": "Select Blue as the color for the table", - "copilotToolCallId": "toolu_vrtx_01NJ8jnjNYFnyqzKhfkPjpzd" - }, - { - "type": "tool_call", - "name": "browser_select_option", - "arguments": { - "element": "Color dropdown", - "ref": "e10", - "values": [ - "Blue" - ] - }, - "id": "toolu_vrtx_01NJ8jnjNYFnyqzKhfkPjpzd" - } - ] - }, - "usage": { - "input": 5547, - "output": 117 - } - }, - "38d64bd91fc32d1fe51d2cf641a6025d58b744fa": { - "result": { - "role": "assistant", - "content": [ - { - "type": "text", - "text": "Fill in the name field with John Doe", - "copilotToolCallId": "toolu_vrtx_01T5ruwZRiLm3nLtFWkc3Gke" - }, - { - "type": "tool_call", - "name": "browser_type", - "arguments": { - "element": "Full Name textbox", - "ref": "e13", - "text": "John Doe" - }, - "id": "toolu_vrtx_01T5ruwZRiLm3nLtFWkc3Gke" - } - ] - }, - "usage": { - "input": 5813, - "output": 118 - } - }, - "6c8756bb54afad695e4e4ea901c27fdeffe93d8b": { - "result": { - "role": "assistant", - "content": [ - { - "type": "text", - "text": "Fill in the email field", - "copilotToolCallId": "toolu_vrtx_01AZBjdBNyC8htApAJx52BZj" - }, - { - "type": "tool_call", - "name": "browser_type", - "arguments": { - "element": "Email Address textbox", - "ref": "e16", - "text": "john@doe.me" - }, - "id": "toolu_vrtx_01AZBjdBNyC8htApAJx52BZj" - } - ] - }, - "usage": { - "input": 5977, - "output": 116 - } - }, - "cf1c146f769b391047d65b77367c7452b0f86517": { - "result": { - "role": "assistant", - "content": [ - { - "type": "text", - "text": "Fill in the street address field", - "copilotToolCallId": "toolu_vrtx_0153HGAJZuNiEAZUrqa8hi5E" - }, - { - "type": "tool_call", - "name": "browser_type", - "arguments": { - "element": "Street Address textbox", - "ref": "e19", - "text": "123 Main St" - }, - "id": "toolu_vrtx_0153HGAJZuNiEAZUrqa8hi5E" - } - ] - }, - "usage": { - "input": 6141, - "output": 116 - } - }, - "18a4b0b23866628c41c26840c0453cfb5745abe4": { - "result": { - "role": "assistant", - "content": [ - { - "type": "text", - "text": "Fill in the city field with Anytown", - "copilotToolCallId": "toolu_vrtx_012buxK7mWWFe97aRhG4oAnQ" - }, - { - "type": "tool_call", - "name": "browser_type", - "arguments": { - "element": "City textbox", - "ref": "e22", - "text": "Anytown" - }, - "id": "toolu_vrtx_012buxK7mWWFe97aRhG4oAnQ" - } - ] - }, - "usage": { - "input": 6304, - "output": 117 - } - }, - "141ace775dfba69a215f874da99dc98394346d83": { - "result": { - "role": "assistant", - "content": [ - { - "type": "text", - "text": "Fill in the state field with XYZ", - "copilotToolCallId": "toolu_vrtx_01S8BLUfWVcRZJxoT2Pxq58c" - }, - { - "type": "tool_call", - "name": "browser_type", - "arguments": { - "element": "State/Province textbox", - "ref": "e25", - "text": "XYZ" - }, - "id": "toolu_vrtx_01S8BLUfWVcRZJxoT2Pxq58c" - } - ] - }, - "usage": { - "input": 6466, - "output": 119 - } - }, - "d3d9754688e89faab5c13025f900fd5d148f70a0": { - "result": { - "role": "assistant", - "content": [ - { - "type": "text", - "text": "Fill in the ZIP code field", - "copilotToolCallId": "toolu_vrtx_016vGQAUKWJCFgoXs6SnXRBN" - }, - { - "type": "tool_call", - "name": "browser_type", - "arguments": { - "element": "ZIP/Postal Code textbox", - "ref": "e28", - "text": "12345" - }, - "id": "toolu_vrtx_016vGQAUKWJCFgoXs6SnXRBN" - } - ] - }, - "usage": { - "input": 6632, - "output": 117 - } - }, - "6f7c8ef402aa3206eeab2817aadf67757c99dd1f": { - "result": { - "role": "assistant", - "content": [ - { - "type": "text", - "text": "Click the Order button to submit the first order (blue table)", - "copilotToolCallId": "toolu_vrtx_01GeRLXFVbGYPe2CX88GrAD1" - }, - { - "type": "tool_call", - "name": "browser_click", - "arguments": { - "element": "Order button", - "ref": "e29" - }, - "id": "toolu_vrtx_01GeRLXFVbGYPe2CX88GrAD1" - } - ] - }, - "usage": { - "input": 6797, - "output": 100 - } - }, - "87d6f9626d8ef40fd4df2a6b5cc87d0a084380db": { - "result": { - "role": "assistant", - "content": [ - { - "type": "text", - "text": "Accept the confirmation dialog for the blue table order", - "copilotToolCallId": "toolu_vrtx_01Gze4ViLhq8iKT3cYxkEswg" - }, - { - "type": "tool_call", - "name": "browser_handle_dialog", - "arguments": { - "accept": true - }, - "id": "toolu_vrtx_01Gze4ViLhq8iKT3cYxkEswg" - } - ] - }, - "usage": { - "input": 6972, - "output": 80 - } - }, - "c85197053f027c8b5644c0dfd452b7860d4f3f9d": { - "result": { - "role": "assistant", - "content": [ - { - "type": "text", - "text": "Click Place Another Order to order the second item (green desk)", - "copilotToolCallId": "toolu_vrtx_016uofKjtBYpbZ1syvkZJz5T" - }, - { - "type": "tool_call", - "name": "browser_click", - "arguments": { - "element": "Place Another Order button", - "ref": "e58" - }, - "id": "toolu_vrtx_016uofKjtBYpbZ1syvkZJz5T" - } - ] - }, - "usage": { - "input": 7859, - "output": 102 - } - }, - "558852b61b70243bc9c07b74d9604eb44159ac3b": { - "result": { - "role": "assistant", - "content": [ - { - "type": "text", - "text": "Select Desk as the second product", - "copilotToolCallId": "toolu_vrtx_01RDGz26dSJMwZaQHo9Y9UPH" - }, - { - "type": "tool_call", - "name": "browser_select_option", - "arguments": { - "element": "Product Type dropdown", - "ref": "e7", - "values": [ - "Desk" - ] - }, - "id": "toolu_vrtx_01RDGz26dSJMwZaQHo9Y9UPH" - } - ] - }, - "usage": { - "input": 8581, - "output": 118 - } - }, - "9074d89eba6dd89300b2705017c4581e3a46688d": { - "result": { - "role": "assistant", - "content": [ - { - "type": "text", - "text": "Select Green as the color for the desk", - "copilotToolCallId": "toolu_vrtx_01QKP2UHW1cFDyC2LMczLaZU" - }, - { - "type": "tool_call", - "name": "browser_select_option", - "arguments": { - "element": "Color dropdown", - "ref": "e10", - "values": [ - "Green" - ] - }, - "id": "toolu_vrtx_01QKP2UHW1cFDyC2LMczLaZU" - } - ] - }, - "usage": { - "input": 8847, - "output": 117 - } - }, - "51a702cdde8dd0004e8c6bd30ece543f8145edf7": { - "result": { - "role": "assistant", - "content": [ - { - "type": "text", - "text": "Fill in all the form fields with the customer information", - "copilotToolCallId": "toolu_vrtx_01QvTrbQvU4wCUPn4dTMhnMZ" - }, - { - "type": "tool_call", - "name": "browser_fill_form", - "arguments": { - "fields": [ - { - "name": "Full Name", - "type": "textbox", - "ref": "e13", - "value": "John Doe" - }, - { - "name": "Email Address", - "type": "textbox", - "ref": "e16", - "value": "john@doe.me" - }, - { - "name": "Street Address", - "type": "textbox", - "ref": "e19", - "value": "123 Main St" - }, - { - "name": "City", - "type": "textbox", - "ref": "e22", - "value": "Anytown" - }, - { - "name": "State/Province", - "type": "textbox", - "ref": "e25", - "value": "XYZ" - }, - { - "name": "ZIP/Postal Code", - "type": "textbox", - "ref": "e28", - "value": "12345" - } - ] - }, - "id": "toolu_vrtx_01QvTrbQvU4wCUPn4dTMhnMZ" - } - ] - }, - "usage": { - "input": 9113, - "output": 274 - } - }, - "650b847697eb61f2adca523992b65f63ae77e4f0": { - "result": { - "role": "assistant", - "content": [ - { - "type": "text", - "text": "Click the Order button to submit the second order (green desk)", - "copilotToolCallId": "toolu_vrtx_019YvTNKEuB34rhe5Tonqikz" - }, - { - "type": "tool_call", - "name": "browser_click", - "arguments": { - "element": "Order button", - "ref": "e29" - }, - "id": "toolu_vrtx_019YvTNKEuB34rhe5Tonqikz" - } - ] - }, - "usage": { - "input": 9565, - "output": 100 - } - }, - "bb66b168e193d00bbfe4c65cff32cb0648ceb9cd": { - "result": { - "role": "assistant", - "content": [ - { - "type": "text", - "text": "Accept the confirmation dialog for the green desk order", - "copilotToolCallId": "toolu_vrtx_0122xueFnLyirRBuERQXnvJc" - }, - { - "type": "tool_call", - "name": "browser_handle_dialog", - "arguments": { - "accept": true - }, - "id": "toolu_vrtx_0122xueFnLyirRBuERQXnvJc" - } - ] - }, - "usage": { - "input": 9740, - "output": 80 - } - }, - "9c742682c192e1bea527cdf10420c6b8b5d8e6a7": { - "result": { - "role": "assistant", - "content": [ - { - "type": "text", - "text": "Report the combined total price and maximum delivery days for both orders: blue table ($353.98) + green desk ($299.98) = $653.96 total, and both have 3 days estimated delivery", - "copilotToolCallId": "toolu_vrtx_014LwrYe1UWGYPtXRDZzaLMv" - }, - { - "type": "tool_call", - "name": "report_result", - "arguments": { - "price": 653.96, - "deliveryDays": 3 - }, - "id": "toolu_vrtx_014LwrYe1UWGYPtXRDZzaLMv" - } - ] - }, - "usage": { - "input": 10628, - "output": 136 - } - } -} \ No newline at end of file diff --git a/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat1.json b/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat1.json deleted file mode 100644 index 498673483671e..0000000000000 --- a/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat1.json +++ /dev/null @@ -1,423 +0,0 @@ -{ - "c6e2436f30fc74b09258d515ca66258219b1d296": { - "result": { - "role": "assistant", - "content": [ - { - "type": "text", - "text": "I'll help you complete this task. Let me start by navigating to the form and then filling it out with the specified details." - } - ] - }, - "usage": { - "input": 4515, - "output": 116 - } - }, - "6b58aecc88682e5cf9ac78c5db916972bf2aa632": { - "result": { - "role": "assistant", - "content": [ - { - "type": "text", - "text": "Navigate to the form page", - "copilotToolCallId": "toolu_vrtx_01KXNJS2AF6Lut6PuyWe3gbp" - }, - { - "type": "tool_call", - "name": "browser_navigate", - "arguments": { - "url": "http://localhost:/evals/fill-form.html" - }, - "id": "toolu_vrtx_01KXNJS2AF6Lut6PuyWe3gbp" - } - ] - }, - "usage": { - "input": 4573, - "output": 88 - } - }, - "39728354cc6d356e4d5dc072fe8519aaf1cd48fb": { - "result": { - "role": "assistant", - "content": [ - { - "type": "text", - "text": "Select Table as the product type", - "copilotToolCallId": "toolu_vrtx_01TiqCpSz8NCbxczVXWJKp7K" - }, - { - "type": "tool_call", - "name": "browser_select_option", - "arguments": { - "element": "Product Type combobox", - "ref": "e7", - "values": [ - "Table" - ] - }, - "id": "toolu_vrtx_01TiqCpSz8NCbxczVXWJKp7K" - } - ] - }, - "usage": { - "input": 5282, - "output": 118 - } - }, - "47ef8fec29b9227dfaf29a5bcdda43d0f2ac1f91": { - "result": { - "role": "assistant", - "content": [ - { - "type": "text", - "text": "Select Blue as the color", - "copilotToolCallId": "toolu_vrtx_011NPSqEP7jWBGC7gawFgsiQ" - }, - { - "type": "tool_call", - "name": "browser_select_option", - "arguments": { - "element": "Color combobox", - "ref": "e10", - "values": [ - "Blue" - ] - }, - "id": "toolu_vrtx_011NPSqEP7jWBGC7gawFgsiQ" - } - ] - }, - "usage": { - "input": 5547, - "output": 116 - } - }, - "176626204d6ba6bec6660c2fcaa67d8d6abe8a8f": { - "result": { - "role": "assistant", - "content": [ - { - "type": "text", - "text": "Fill in the customer details for the first order", - "copilotToolCallId": "toolu_vrtx_01CfyfDD5bEBhKQy3st2iuU8" - }, - { - "type": "tool_call", - "name": "browser_fill_form", - "arguments": { - "fields": [ - { - "name": "Full Name", - "type": "textbox", - "ref": "e13", - "value": "John Doe" - }, - { - "name": "Email Address", - "type": "textbox", - "ref": "e16", - "value": "john@doe.me" - }, - { - "name": "Street Address", - "type": "textbox", - "ref": "e19", - "value": "123 Main St" - }, - { - "name": "City", - "type": "textbox", - "ref": "e22", - "value": "Anytown" - }, - { - "name": "State/Province", - "type": "textbox", - "ref": "e25", - "value": "XYZ state" - }, - { - "name": "ZIP/Postal Code", - "type": "textbox", - "ref": "e28", - "value": "12345" - } - ] - }, - "id": "toolu_vrtx_01CfyfDD5bEBhKQy3st2iuU8" - } - ] - }, - "usage": { - "input": 5812, - "output": 346 - } - }, - "62da0871a0e720786fdb286a6276b211cf5be6ba": { - "result": { - "role": "assistant", - "content": [ - { - "type": "text", - "text": "Click the Order button to submit the first order (blue table)", - "copilotToolCallId": "toolu_vrtx_01LED3srq9PN9dh7CsxLwbMS" - }, - { - "type": "tool_call", - "name": "browser_click", - "arguments": { - "element": "Order button", - "ref": "e29" - }, - "id": "toolu_vrtx_01LED3srq9PN9dh7CsxLwbMS" - } - ] - }, - "usage": { - "input": 6265, - "output": 100 - } - }, - "0eaa90ef5e1a7a4b998555530a6bee89bf1c8edd": { - "result": { - "role": "assistant", - "content": [ - { - "type": "text", - "text": "Confirm the order for the blue table", - "copilotToolCallId": "toolu_vrtx_01F2ynTUAQiW8xu2ExsYZWFk" - }, - { - "type": "tool_call", - "name": "browser_handle_dialog", - "arguments": { - "accept": true - }, - "id": "toolu_vrtx_01F2ynTUAQiW8xu2ExsYZWFk" - } - ] - }, - "usage": { - "input": 6440, - "output": 79 - } - }, - "e0a73223c0bda0ff2bfb65c504ba2f60c28265f8": { - "result": { - "role": "assistant", - "content": [ - { - "type": "text", - "text": "Click the Place Another Order button to order the green desk", - "copilotToolCallId": "toolu_vrtx_01SzH72CYSm1nJfpo3ywV31H" - }, - { - "type": "tool_call", - "name": "browser_click", - "arguments": { - "element": "Place Another Order button", - "ref": "e58" - }, - "id": "toolu_vrtx_01SzH72CYSm1nJfpo3ywV31H" - } - ] - }, - "usage": { - "input": 7329, - "output": 101 - } - }, - "e2dfbbc04b8d1a4d5fdfe8518ae4c603c66056c5": { - "result": { - "role": "assistant", - "content": [ - { - "type": "text", - "text": "Select Desk as the product type for second order", - "copilotToolCallId": "toolu_vrtx_014spNiRnyJFG9S5XxdedDdq" - }, - { - "type": "tool_call", - "name": "browser_select_option", - "arguments": { - "element": "Product Type combobox", - "ref": "e7", - "values": [ - "Desk" - ] - }, - "id": "toolu_vrtx_014spNiRnyJFG9S5XxdedDdq" - } - ] - }, - "usage": { - "input": 8050, - "output": 123 - } - }, - "1cd9ece884fc13247bbabd1520434f0536c2e5ab": { - "result": { - "role": "assistant", - "content": [ - { - "type": "text", - "text": "Select Green as the color for the desk", - "copilotToolCallId": "toolu_vrtx_015a4ReJ8RSgxJgt1R7Ap6Ry" - }, - { - "type": "tool_call", - "name": "browser_select_option", - "arguments": { - "element": "Color combobox", - "ref": "e10", - "values": [ - "Green" - ] - }, - "id": "toolu_vrtx_015a4ReJ8RSgxJgt1R7Ap6Ry" - } - ] - }, - "usage": { - "input": 8321, - "output": 119 - } - }, - "dee9627bf01b277cb98d2bb4806cff6c9c44ac00": { - "result": { - "role": "assistant", - "content": [ - { - "type": "text", - "text": "Fill in the customer details for the second order", - "copilotToolCallId": "toolu_vrtx_01LNwyANy655VYsrtN4HqMv8" - }, - { - "type": "tool_call", - "name": "browser_fill_form", - "arguments": { - "fields": [ - { - "name": "Full Name", - "ref": "e13", - "type": "textbox", - "value": "John Doe" - }, - { - "name": "Email Address", - "ref": "e16", - "type": "textbox", - "value": "john@doe.me" - }, - { - "name": "Street Address", - "ref": "e19", - "type": "textbox", - "value": "123 Main St" - }, - { - "name": "City", - "ref": "e22", - "type": "textbox", - "value": "Anytown" - }, - { - "name": "State/Province", - "ref": "e25", - "type": "textbox", - "value": "XYZ state" - }, - { - "name": "ZIP/Postal Code", - "ref": "e28", - "type": "textbox", - "value": "12345" - } - ] - }, - "id": "toolu_vrtx_01LNwyANy655VYsrtN4HqMv8" - } - ] - }, - "usage": { - "input": 8589, - "output": 266 - } - }, - "df89bda38bd650cff3e4448a96c75f49e89307dc": { - "result": { - "role": "assistant", - "content": [ - { - "type": "text", - "text": "Click the Order button to submit the second order (green desk)", - "copilotToolCallId": "toolu_vrtx_01U89qrzqXpLUfh3H3RqGRvq" - }, - { - "type": "tool_call", - "name": "browser_click", - "arguments": { - "element": "Order button", - "ref": "e29" - }, - "id": "toolu_vrtx_01U89qrzqXpLUfh3H3RqGRvq" - } - ] - }, - "usage": { - "input": 9042, - "output": 100 - } - }, - "c0ae7023c9ffb0c07734dbde9a14115b31934127": { - "result": { - "role": "assistant", - "content": [ - { - "type": "text", - "text": "Confirm the order for the green desk", - "copilotToolCallId": "toolu_vrtx_01XSXSFXhqkeCVMxVmDY2Smn" - }, - { - "type": "tool_call", - "name": "browser_handle_dialog", - "arguments": { - "accept": true - }, - "id": "toolu_vrtx_01XSXSFXhqkeCVMxVmDY2Smn" - } - ] - }, - "usage": { - "input": 9217, - "output": 79 - } - }, - "f21c818657e9219c65e66dea2e83dfb49a576e18": { - "result": { - "role": "assistant", - "content": [ - { - "type": "text", - "text": "Report the total price for both orders and maximum delivery time", - "copilotToolCallId": "toolu_vrtx_014ZSZ3KTaH5tcv1mgsLoZ2u" - }, - { - "type": "tool_call", - "name": "report_result", - "arguments": { - "price": 653.96, - "deliveryDays": 3 - }, - "id": "toolu_vrtx_014ZSZ3KTaH5tcv1mgsLoZ2u" - } - ] - }, - "usage": { - "input": 10107, - "output": 102 - } - } -} \ No newline at end of file diff --git a/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat2.json b/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat2.json deleted file mode 100644 index e89529a282453..0000000000000 --- a/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat2.json +++ /dev/null @@ -1,423 +0,0 @@ -{ - "c6e2436f30fc74b09258d515ca66258219b1d296": { - "result": { - "role": "assistant", - "content": [ - { - "type": "text", - "text": "I'll navigate to the form and complete the order for you." - } - ] - }, - "usage": { - "input": 4515, - "output": 103 - } - }, - "056d6552f3ad9000456a3a8678731d2b5cba6578": { - "result": { - "role": "assistant", - "content": [ - { - "type": "text", - "text": "Navigate to the form page to start ordering items", - "copilotToolCallId": "toolu_vrtx_012ZazvgwjfAJc6AdPfD2Rg3" - }, - { - "type": "tool_call", - "name": "browser_navigate", - "arguments": { - "url": "http://localhost:/evals/fill-form.html" - }, - "id": "toolu_vrtx_012ZazvgwjfAJc6AdPfD2Rg3" - } - ] - }, - "usage": { - "input": 4559, - "output": 92 - } - }, - "f50b69dbcb4642c32ad4eccbd460b6a2d626c64e": { - "result": { - "role": "assistant", - "content": [ - { - "type": "text", - "text": "Select 'Table' as the first product", - "copilotToolCallId": "toolu_vrtx_01SvfQZwupogGaJcUE4csf5h" - }, - { - "type": "tool_call", - "name": "browser_select_option", - "arguments": { - "element": "Product Type dropdown", - "ref": "e7", - "values": [ - "Table" - ] - }, - "id": "toolu_vrtx_01SvfQZwupogGaJcUE4csf5h" - } - ] - }, - "usage": { - "input": 5272, - "output": 119 - } - }, - "943ffec08bf6a7d2c6bb94c3cd4de405f762141d": { - "result": { - "role": "assistant", - "content": [ - { - "type": "text", - "text": "Select 'Blue' as the color for the table", - "copilotToolCallId": "toolu_vrtx_01C7PuTtmKUeNfjnAhyzGLao" - }, - { - "type": "tool_call", - "name": "browser_select_option", - "arguments": { - "element": "Color dropdown", - "ref": "e10", - "values": [ - "Blue" - ] - }, - "id": "toolu_vrtx_01C7PuTtmKUeNfjnAhyzGLao" - } - ] - }, - "usage": { - "input": 5538, - "output": 120 - } - }, - "d97266b46b1ce270203e032759770d7130908c75": { - "result": { - "role": "assistant", - "content": [ - { - "type": "text", - "text": "Fill in all the customer information fields", - "copilotToolCallId": "toolu_vrtx_01Uwk1FBk6rrSc6dYnaEY64e" - }, - { - "type": "tool_call", - "name": "browser_fill_form", - "arguments": { - "fields": [ - { - "name": "Full Name", - "type": "textbox", - "ref": "e13", - "value": "John Doe" - }, - { - "name": "Email Address", - "type": "textbox", - "ref": "e16", - "value": "john@doe.me" - }, - { - "name": "Street Address", - "type": "textbox", - "ref": "e19", - "value": "123 Main St" - }, - { - "name": "City", - "type": "textbox", - "ref": "e22", - "value": "Anytown" - }, - { - "name": "State/Province", - "type": "textbox", - "ref": "e25", - "value": "XYZ" - }, - { - "name": "ZIP/Postal Code", - "type": "textbox", - "ref": "e28", - "value": "12345" - } - ] - }, - "id": "toolu_vrtx_01Uwk1FBk6rrSc6dYnaEY64e" - } - ] - }, - "usage": { - "input": 5807, - "output": 343 - } - }, - "fb4ee5ed9895e242986778145316cbad239ec245": { - "result": { - "role": "assistant", - "content": [ - { - "type": "text", - "text": "Submit the first order (blue table)", - "copilotToolCallId": "toolu_vrtx_01SuepPcu63ezb7A4T1zuKt8" - }, - { - "type": "tool_call", - "name": "browser_click", - "arguments": { - "element": "Order button", - "ref": "e29" - }, - "id": "toolu_vrtx_01SuepPcu63ezb7A4T1zuKt8" - } - ] - }, - "usage": { - "input": 6256, - "output": 95 - } - }, - "00580f7cb3850050b9aba09b7a670a2013e977f2": { - "result": { - "role": "assistant", - "content": [ - { - "type": "text", - "text": "Accept the confirmation dialog for the blue table order", - "copilotToolCallId": "toolu_vrtx_01D7BVzGcbb5GG141CaRGA6b" - }, - { - "type": "tool_call", - "name": "browser_handle_dialog", - "arguments": { - "accept": true - }, - "id": "toolu_vrtx_01D7BVzGcbb5GG141CaRGA6b" - } - ] - }, - "usage": { - "input": 6426, - "output": 80 - } - }, - "e534d18f4fdb32318c9488f5656d6f51c0eec040": { - "result": { - "role": "assistant", - "content": [ - { - "type": "text", - "text": "Click button to place the second order (green desk)", - "copilotToolCallId": "toolu_vrtx_01ACMDkwLt9DH2EQrsUZSuAj" - }, - { - "type": "tool_call", - "name": "browser_click", - "arguments": { - "element": "Place Another Order button", - "ref": "e58" - }, - "id": "toolu_vrtx_01ACMDkwLt9DH2EQrsUZSuAj" - } - ] - }, - "usage": { - "input": 7313, - "output": 100 - } - }, - "30d16e0ea65c3554058fcc2a359f36bfde57fbeb": { - "result": { - "role": "assistant", - "content": [ - { - "type": "text", - "text": "Select 'Desk' as the second product", - "copilotToolCallId": "toolu_vrtx_01Ew1Sw4U4sGhuZquXBSXg81" - }, - { - "type": "tool_call", - "name": "browser_select_option", - "arguments": { - "element": "Product Type dropdown", - "ref": "e7", - "values": [ - "Desk" - ] - }, - "id": "toolu_vrtx_01Ew1Sw4U4sGhuZquXBSXg81" - } - ] - }, - "usage": { - "input": 8033, - "output": 121 - } - }, - "2ff7a84f3e5629e27240bf3e49eddcd874909658": { - "result": { - "role": "assistant", - "content": [ - { - "type": "text", - "text": "Select 'Green' as the color for the desk", - "copilotToolCallId": "toolu_vrtx_011guHAt5XDWGYodYLbNceRS" - }, - { - "type": "tool_call", - "name": "browser_select_option", - "arguments": { - "element": "Color dropdown", - "ref": "e10", - "values": [ - "Green" - ] - }, - "id": "toolu_vrtx_011guHAt5XDWGYodYLbNceRS" - } - ] - }, - "usage": { - "input": 8302, - "output": 120 - } - }, - "9edcdd3441b58a8586ed160e9c250f12f51d1ecd": { - "result": { - "role": "assistant", - "content": [ - { - "type": "text", - "text": "Fill in the customer information fields for the second order", - "copilotToolCallId": "toolu_vrtx_01B9FkesqykHvZqZn8EeN6zz" - }, - { - "type": "tool_call", - "name": "browser_fill_form", - "arguments": { - "fields": [ - { - "name": "Full Name", - "ref": "e13", - "type": "textbox", - "value": "John Doe" - }, - { - "name": "Email Address", - "ref": "e16", - "type": "textbox", - "value": "john@doe.me" - }, - { - "name": "Street Address", - "ref": "e19", - "type": "textbox", - "value": "123 Main St" - }, - { - "name": "City", - "ref": "e22", - "type": "textbox", - "value": "Anytown" - }, - { - "name": "State/Province", - "ref": "e25", - "type": "textbox", - "value": "XYZ" - }, - { - "name": "ZIP/Postal Code", - "ref": "e28", - "type": "textbox", - "value": "12345" - } - ] - }, - "id": "toolu_vrtx_01B9FkesqykHvZqZn8EeN6zz" - } - ] - }, - "usage": { - "input": 8571, - "output": 266 - } - }, - "6fdde65702d23060ed79f4cf3639b01fba119365": { - "result": { - "role": "assistant", - "content": [ - { - "type": "text", - "text": "Submit the second order (green desk)", - "copilotToolCallId": "toolu_vrtx_01NMJ3ZaZUyvycfuW1X88mKS" - }, - { - "type": "tool_call", - "name": "browser_click", - "arguments": { - "element": "Order button", - "ref": "e29" - }, - "id": "toolu_vrtx_01NMJ3ZaZUyvycfuW1X88mKS" - } - ] - }, - "usage": { - "input": 9023, - "output": 95 - } - }, - "d9b25fb9eb9dd671e12b2578ca57750584c7d22d": { - "result": { - "role": "assistant", - "content": [ - { - "type": "text", - "text": "Accept the confirmation dialog for the green desk order", - "copilotToolCallId": "toolu_vrtx_0194ZbMphKfywXCJrotBARzJ" - }, - { - "type": "tool_call", - "name": "browser_handle_dialog", - "arguments": { - "accept": true - }, - "id": "toolu_vrtx_0194ZbMphKfywXCJrotBARzJ" - } - ] - }, - "usage": { - "input": 9193, - "output": 80 - } - }, - "fc2e5cead2dfb952bc04c51ffd6f69911587c20d": { - "result": { - "role": "assistant", - "content": [ - { - "type": "text", - "text": "Report the total price and maximum delivery time for both orders completed: blue table ($353.98, 3 days) and green desk ($299.98, 3 days)", - "copilotToolCallId": "toolu_vrtx_01AW3AuiY8uPE6qBtY6Nn5GF" - }, - { - "type": "tool_call", - "name": "report_result", - "arguments": { - "price": 653.96, - "deliveryDays": 3 - }, - "id": "toolu_vrtx_01AW3AuiY8uPE6qBtY6Nn5GF" - } - ] - }, - "usage": { - "input": 10081, - "output": 128 - } - } -} \ No newline at end of file diff --git a/tests/mcp/autowait.spec.ts b/tests/mcp/autowait.spec.ts index d37b6fd6cb603..d5d13c16e52fa 100644 --- a/tests/mcp/autowait.spec.ts +++ b/tests/mcp/autowait.spec.ts @@ -34,6 +34,6 @@ test('racy navigation destroys context', async ({ client, server }) => { }, })).toHaveResponse({ code: `await page.getByRole('button', { name: 'Submit' }).click();`, - pageState: expect.stringContaining(`- Page URL: about:blank`), + page: expect.stringContaining(`about:blank`), }); }); diff --git a/tests/mcp/cdp.spec.ts b/tests/mcp/cdp.spec.ts index 45eab8f543e43..0f11a03bf8a38 100644 --- a/tests/mcp/cdp.spec.ts +++ b/tests/mcp/cdp.spec.ts @@ -28,7 +28,7 @@ test('cdp server', async ({ cdpServer, startClient, server }) => { name: 'browser_navigate', arguments: { url: server.HELLO_WORLD }, })).toHaveResponse({ - pageState: expect.stringContaining(`- generic [active] [ref=e1]: Hello, world!`), + snapshot: expect.stringContaining(`- generic [active] [ref=e1]: Hello, world!`), }); }); @@ -46,19 +46,18 @@ test('cdp server reuse tab', async ({ cdpServer, startClient, server }) => { ref: 'f0', }, })).toHaveResponse({ - result: `Error: Ref f0 not found in the current page snapshot. Try capturing new snapshot.`, + error: `Error: Ref f0 not found in the current page snapshot. Try capturing new snapshot.`, isError: true, }); expect(await client.callTool({ name: 'browser_snapshot', })).toHaveResponse({ - pageState: expect.stringContaining(`- Page URL: ${server.HELLO_WORLD} -- Page Title: Title -- Page Snapshot: -\`\`\`yaml + page: `- Page URL: ${server.HELLO_WORLD} +- Page Title: Title`, + snapshot: `\`\`\`yaml - generic [active] [ref=e1]: Hello, world! -\`\`\``), +\`\`\``, }); }); @@ -74,7 +73,7 @@ test('should throw connection error and allow re-connecting', async ({ cdpServer name: 'browser_navigate', arguments: { url: server.PREFIX }, })).toHaveResponse({ - result: expect.stringContaining(`Error: browserType.connectOverCDP: connect ECONNREFUSED`), + error: expect.stringContaining(`Error: browserType.connectOverCDP: connect ECONNREFUSED`), isError: true, }); await cdpServer.start(); @@ -82,7 +81,7 @@ test('should throw connection error and allow re-connecting', async ({ cdpServer name: 'browser_navigate', arguments: { url: server.PREFIX }, })).toHaveResponse({ - pageState: expect.stringContaining(`- generic [active] [ref=e1]: Hello, world!`), + snapshot: expect.stringContaining(`- generic [active] [ref=e1]: Hello, world!`), }); }); diff --git a/tests/mcp/click.spec.ts b/tests/mcp/click.spec.ts index 72a37533a4ade..7d3d6d548917f 100644 --- a/tests/mcp/click.spec.ts +++ b/tests/mcp/click.spec.ts @@ -41,7 +41,7 @@ test('browser_click', async ({ client, server }) => { }, })).toHaveResponse({ code: `await page.getByRole('button', { name: 'Submit' }).click();`, - pageState: expect.stringContaining(`button "Submit" [active] [ref=e2]`), + snapshot: expect.stringContaining(`button "Submit" [active] [ref=e2]`), }); }); @@ -70,7 +70,7 @@ test('browser_click (double)', async ({ client, server }) => { }, })).toHaveResponse({ code: `await page.getByRole('heading', { name: 'Click me' }).dblclick();`, - pageState: expect.stringContaining(`heading "Double clicked" [level=1] [ref=e3]`), + snapshot: expect.stringContaining(`heading "Double clicked" [level=1] [ref=e3]`), }); }); @@ -100,7 +100,7 @@ test('browser_click (right)', async ({ client, server }) => { }); expect(result).toHaveResponse({ code: `await page.getByRole('button', { name: 'Menu' }).click({ button: 'right' });`, - pageState: expect.stringContaining(`button "Right clicked"`), + snapshot: expect.stringContaining(`button "Right clicked"`), }); }); @@ -131,7 +131,7 @@ test('browser_click (modifiers)', async ({ client, server, mcpBrowser }) => { }, })).toHaveResponse({ code: `await page.getByRole('button', { name: 'Submit' }).click({ modifiers: ['Control'] });`, - pageState: expect.stringContaining(`generic [ref=e3]: ctrlKey:true metaKey:false shiftKey:false altKey:false`), + snapshot: expect.stringContaining(`generic [ref=e3]: ctrlKey:true metaKey:false shiftKey:false altKey:false`), }); } @@ -144,7 +144,7 @@ test('browser_click (modifiers)', async ({ client, server, mcpBrowser }) => { }, })).toHaveResponse({ code: `await page.getByRole('button', { name: 'Submit' }).click({ modifiers: ['Shift'] });`, - pageState: expect.stringContaining(`generic [ref=e3]: ctrlKey:false metaKey:false shiftKey:true altKey:false`), + snapshot: expect.stringContaining(`generic [ref=e3]: ctrlKey:false metaKey:false shiftKey:true altKey:false`), }); expect(await client.callTool({ @@ -156,7 +156,7 @@ test('browser_click (modifiers)', async ({ client, server, mcpBrowser }) => { }, })).toHaveResponse({ code: `await page.getByRole('button', { name: 'Submit' }).click({ modifiers: ['Shift', 'Alt'] });`, - pageState: expect.stringContaining(`generic [ref=e3]: ctrlKey:false metaKey:false shiftKey:true altKey:true`), + snapshot: expect.stringContaining(`generic [ref=e3]: ctrlKey:false metaKey:false shiftKey:true altKey:true`), }); }); diff --git a/tests/mcp/config.spec.ts b/tests/mcp/config.spec.ts index b13217617507d..2d8f7721cb7dc 100644 --- a/tests/mcp/config.spec.ts +++ b/tests/mcp/config.spec.ts @@ -39,7 +39,7 @@ test('config user data dir', async ({ startClient, server }, testInfo) => { name: 'browser_navigate', arguments: { url: server.PREFIX }, })).toHaveResponse({ - pageState: expect.stringContaining(`Hello, world!`), + snapshot: expect.stringContaining(`Hello, world!`), }); const files = await fs.promises.readdir(config.browser!.userDataDir!); @@ -53,7 +53,7 @@ test('executable path', async ({ startClient, server }, testInfo) => { arguments: { url: server.PREFIX }, })).toHaveResponse({ isError: true, - result: expect.stringMatching(/Failed to launch.*missing-executable/), + error: expect.stringMatching(/Failed to launch.*missing-executable/), }); }); @@ -73,7 +73,7 @@ test.describe(() => { name: 'browser_navigate', arguments: { url: 'data:text/html,' }, })).toHaveResponse({ - pageState: expect.stringContaining(`Firefox`), + page: expect.stringContaining(`Firefox`), }); }); }); diff --git a/tests/mcp/console.spec.ts b/tests/mcp/console.spec.ts index c627c08a5443b..2fa1e6efcaa98 100644 --- a/tests/mcp/console.spec.ts +++ b/tests/mcp/console.spec.ts @@ -95,7 +95,7 @@ test('recent console messages', async ({ client, server }) => { }); expect(response).toHaveResponse({ - consoleMessages: expect.stringContaining(`- [LOG] Hello, world! @`), + events: expect.stringContaining(`- [LOG] Hello, world! @`), }); }); @@ -121,8 +121,8 @@ test('recent console messages filter', async ({ startClient, server }) => { }, })); - expect(response.consoleMessages).toContain('console.error'); - expect(response.consoleMessages).not.toContain('console.log'); + expect(response.events).toContain('console.error'); + expect(response.events).not.toContain('console.log'); }); test('browser_console_messages default level', async ({ client, server }) => { diff --git a/tests/mcp/core.spec.ts b/tests/mcp/core.spec.ts index a02b31f179049..57ab3ad88e105 100644 --- a/tests/mcp/core.spec.ts +++ b/tests/mcp/core.spec.ts @@ -24,10 +24,9 @@ test('browser_navigate', async ({ client, server }) => { arguments: { url: server.HELLO_WORLD }, })).toHaveResponse({ code: `await page.goto('${server.HELLO_WORLD}');`, - pageState: `- Page URL: ${server.HELLO_WORLD} -- Page Title: Title -- Page Snapshot: -\`\`\`yaml + page: `- Page URL: ${server.HELLO_WORLD} +- Page Title: Title`, + snapshot: `\`\`\`yaml - generic [active] [ref=e1]: Hello, world! \`\`\``, }); @@ -38,7 +37,7 @@ test('browser_navigate blocks file:// URLs by default', async ({ client }) => { name: 'browser_navigate', arguments: { url: 'file:///etc/passwd' }, })).toHaveResponse({ - result: expect.stringContaining('Error: Access to "file:" URL is blocked. Allowed protocols: http:, https:, about:, data:. Attempted URL: file:///etc/passwd'), + error: expect.stringContaining('Error: Access to "file:" URL is blocked. Allowed protocols: http:, https:, about:, data:. Attempted URL: file:///etc/passwd'), isError: true, }); }); @@ -49,10 +48,8 @@ test('browser_navigate allows about:, data: and javascript: protocols', async ({ arguments: { url: 'about:blank' }, })).toHaveResponse({ code: `await page.goto('about:blank');`, - pageState: `- Page URL: about:blank -- Page Title: -- Page Snapshot: -\`\`\`yaml + page: `- Page URL: about:blank`, + snapshot: `\`\`\`yaml \`\`\``, }); @@ -62,10 +59,8 @@ test('browser_navigate allows about:, data: and javascript: protocols', async ({ arguments: { url: 'data:text/html,

Hello

' }, })).toHaveResponse({ code: `await page.goto('data:text/html,

Hello

');`, - pageState: `- Page URL: data:text/html,

Hello

-- Page Title: -- Page Snapshot: -\`\`\`yaml + page: `- Page URL: data:text/html,

Hello

`, + snapshot: `\`\`\`yaml - heading \"Hello\" [level=1] [ref=e2] \`\`\``, }); @@ -92,10 +87,8 @@ test('browser_navigate can navigate to file:// URLs allowUnrestrictedFileAccess name: 'browser_navigate', arguments: { url }, })).toHaveResponse({ - pageState: `- Page URL: ${url} -- Page Title: -- Page Snapshot: -\`\`\`yaml + page: `- Page URL: ${url}`, + snapshot: `\`\`\`yaml - generic [ref=e2]: Test file content \`\`\``, }); @@ -123,10 +116,8 @@ test('browser_select_option', async ({ client, server }) => { values: ['bar'], }, })).toHaveResponse({ - pageState: `- Page URL: ${server.PREFIX}/ -- Page Title: Title -- Page Snapshot: -\`\`\`yaml + page: undefined, // Did not change. + snapshot: `\`\`\`yaml - combobox [ref=e2]: - option "Foo" - option "Bar" [selected] @@ -158,9 +149,11 @@ test('browser_select_option (multiple)', async ({ client, server }) => { }, })).toHaveResponse({ code: `await page.getByRole('listbox').selectOption(['bar', 'baz']);`, - pageState: expect.stringContaining(` + page: undefined, // did not change + snapshot: `\`\`\`yaml - option "Bar" [selected] [ref=e4] -- option "Baz" [selected] [ref=e5]`), +- option "Baz" [selected] [ref=e5] +\`\`\``, }); }); @@ -189,7 +182,7 @@ test('browser_resize', async ({ client, server }) => { code: `await page.setViewportSize({ width: 390, height: 780 });`, }); await expect.poll(() => client.callTool({ name: 'browser_snapshot' })).toHaveResponse({ - pageState: expect.stringContaining(`Window size: 390x780`), + snapshot: expect.stringContaining(`Window size: 390x780`), }); }); @@ -210,7 +203,7 @@ test('old locator error message', async ({ client, server }) => { url: server.PREFIX, }, })).toHaveResponse({ - pageState: expect.stringContaining(` + snapshot: expect.stringContaining(` - button "Button 1" [ref=e2] - button "Button 2" [ref=e3]`), }); @@ -230,7 +223,7 @@ test('old locator error message', async ({ client, server }) => { ref: 'e3', }, })).toHaveResponse({ - result: expect.stringContaining(`Ref e3 not found in the current page snapshot. Try capturing new snapshot.`), + error: expect.stringContaining(`Ref e3 not found in the current page snapshot. Try capturing new snapshot.`), isError: true, }); }); @@ -252,6 +245,6 @@ test('visibility: hidden > visible should be shown', { annotation: { type: 'issu expect(await client.callTool({ name: 'browser_snapshot' })).toHaveResponse({ - pageState: expect.stringContaining(`- button "Button"`), + snapshot: expect.stringContaining(`- button "Button"`), }); }); diff --git a/tests/mcp/device.spec.ts b/tests/mcp/device.spec.ts index 2207b5b25f049..696834149269a 100644 --- a/tests/mcp/device.spec.ts +++ b/tests/mcp/device.spec.ts @@ -40,6 +40,6 @@ test('--device should work', async ({ startClient, server }) => { url: server.PREFIX, }, })).toHaveResponse({ - pageState: expect.stringContaining(`393x659`), + snapshot: expect.stringContaining(`393x659`), }); }); diff --git a/tests/mcp/dialogs.spec.ts b/tests/mcp/dialogs.spec.ts index 79993805fe98f..8f3dba07f7375 100644 --- a/tests/mcp/dialogs.spec.ts +++ b/tests/mcp/dialogs.spec.ts @@ -22,7 +22,7 @@ test('alert dialog', async ({ client, server }) => { name: 'browser_navigate', arguments: { url: server.PREFIX }, })).toHaveResponse({ - pageState: expect.stringContaining(`- button "Button" [ref=e2]`), + snapshot: expect.stringContaining(`- button "Button" [ref=e2]`), }); expect(await client.callTool({ @@ -54,7 +54,7 @@ test('alert dialog', async ({ client, server }) => { }, })).toHaveResponse({ modalState: undefined, - pageState: expect.stringContaining(`Page Title: Title`), + page: expect.stringContaining(`- Page Title: Title`), }); }); @@ -70,7 +70,7 @@ test('two alert dialogs', async ({ client, server }) => { name: 'browser_navigate', arguments: { url: server.PREFIX }, })).toHaveResponse({ - pageState: expect.stringContaining(`- button "Button" [ref=e2]`), + snapshot: expect.stringContaining(`- button "Button" [ref=e2]`), }); expect(await client.callTool({ @@ -119,7 +119,7 @@ test('confirm dialog (true)', async ({ client, server }) => { name: 'browser_navigate', arguments: { url: server.PREFIX }, })).toHaveResponse({ - pageState: expect.stringContaining(`- button "Button" [ref=e2]`), + snapshot: expect.stringContaining(`- button "Button" [ref=e2]`), }); expect(await client.callTool({ @@ -139,7 +139,7 @@ test('confirm dialog (true)', async ({ client, server }) => { }, })).toHaveResponse({ modalState: undefined, - pageState: expect.stringContaining(`generic [active] [ref=e1]: "true"`), + snapshot: expect.stringContaining(`generic [active] [ref=e1]: "true"`), }); }); @@ -155,7 +155,7 @@ test('confirm dialog (false)', async ({ client, server }) => { name: 'browser_navigate', arguments: { url: server.PREFIX }, })).toHaveResponse({ - pageState: expect.stringContaining(`- button "Button" [ref=e2]`), + snapshot: expect.stringContaining(`- button "Button" [ref=e2]`), }); expect(await client.callTool({ @@ -175,7 +175,7 @@ test('confirm dialog (false)', async ({ client, server }) => { }, })).toHaveResponse({ modalState: undefined, - pageState: expect.stringContaining(`generic [active] [ref=e1]: "false"`), + snapshot: expect.stringContaining(`generic [active] [ref=e1]: "false"`), }); }); @@ -191,7 +191,7 @@ test('prompt dialog', async ({ client, server }) => { name: 'browser_navigate', arguments: { url: server.PREFIX }, })).toHaveResponse({ - pageState: expect.stringContaining(`- button "Button" [ref=e2]`), + snapshot: expect.stringContaining(`- button "Button" [ref=e2]`), }); expect(await client.callTool({ @@ -213,7 +213,7 @@ test('prompt dialog', async ({ client, server }) => { }); expect(result).toHaveResponse({ - pageState: expect.stringContaining(`generic [active] [ref=e1]: Answer`), + snapshot: expect.stringContaining(`generic [active] [ref=e1]: Answer`), }); }); @@ -223,7 +223,7 @@ test('alert dialog w/ race', async ({ client, server }) => { name: 'browser_navigate', arguments: { url: server.PREFIX }, })).toHaveResponse({ - pageState: expect.stringContaining(`- button "Button" [ref=e2]`), + snapshot: expect.stringContaining(`- button "Button" [ref=e2]`), }); expect(await client.callTool({ @@ -246,8 +246,7 @@ test('alert dialog w/ race', async ({ client, server }) => { expect(result).toHaveResponse({ modalState: undefined, - pageState: expect.stringContaining(`- Page URL: ${server.PREFIX}/ -- Page Title: Title -- Page Snapshot:`), + page: expect.stringContaining(`- Page URL: ${server.PREFIX}/ +- Page Title: Title`), }); }); diff --git a/tests/mcp/evaluate.spec.ts b/tests/mcp/evaluate.spec.ts index 68c8a39a398db..cb12bb868da3e 100644 --- a/tests/mcp/evaluate.spec.ts +++ b/tests/mcp/evaluate.spec.ts @@ -21,7 +21,7 @@ test('browser_evaluate', async ({ client, server }) => { name: 'browser_navigate', arguments: { url: server.HELLO_WORLD }, })).toHaveResponse({ - pageState: expect.stringContaining(`- Page Title: Title`), + page: expect.stringContaining(`- Page Title: Title`), }); expect(await client.callTool({ @@ -62,7 +62,7 @@ test('browser_evaluate object', async ({ client, server }) => { name: 'browser_navigate', arguments: { url: server.HELLO_WORLD }, })).toHaveResponse({ - pageState: expect.stringContaining(`- Page Title: Title`), + page: expect.stringContaining(`- Page Title: Title`), }); expect(await client.callTool({ @@ -81,7 +81,7 @@ test('browser_evaluate (error)', async ({ client, server }) => { name: 'browser_navigate', arguments: { url: server.HELLO_WORLD }, })).toHaveResponse({ - pageState: expect.stringContaining(`- Page Title: Title`), + page: expect.stringContaining(`- Page Title: Title`), }); const result = await client.callTool({ diff --git a/tests/mcp/files.spec.ts b/tests/mcp/files.spec.ts index 538a2370d1893..526ee9fac3c58 100644 --- a/tests/mcp/files.spec.ts +++ b/tests/mcp/files.spec.ts @@ -15,7 +15,9 @@ */ import fs from 'fs/promises'; -import { test, expect } from './fixtures'; +import path from 'path'; + +import { test, expect, parseResponse } from './fixtures'; test('browser_file_upload', async ({ client, server }, testInfo) => { server.setContent('/', ` @@ -27,7 +29,7 @@ test('browser_file_upload', async ({ client, server }, testInfo) => { name: 'browser_navigate', arguments: { url: server.PREFIX }, })).toHaveResponse({ - pageState: expect.stringContaining(`- generic [active] [ref=e1]: + snapshot: expect.stringContaining(`- generic [active] [ref=e1]: - button "Choose File" [ref=e2] - button "Button" [ref=e3]`), }); @@ -38,8 +40,8 @@ test('browser_file_upload', async ({ client, server }, testInfo) => { arguments: { paths: [] }, })).toHaveResponse({ isError: true, - result: expect.stringContaining(`The tool "browser_file_upload" can only be used when there is related modal state present.`), - modalState: expect.stringContaining(`- There is no modal state present`), + error: expect.stringContaining(`The tool "browser_file_upload" can only be used when there is related modal state present.`), + modalState: undefined, }); } @@ -94,7 +96,8 @@ test('browser_file_upload', async ({ client, server }, testInfo) => { }); expect(response).toHaveResponse({ - result: `Error: Tool "browser_click" does not handle the modal state.`, + isError: true, + error: `Error: Tool "browser_click" does not handle the modal state.`, modalState: expect.stringContaining(`- [File chooser]: can be handled by the "browser_file_upload" tool`), }); } @@ -112,18 +115,26 @@ test('clicking on download link emits download', async ({ startClient, server }, name: 'browser_navigate', arguments: { url: server.PREFIX }, })).toHaveResponse({ - pageState: expect.stringContaining(`- link "Download" [ref=e2]`), + snapshot: expect.stringContaining(`- link "Download" [ref=e2]`), }); - await client.callTool({ + + const response = await client.callTool({ name: 'browser_click', arguments: { element: 'Download link', ref: 'e2', }, }); - await expect.poll(() => client.callTool({ name: 'browser_snapshot' })).toHaveResponse({ - downloads: `- Downloaded file test.txt to ${testInfo.outputPath('output', 'test.txt')}`, - }); + const parsed = parseResponse(response); + let events = parsed.events; + await expect.poll(async () => { + const r = await client.callTool({ name: 'browser_snapshot' }); + const p = parseResponse(r); + if (p.events) + events += '\n' + p.events; + return events; + }).toBe(`- Downloading file test.txt ... +- Downloaded file test.txt to "output${path.sep}test.txt"`); }); test('navigating to download link emits download', async ({ startClient, server, mcpBrowser }, testInfo) => { @@ -146,7 +157,7 @@ test('navigating to download link emits download', async ({ startClient, server, url: server.PREFIX + '/download', }, })).toHaveResponse({ - downloads: expect.stringMatching(`- Downloaded file test\.txt to|- Downloading file test\.txt`), + events: expect.stringMatching(`- Downloaded file test\.txt to|- Downloading file test\.txt`), }); }); @@ -214,7 +225,7 @@ test('file upload restricted to roots by default', async ({ startClient, server }, })).toHaveResponse({ isError: true, - result: expect.stringMatching('File access denied: .* is outside allowed roots'), + error: expect.stringMatching('File access denied: .* is outside allowed roots'), }); }); @@ -275,7 +286,7 @@ test('file upload is restricted to cwd if no roots are configured', async ({ sta }, })).toHaveResponse({ isError: true, - result: expect.stringMatching('File access denied: .* is outside allowed roots. Allowed roots: ' + rootDir.replace(/\\/g, '\\\\')), + error: expect.stringMatching('File access denied: .* is outside allowed roots. Allowed roots: ' + rootDir.replace(/\\/g, '\\\\')), }); }); diff --git a/tests/mcp/fixtures.ts b/tests/mcp/fixtures.ts index ea5c6d6947d06..46f6d67994007 100644 --- a/tests/mcp/fixtures.ts +++ b/tests/mcp/fixtures.ts @@ -223,16 +223,27 @@ type Response = Awaited>; export const expect = baseExpect.extend({ toHaveResponse(response: Response, object: any) { const parsed = parseResponse(response); + const text = parsed.text; const isNot = this.isNot; + + const keys = Object.keys(object); + for (const key of Object.keys(parsed)) { + if (!keys.includes(key)) + delete parsed[key]; + } + try { - if (isNot) + if (isNot) { expect(parsed).not.toEqual(expect.objectContaining(object)); - else + } else { expect(parsed).toEqual(expect.objectContaining(object)); + if (parsed.isError && !object.isError) + throw new Error('Response is an error, but expected is not'); + } } catch (e) { return { pass: isNot, - message: () => e.message, + message: () => e.message + '\n\nResponse text:\n' + text, }; } return { @@ -321,8 +332,3 @@ export async function prepareDebugTest(startClient: StartClient, testFile?: stri const [, id] = listResult.content[0].text.match(/\[id=([^\]]+)\]/); return { client, id }; } - -export const lowireMeta = { - 'dev.lowire/history': true, - 'dev.lowire/state': true, -}; diff --git a/tests/mcp/form.spec.ts b/tests/mcp/form.spec.ts index 893c4517889e2..fdc6fd186f119 100644 --- a/tests/mcp/form.spec.ts +++ b/tests/mcp/form.spec.ts @@ -106,18 +106,18 @@ await page.getByRole('checkbox', { name: 'Subscribe to newsletter' }).setChecked }, }); expect.soft(response).toHaveResponse({ - pageState: expect.stringMatching(/textbox "Name".*John Doe/), + snapshot: expect.stringMatching(/textbox "Name".*John Doe/), }); expect.soft(response).toHaveResponse({ - pageState: expect.stringMatching(/textbox "Email".*john.doe@example.com/), + snapshot: expect.stringMatching(/textbox "Email".*john.doe@example.com/), }); expect.soft(response).toHaveResponse({ - pageState: expect.stringMatching(/slider "Age".*"25"/), + snapshot: expect.stringMatching(/slider "Age".*"25"/), }); expect.soft(response).toHaveResponse({ - pageState: expect.stringContaining('option \"United States\" [selected]'), + snapshot: expect.stringContaining('option \"United States\" [selected]'), }); expect.soft(response).toHaveResponse({ - pageState: expect.stringContaining('checkbox \"Subscribe to newsletter\" [checked]'), + snapshot: expect.stringContaining('checkbox \"Subscribe to newsletter\" [checked]'), }); }); diff --git a/tests/mcp/generator.spec.ts b/tests/mcp/generator.spec.ts index 74f1bc2fcea33..af3a1b369528f 100644 --- a/tests/mcp/generator.spec.ts +++ b/tests/mcp/generator.spec.ts @@ -164,7 +164,7 @@ test('click after generator_log_action', async ({ startClient }) => { }, })).toHaveResponse({ code: `await page.getByRole('button', { name: 'Submit' }).click();`, - pageState: expect.stringContaining(`- button "Submit"`), + snapshot: expect.stringContaining(`- button "Submit"`), }); expect(await client.callTool({ diff --git a/tests/mcp/headed.spec.ts b/tests/mcp/headed.spec.ts index 56fd4923c62a9..6205485f72580 100644 --- a/tests/mcp/headed.spec.ts +++ b/tests/mcp/headed.spec.ts @@ -41,7 +41,7 @@ for (const mcpHeadless of [false, true]) { }); expect(response).toHaveResponse({ - pageState: (mcpHeadless ? expect : expect.not).stringContaining(`HeadlessChrome`), + snapshot: (mcpHeadless ? expect : expect.not).stringContaining(`HeadlessChrome`), }); }); }); diff --git a/tests/mcp/http.spec.ts b/tests/mcp/http.spec.ts index 22e0ce4d33139..9c0bb9cbb691e 100644 --- a/tests/mcp/http.spec.ts +++ b/tests/mcp/http.spec.ts @@ -284,7 +284,7 @@ test('http transport shared context', async ({ serverEndpoint, server }) => { }); // Should have at least one tab (the one created by client1) - expect(tabsResult.content[0]?.text).toContain('tabs'); + expect(tabsResult.content[0]?.text).toContain('Title'); await transport1.terminateSession(); await client1.close(); diff --git a/tests/mcp/iframes.spec.ts b/tests/mcp/iframes.spec.ts index aef7e0787904e..925a1c8d2bb70 100644 --- a/tests/mcp/iframes.spec.ts +++ b/tests/mcp/iframes.spec.ts @@ -23,7 +23,7 @@ test('stitched aria frames', async ({ client }) => { url: `data:text/html,

Hello

`, }, })).toHaveResponse({ - pageState: expect.stringContaining(`- generic [active] [ref=e1]: + snapshot: expect.stringContaining(`- generic [active] [ref=e1]: - heading "Hello" [level=1] [ref=e2] - iframe [ref=e3]: - generic [active] [ref=f1e1]: diff --git a/tests/mcp/init-page.spec.ts b/tests/mcp/init-page.spec.ts index ad363f7212b5c..58c334681d1d5 100644 --- a/tests/mcp/init-page.spec.ts +++ b/tests/mcp/init-page.spec.ts @@ -32,7 +32,7 @@ test('--init-page', async ({ startClient }) => { name: 'browser_snapshot', arguments: {}, })).toHaveResponse({ - pageState: expect.stringContaining('Hello world'), + snapshot: expect.stringContaining('Hello world'), }); }); @@ -65,7 +65,7 @@ test('--init-page w/ --init-script', async ({ startClient, server }) => { url: server.PREFIX, }, })).toHaveResponse({ - pageState: expect.stringContaining('Hello world'), + snapshot: expect.stringContaining('Hello world'), }); expect(await client.callTool({ diff --git a/tests/mcp/install.spec.ts b/tests/mcp/install.spec.ts index 70fde07597ee7..feb8081225c05 100644 --- a/tests/mcp/install.spec.ts +++ b/tests/mcp/install.spec.ts @@ -21,6 +21,6 @@ test('browser_install', async ({ client, mcpBrowser }) => { expect(await client.callTool({ name: 'browser_install', })).toHaveResponse({ - tabs: expect.stringContaining(`No open tabs`), + result: expect.stringContaining(`No open tabs`), }); }); diff --git a/tests/mcp/launch.spec.ts b/tests/mcp/launch.spec.ts index c008c72e86a92..67d90077abba3 100644 --- a/tests/mcp/launch.spec.ts +++ b/tests/mcp/launch.spec.ts @@ -33,14 +33,13 @@ test('test reopen browser', async ({ startClient, server }) => { name: 'browser_close', })).toHaveResponse({ code: `await page.close()`, - tabs: `No open tabs. Use the "browser_navigate" tool to navigate to a page first.`, }); expect(await client.callTool({ name: 'browser_navigate', arguments: { url: server.HELLO_WORLD }, })).toHaveResponse({ - pageState: expect.stringContaining(`- generic [active] [ref=e1]: Hello, world!`), + snapshot: expect.stringContaining(`- generic [active] [ref=e1]: Hello, world!`), }); await client.close(); @@ -72,7 +71,7 @@ test('executable path', async ({ startClient, server }) => { arguments: { url: server.HELLO_WORLD }, }); expect(response).toHaveResponse({ - result: expect.stringContaining(`executable doesn't exist`), + error: expect.stringContaining(`executable doesn't exist`), isError: true, }); }); @@ -94,7 +93,7 @@ test('persistent context', async ({ startClient, server }, testInfo) => { name: 'browser_navigate', arguments: { url: server.PREFIX }, })).toHaveResponse({ - pageState: expect.stringContaining(`Storage: NO`), + snapshot: expect.stringContaining(`Storage: NO`), }); await new Promise(resolve => setTimeout(resolve, 3000)); @@ -110,7 +109,7 @@ test('persistent context', async ({ startClient, server }, testInfo) => { name: 'browser_navigate', arguments: { url: server.PREFIX }, })).toHaveResponse({ - pageState: expect.stringContaining(`Storage: YES`), + snapshot: expect.stringContaining(`Storage: YES`), }); }); @@ -129,7 +128,7 @@ test('isolated context', async ({ startClient, server }) => { name: 'browser_navigate', arguments: { url: server.PREFIX }, })).toHaveResponse({ - pageState: expect.stringContaining(`Storage: NO`), + snapshot: expect.stringContaining(`Storage: NO`), }); await client1.callTool({ @@ -141,7 +140,7 @@ test('isolated context', async ({ startClient, server }) => { name: 'browser_navigate', arguments: { url: server.PREFIX }, })).toHaveResponse({ - pageState: expect.stringContaining(`Storage: NO`), + snapshot: expect.stringContaining(`Storage: NO`), }); }); @@ -172,7 +171,7 @@ test('isolated context with storage state', async ({ startClient, server }, test name: 'browser_navigate', arguments: { url: server.PREFIX }, })).toHaveResponse({ - pageState: expect.stringContaining(`Storage: session-value`), + snapshot: expect.stringContaining(`Storage: session-value`), }); }); @@ -196,16 +195,18 @@ exit 1 }); expect.soft(result).toHaveResponse({ isError: true, - result: expect.stringContaining(`Bogus browser script`), + error: expect.stringContaining(`Bogus browser script`), }); // Chromium waits for the CDP endpoint, so we know if the process failed to launch // before connecting. if (mcpBrowser === 'chromium') { expect.soft(result).toHaveResponse({ - result: expect.stringContaining(`Failed to launch the browser process.`), + isError: true, + error: expect.stringContaining(`Failed to launch the browser process.`), }); } expect.soft(result).toHaveResponse({ - result: expect.not.stringContaining(`Browser is already in use`), + isError: true, + error: expect.not.stringContaining(`Browser is already in use`), }); }); diff --git a/tests/mcp/pdf.spec.ts b/tests/mcp/pdf.spec.ts index 80faf0b4b2b21..5f0710302b7ec 100644 --- a/tests/mcp/pdf.spec.ts +++ b/tests/mcp/pdf.spec.ts @@ -28,7 +28,7 @@ test('save as pdf unavailable', async ({ startClient, server }) => { expect(await client.callTool({ name: 'browser_pdf_save', })).toHaveResponse({ - result: 'Error: Tool "browser_pdf_save" not found', + error: 'Tool "browser_pdf_save" not found', isError: true, }); }); @@ -44,14 +44,14 @@ test('save as pdf', async ({ startClient, mcpBrowser, server }, testInfo) => { name: 'browser_navigate', arguments: { url: server.HELLO_WORLD }, })).toHaveResponse({ - pageState: expect.stringContaining(`- generic [active] [ref=e1]: Hello, world!`), + snapshot: expect.stringContaining(`- generic [active] [ref=e1]: Hello, world!`), }); expect(await client.callTool({ name: 'browser_pdf_save', })).toHaveResponse({ code: expect.stringContaining(`await page.pdf(`), - files: expect.stringMatching(/\[Page saved as PDF\]\(.*page-[^:]+.pdf\)/), + result: expect.stringMatching(/\[Page as pdf\]\(.*page-[^:]+.pdf\)/), }); }); @@ -66,7 +66,7 @@ test('save as pdf (filename: output.pdf)', async ({ startClient, mcpBrowser, ser name: 'browser_navigate', arguments: { url: server.HELLO_WORLD }, })).toHaveResponse({ - pageState: expect.stringContaining(`- generic [active] [ref=e1]: Hello, world!`), + snapshot: expect.stringContaining(`- generic [active] [ref=e1]: Hello, world!`), }); expect(await client.callTool({ @@ -75,7 +75,7 @@ test('save as pdf (filename: output.pdf)', async ({ startClient, mcpBrowser, ser filename: 'output.pdf', }, })).toHaveResponse({ - files: expect.stringContaining(`output.pdf`), + result: expect.stringContaining(`output.pdf`), code: expect.stringContaining(`await page.pdf(`), }); diff --git a/tests/mcp/planner.spec.ts b/tests/mcp/planner.spec.ts index 30fbe74283234..736b56b61d284 100644 --- a/tests/mcp/planner.spec.ts +++ b/tests/mcp/planner.spec.ts @@ -62,7 +62,7 @@ test('planner_setup_page', async ({ startClient }) => { }, })).toHaveResponse({ code: `await page.getByRole('button', { name: 'Submit' }).click();`, - pageState: expect.stringContaining(`- button "Submit"`), + snapshot: expect.stringContaining(`- button "Submit"`), }); }); diff --git a/tests/mcp/roots.spec.ts b/tests/mcp/roots.spec.ts index 13de9263c66c7..e386428157bca 100644 --- a/tests/mcp/roots.spec.ts +++ b/tests/mcp/roots.spec.ts @@ -151,6 +151,6 @@ test('should return relative paths when root is specified', async ({ startClient name: 'browser_take_screenshot', arguments: { filename: 'screenshot.png' }, })).toHaveResponse({ - files: expect.stringContaining(`- [Screenshot of viewport](output${path.sep}screenshot.png)`), + result: expect.stringContaining(`[Screenshot of viewport](output${path.sep}screenshot.png)`), }); }); diff --git a/tests/mcp/run-code.spec.ts b/tests/mcp/run-code.spec.ts index a8e3149714198..bc673f2d888a5 100644 --- a/tests/mcp/run-code.spec.ts +++ b/tests/mcp/run-code.spec.ts @@ -34,7 +34,7 @@ test('browser_run_code', async ({ client, server }) => { }, })).toHaveResponse({ code: `await (${code})(page);`, - consoleMessages: expect.stringContaining('- [LOG] Submit'), + events: expect.stringContaining('- [LOG] Submit'), }); }); @@ -54,7 +54,7 @@ test('browser_run_code block', async ({ client, server }) => { }, })).toHaveResponse({ code: expect.stringContaining(`await page.getByRole(\"button\", { name: \"Submit\" }).click()`), - consoleMessages: expect.stringMatching(/\[LOG\] Submit.*\n.*\[LOG\] Submit/), + events: expect.stringMatching(/\[LOG\] Submit.*\n.*\[LOG\] Submit/), }); }); @@ -73,7 +73,8 @@ test('browser_run_code no-require', async ({ client, server }) => { code: `(page) => { require('fs'); }`, }, })).toHaveResponse({ - result: expect.stringContaining(`ReferenceError: require is not defined`), + error: expect.stringContaining(`ReferenceError: require is not defined`), + isError: true, }); }); @@ -89,7 +90,7 @@ test('browser_run_code blocks fetch of file:// URLs by default', async ({ client code: `async (page) => { await page.request.get('file:///etc/passwd'); }`, }, })).toHaveResponse({ - result: expect.stringContaining('Error: apiRequestContext.get: Access to "file:" URL is blocked. Allowed protocols: http:, https:, about:, data:. Attempted URL: file:///etc/passwd'), + error: expect.stringContaining('Error: apiRequestContext.get: Access to "file:" URL is blocked. Allowed protocols: http:, https:, about:, data:. Attempted URL: file:///etc/passwd'), isError: true, }); }); @@ -141,7 +142,7 @@ test('browser_run_code restricts setInputFiles to roots by default', async ({ st }, })).toHaveResponse({ isError: true, - result: expect.stringMatching('File access denied: .* is outside allowed roots'), + error: expect.stringMatching('File access denied: .* is outside allowed roots'), }); }); @@ -162,7 +163,7 @@ test('browser_run_code return value', async ({ client, server }) => { }, })).toHaveResponse({ code: `await (${code})(page);`, - consoleMessages: expect.stringContaining('- [LOG] Submit'), + events: expect.stringContaining('- [LOG] Submit'), result: '{"message":"Hello, world!"}', }); }); diff --git a/tests/mcp/screenshot.spec.ts b/tests/mcp/screenshot.spec.ts index 2fb287e81f835..b83ac5b9e8d8e 100644 --- a/tests/mcp/screenshot.spec.ts +++ b/tests/mcp/screenshot.spec.ts @@ -50,7 +50,7 @@ test('browser_take_screenshot (element)', async ({ startClient, server }, testIn name: 'browser_navigate', arguments: { url: server.HELLO_WORLD }, })).toHaveResponse({ - pageState: expect.stringContaining(`[ref=e1]`), + snapshot: expect.stringContaining(`[ref=e1]`), }); expect(await client.callTool({ @@ -363,7 +363,7 @@ test('browser_take_screenshot (fullPage with element should error)', async ({ st name: 'browser_navigate', arguments: { url: server.HELLO_WORLD }, })).toHaveResponse({ - pageState: expect.stringContaining(`[ref=e1]`), + snapshot: expect.stringContaining(`[ref=e1]`), }); const result = await client.callTool({ @@ -391,7 +391,7 @@ test('browser_take_screenshot (viewport without snapshot)', async ({ startClient action: 'list', }, })).toHaveResponse({ - tabs: `- 0: (current) [] (about:blank)`, + result: `- Page URL: about:blank`, }); // This should work without requiring a snapshot since it's a viewport screenshot diff --git a/tests/mcp/secrets.spec.ts b/tests/mcp/secrets.spec.ts index bfd806a91e45f..6877a83f341ac 100644 --- a/tests/mcp/secrets.spec.ts +++ b/tests/mcp/secrets.spec.ts @@ -16,7 +16,7 @@ import fs from 'node:fs'; -import { test, expect, lowireMeta, parseResponse } from './fixtures'; +import { test, expect } from './fixtures'; test('browser_type', async ({ startClient, server }) => { const secretsFile = test.info().outputPath('secrets.env'); @@ -53,7 +53,7 @@ test('browser_type', async ({ startClient, server }) => { expect(response).toHaveResponse({ code: `await page.getByRole('textbox').fill(process.env['X-PASSWORD']); await page.getByRole('textbox').press('Enter');`, - pageState: expect.stringMatching(/textbox (\[active\] )?\[ref=e2\]: X-PASSWORD<\/secret>/), + snapshot: expect.stringMatching(/textbox (\[active\] )?\[ref=e2\]: X-PASSWORD<\/secret>/), }); } @@ -123,73 +123,6 @@ await page.getByRole('textbox', { name: 'Password' }).fill(process.env['X-PASSWO name: 'browser_snapshot', arguments: {}, })).toHaveResponse({ - pageState: expect.stringContaining(`- textbox \"Password\" [active] [ref=e6]: X-PASSWORD`), + snapshot: expect.stringContaining(`- textbox \"Password\" [active] [ref=e6]: X-PASSWORD`), }); }); - - -test('structured response', async ({ startClient, server }) => { - const secretsFile = test.info().outputPath('secrets.env'); - await fs.promises.writeFile(secretsFile, 'X-PASSWORD=password123'); - - const { client } = await startClient({ - args: ['--secrets', secretsFile], - }); - - server.setContent('/', ` - - - - - `, 'text/html'); - - await client.callTool({ - name: 'browser_navigate', - arguments: { - url: server.PREFIX, - _meta: lowireMeta - }, - }); - - { - const response = await client.callTool({ - name: 'browser_type', - arguments: { - element: 'textbox', - ref: 'e2', - text: 'X-PASSWORD', - submit: true, - _meta: lowireMeta - }, - }); - const { _meta } = parseResponse(response); - expect(_meta).toEqual({ - 'dev.lowire/history': [ - { category: 'code', content: `await page.getByRole('textbox').fill(process.env['X-PASSWORD']); -await page.getByRole('textbox').press('Enter');` }, - { category: 'console', content: expect.stringContaining('[LOG] Key pressed: Enter , Text: X-PASSWORD') }, - ], - 'dev.lowire/state': { - page: expect.stringMatching(/textbox (\[active\] )?\[ref=e2\]: X-PASSWORD<\/secret>/), - }, - }); - } - - { - const response = await client.callTool({ - name: 'browser_console_messages', - arguments: { - _meta: lowireMeta - }, - }); - const { _meta } = parseResponse(response); - expect(_meta).toEqual({ - 'dev.lowire/history': [ - { category: 'result', content: expect.stringContaining('[LOG] Key pressed: Enter , Text: X-PASSWORD') }, - ], - 'dev.lowire/state': { - page: expect.any(String), - }, - }); - } -}); diff --git a/tests/mcp/session-log.spec.ts b/tests/mcp/session-log.spec.ts index 7e072aa55bad3..eaffa5a026e8c 100644 --- a/tests/mcp/session-log.spec.ts +++ b/tests/mcp/session-log.spec.ts @@ -19,7 +19,9 @@ import path from 'path'; import { test, expect } from './fixtures'; -test('session log should record tool calls', async ({ startClient, server }, testInfo) => { +test('session log should record tool calls', async ({ startClient, server, mcpBrowser }, testInfo) => { + test.skip(mcpBrowser === 'webkit'); + const { client, stderr } = await startClient({ args: [ '--save-session', @@ -54,218 +56,30 @@ test('session log should record tool calls', async ({ startClient, server }, tes "url": "http://localhost:${server.PORT}" } \`\`\` -- Code -\`\`\`js -await page.goto('http://localhost:${server.PORT}'); -\`\`\` -- Snapshot: 001.snapshot.yml - - -### Tool call: browser_click -- Args -\`\`\`json -{ - "element": "Submit button", - "ref": "e2" -} -\`\`\` -- Code -\`\`\`js -await page.getByRole('button', { name: 'Submit' }).click(); -\`\`\` -- Snapshot: 002.snapshot.yml - -`); -}); - -test('session log should record user action', async ({ cdpServer, startClient }, testInfo) => { - const browserContext = await cdpServer.start(); - const { client, stderr } = await startClient({ - args: [ - '--save-session', - '--output-dir', testInfo.outputPath('output'), - `--cdp-endpoint=${cdpServer.endpoint}`, - ], - }); - - // Force browser context creation. - await client.callTool({ - name: 'browser_snapshot', - }); - - const [page] = browserContext.pages(); - await page.setContent(` - - - `); - - await page.getByRole('button', { name: 'Button 1' }).click(); - - const output = stderr().split('\n').filter(line => line.startsWith('Session: '))[0]; - const sessionFolder = output.substring('Session: '.length); - - await expect.poll(() => readSessionLog(sessionFolder)).toBe(` -### Tool call: browser_snapshot -- Args -\`\`\`json -{} -\`\`\` -- Snapshot: 001.snapshot.yml - - -### User action: click -- Args +- Result \`\`\`json { - "name": "click", - "ref": "e2", - "button": "left", - "modifiers": 0, - "clickCount": 1 + "code": "await page.goto('http://localhost:${server.PORT}');", + "page": "- Page URL: http://localhost:${server.PORT}/\\n- Page Title: Title", + "snapshot": "\`\`\`yaml\\n- button \\"Submit\\" [ref=e2]\\n\`\`\`" } \`\`\` -- Code -\`\`\`js -await page.getByRole('button', { name: 'Button 1' }).click(); -\`\`\` -- Snapshot: 002.snapshot.yml - -`); -}); - -test('session log should update user action', async ({ cdpServer, startClient }, testInfo) => { - const browserContext = await cdpServer.start(); - const { client, stderr } = await startClient({ - args: [ - '--save-session', - '--output-dir', testInfo.outputPath('output'), - `--cdp-endpoint=${cdpServer.endpoint}`, - ], - }); - - // Force browser context creation. - await client.callTool({ - name: 'browser_snapshot', - }); - const [page] = browserContext.pages(); - await page.setContent(` - - - `); - - await page.getByRole('button', { name: 'Button 1' }).dblclick(); - - const output = stderr().split('\n').filter(line => line.startsWith('Session: '))[0]; - const sessionFolder = output.substring('Session: '.length); - - await expect.poll(() => readSessionLog(sessionFolder)).toBe(` -### Tool call: browser_snapshot -- Args -\`\`\`json -{} -\`\`\` -- Snapshot: 001.snapshot.yml - - -### User action: click -- Args -\`\`\`json -{ - "name": "click", - "ref": "e2", - "button": "left", - "modifiers": 0, - "clickCount": 2 -} -\`\`\` -- Code -\`\`\`js -await page.getByRole('button', { name: 'Button 1' }).dblclick(); -\`\`\` -- Snapshot: 002.snapshot.yml - -`); -}); - -test('session log should record tool calls and user actions', async ({ cdpServer, startClient }, testInfo) => { - const browserContext = await cdpServer.start(); - const { client, stderr } = await startClient({ - args: [ - '--save-session', - '--output-dir', testInfo.outputPath('output'), - `--cdp-endpoint=${cdpServer.endpoint}`, - ], - }); - - const [page] = browserContext.pages(); - await page.setContent(` - - - `); - - await client.callTool({ - name: 'browser_snapshot', - }); - - // Manual action. - await page.getByRole('button', { name: 'Button 1' }).click(); - - // This is to simulate a delay after the user action before the tool action. - await new Promise(resolve => setTimeout(resolve, 1000)); - - // Tool action. - await client.callTool({ - name: 'browser_click', - arguments: { - element: 'Button 2', - ref: 'e3', - }, - }); - - const output = stderr().split('\n').filter(line => line.startsWith('Session: '))[0]; - const sessionFolder = output.substring('Session: '.length); - await expect.poll(() => readSessionLog(sessionFolder)).toBe(` -### Tool call: browser_snapshot -- Args -\`\`\`json -{} -\`\`\` -- Snapshot: 001.snapshot.yml - - -### User action: click +### Tool call: browser_click - Args \`\`\`json { - "name": "click", - "ref": "e2", - "button": "left", - "modifiers": 0, - "clickCount": 1 + "element": "Submit button", + "ref": "e2" } \`\`\` -- Code -\`\`\`js -await page.getByRole('button', { name: 'Button 1' }).click(); -\`\`\` -- Snapshot: 002.snapshot.yml - - -### Tool call: browser_click -- Args +- Result \`\`\`json { - "element": "Button 2", - "ref": "e3" + "code": "await page.getByRole('button', { name: 'Submit' }).click();", + "snapshot": "\`\`\`yaml\\n- button \\"Submit\\" [active] [ref=e2]\\n\`\`\`" } \`\`\` -- Code -\`\`\`js -await page.getByRole('button', { name: 'Button 2' }).click(); -\`\`\` -- Snapshot: 003.snapshot.yml - `); }); diff --git a/tests/mcp/snapshot-diff.spec.ts b/tests/mcp/snapshot-diff.spec.ts index 53e36985c1396..1e08c35517c7d 100644 --- a/tests/mcp/snapshot-diff.spec.ts +++ b/tests/mcp/snapshot-diff.spec.ts @@ -50,7 +50,7 @@ test('should return aria snapshot diff', async ({ client, server }) => { url: server.PREFIX, }, })).toHaveResponse({ - pageState: expect.stringContaining(` + snapshot: expect.stringContaining(` - button "Button 1" [ref=e2] - button "Button 2" [active] [ref=e3] - list [ref=e4]:${listitems}`), @@ -63,7 +63,9 @@ test('should return aria snapshot diff', async ({ client, server }) => { ref: 'e3', }, })).toHaveResponse({ - pageState: expect.not.stringContaining(`Page Snapshot`), + snapshot: `\`\`\`yaml + +\`\`\``, }); expect(await client.callTool({ @@ -73,26 +75,24 @@ test('should return aria snapshot diff', async ({ client, server }) => { ref: 'e2', }, })).toHaveResponse({ - pageState: expect.stringContaining(`Page Snapshot: -\`\`\`yaml + snapshot: `\`\`\`yaml - generic [ref=e1]: - button "Button 1" [active] [ref=e2] - button "Button 2new text" [ref=e105] - ref=e4 [unchanged] -\`\`\``), +\`\`\``, }); // browser_snapshot forces a full snapshot. expect(await client.callTool({ name: 'browser_snapshot', })).toHaveResponse({ - pageState: expect.stringContaining(`Page Snapshot: -\`\`\`yaml + snapshot: `\`\`\`yaml - generic [ref=e1]: - button "Button 1" [active] [ref=e2] - button "Button 2new text" [ref=e105] - list [ref=e4]:${listitems} -\`\`\``), +\`\`\``, }); }); @@ -136,7 +136,7 @@ test('should reset aria snapshot diff upon navigation', async ({ client, server url: server.PREFIX + '/before', }, })).toHaveResponse({ - pageState: expect.stringContaining(` + snapshot: expect.stringContaining(` - button "Button 1" [ref=e2] - button "Button 2" [active] [ref=e3]`), }); @@ -148,7 +148,7 @@ test('should reset aria snapshot diff upon navigation', async ({ client, server ref: 'e2', }, })).toHaveResponse({ - pageState: expect.stringContaining(` + snapshot: expect.stringContaining(` - button "Button 1" [ref=e2] - button "Button 2" [active] [ref=e3]`), }); diff --git a/tests/mcp/snapshot-mode.spec.ts b/tests/mcp/snapshot-mode.spec.ts index f2539d7e44bc5..327a7fd991170 100644 --- a/tests/mcp/snapshot-mode.spec.ts +++ b/tests/mcp/snapshot-mode.spec.ts @@ -31,7 +31,7 @@ test('should respect --snapshot-mode=full', async ({ startClient, server }) => { url: server.PREFIX, }, })).toHaveResponse({ - pageState: expect.stringContaining(` + snapshot: expect.stringContaining(` - button "Button 1" [ref=e2]`), }); @@ -45,7 +45,7 @@ test('should respect --snapshot-mode=full', async ({ startClient, server }) => { }`, }, })).toHaveResponse({ - pageState: expect.stringContaining(` + snapshot: expect.stringContaining(` - button "Button 1" [ref=e2] - button "Button 2" [ref=e3]`), }); @@ -64,7 +64,7 @@ test('should respect --snapshot-mode=incremental', async ({ startClient, server url: server.PREFIX, }, })).toHaveResponse({ - pageState: expect.stringContaining(` + snapshot: expect.stringContaining(` - button "Button 1" [ref=e2]`), }); @@ -78,7 +78,7 @@ test('should respect --snapshot-mode=incremental', async ({ startClient, server }`, }, })).toHaveResponse({ - pageState: expect.stringContaining(` + snapshot: expect.stringContaining(` - generic [active] [ref=e1]: - ref=e2 [unchanged] - button \"Button 2\" [ref=e3]`), @@ -98,8 +98,7 @@ test('should respect --snapshot-mode=none', async ({ startClient, server }) => { url: server.PREFIX, }, })).toHaveResponse({ - pageState: `- Page URL: ${server.PREFIX}/ -- Page Title:` + page: `- Page URL: ${server.PREFIX}/`, }); }); @@ -121,14 +120,9 @@ test('should respect snapshot[filename]', async ({ startClient, server }, testIn expect(await client.callTool({ name: 'browser_snapshot', arguments: { - filename: 'snapshot1.md', + filename: 'snapshot1.yml', }, - })).toHaveResponse({ - pageState: undefined, - files: expect.stringMatching(/\[Saved snapshot\]\(.*md\)/) - }); + })).toHaveTextResponse(expect.stringContaining('- [Page snapshot](output' + path.sep + 'snapshot1.yml)')); - expect(await fs.promises.readFile(path.join(outputDir, 'snapshot1.md'), 'utf8')).toContain(` -- button "Button 1" [ref=e2] -`); + expect(await fs.promises.readFile(path.join(outputDir, 'snapshot1.yml'), 'utf8')).toContain(`- button "Button 1" [ref=e2]`); }); diff --git a/tests/mcp/sse.spec.ts b/tests/mcp/sse.spec.ts index c1c8c9cdf54a5..5522bb1a10e22 100644 --- a/tests/mcp/sse.spec.ts +++ b/tests/mcp/sse.spec.ts @@ -254,7 +254,7 @@ test('sse transport shared context', async ({ serverEndpoint, server }) => { }); // Should have at least one tab (the one created by client1) - expect(tabsResult.content[0]?.text).toContain('tabs'); + expect(tabsResult.content[0]?.text).toContain('Title'); await client1.close(); diff --git a/tests/mcp/tabs.spec.ts b/tests/mcp/tabs.spec.ts index a0b40d824b783..7ad0af658a32b 100644 --- a/tests/mcp/tabs.spec.ts +++ b/tests/mcp/tabs.spec.ts @@ -40,7 +40,7 @@ test('list initial tabs', async ({ client }) => { action: 'list', }, })).toHaveResponse({ - tabs: `- 0: (current) [] (about:blank)`, + result: `- Page URL: about:blank`, }); }); @@ -52,33 +52,27 @@ test('list first tab', async ({ client }) => { action: 'list', }, })).toHaveResponse({ - tabs: `- 0: [] (about:blank) -- 1: (current) [Tab one] (data:text/html,Tab oneBody one)`, + result: `- 0: [](about:blank) +- 1: (current) [Tab one](data:text/html,Tab oneBody one)`, }); }); test('create new tab', async ({ client }) => { expect(await createTab(client, 'Tab one', 'Body one')).toHaveResponse({ - tabs: `- 0: [] (about:blank) -- 1: (current) [Tab one] (data:text/html,Tab oneBody one)`, - pageState: expect.stringContaining(`- Page URL: data:text/html,Tab oneBody one -- Page Title: Tab one -- Page Snapshot: -\`\`\`yaml + tabs: `- 0: [](about:blank) +- 1: (current) [Tab one](data:text/html,Tab oneBody one)`, + snapshot: `\`\`\`yaml - generic [active] [ref=e1]: Body one -\`\`\``), +\`\`\``, }); expect(await createTab(client, 'Tab two', 'Body two')).toHaveResponse({ - tabs: `- 0: [] (about:blank) -- 1: [Tab one] (data:text/html,Tab oneBody one) -- 2: (current) [Tab two] (data:text/html,Tab twoBody two)`, - pageState: expect.stringContaining(`- Page URL: data:text/html,Tab twoBody two -- Page Title: Tab two -- Page Snapshot: -\`\`\`yaml + tabs: `- 0: [](about:blank) +- 1: [Tab one](data:text/html,Tab oneBody one) +- 2: (current) [Tab two](data:text/html,Tab twoBody two)`, + snapshot: `\`\`\`yaml - generic [active] [ref=e1]: Body two -\`\`\``), +\`\`\``, }); }); @@ -93,15 +87,9 @@ test('select tab', async ({ client }) => { index: 1, }, })).toHaveResponse({ - tabs: `- 0: [] (about:blank) -- 1: (current) [Tab one] (data:text/html,Tab oneBody one) -- 2: [Tab two] (data:text/html,Tab twoBody two)`, - pageState: expect.stringContaining(`- Page URL: data:text/html,Tab oneBody one -- Page Title: Tab one -- Page Snapshot: -\`\`\`yaml -- generic [active] [ref=e1]: Body one -\`\`\``), + result: `- 0: [](about:blank) +- 1: (current) [Tab one](data:text/html,Tab oneBody one) +- 2: [Tab two](data:text/html,Tab twoBody two)`, }); expect(await client.callTool({ @@ -111,10 +99,9 @@ test('select tab', async ({ client }) => { index: 0, }, })).toHaveResponse({ - tabs: `- 0: (current) [] (about:blank) -- 1: [Tab one] (data:text/html,Tab oneBody one) -- 2: [Tab two] (data:text/html,Tab twoBody two)`, - pageState: expect.stringContaining(`- Page URL: about:blank`), + result: `- 0: (current) [](about:blank) +- 1: [Tab one](data:text/html,Tab oneBody one) +- 2: [Tab two](data:text/html,Tab twoBody two)`, }); }); @@ -129,14 +116,8 @@ test('close tab', async ({ client }) => { index: 2, }, })).toHaveResponse({ - tabs: `- 0: [] (about:blank) -- 1: (current) [Tab one] (data:text/html,Tab oneBody one)`, - pageState: expect.stringContaining(`- Page URL: data:text/html,Tab oneBody one -- Page Title: Tab one -- Page Snapshot: -\`\`\`yaml -- generic [active] [ref=e1]: Body one -\`\`\``), + result: `- 0: [](about:blank) +- 1: (current) [Tab one](data:text/html,Tab oneBody one)`, }); }); diff --git a/tests/mcp/terminal.spec.ts b/tests/mcp/terminal.spec.ts new file mode 100644 index 0000000000000..d5438b3dc7e89 --- /dev/null +++ b/tests/mcp/terminal.spec.ts @@ -0,0 +1,112 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import path from 'path'; +import { test, expect } from './fixtures'; + +test('terminal mode', async ({ startClient, server }) => { + server.setContent('/', formHtml, 'text/html'); + const outputDir = test.info().outputPath('output'); + + const { client } = await startClient({ + args: ['--codegen=none', '--output-mode=file', '--snapshot-mode=full', '--output-dir=' + outputDir], + }); + + expect(await client.callTool({ + name: 'browser_navigate', + arguments: { url: server.PREFIX }, + })).toHaveTextResponse(`- Page URL: ${server.PREFIX}/ + +- [Page snapshot](output${path.sep}snapshot-1.yml)`); + + expect(await client.callTool({ + name: 'browser_type', + arguments: { + ref: 'e4', + element: 'Name textbox', + text: 'John Doe', + }, + })).toHaveTextResponse(''); + + expect(await client.callTool({ + name: 'browser_type', + arguments: { + ref: 'e6', + element: 'Email textbox', + text: 'john.doe@example.com', + }, + })).toHaveTextResponse(''); + + expect(await client.callTool({ + name: 'browser_type', + arguments: { + ref: 'e8', + element: 'Age textbox', + text: '25', + }, + })).toHaveTextResponse(''); + + expect(await client.callTool({ + name: 'browser_select_option', + arguments: { + ref: 'e10', + element: 'Country select', + values: ['United States'], + }, + })).toHaveTextResponse(`- [Page snapshot](output${path.sep}snapshot-5.yml)`); + + expect(await client.callTool({ + name: 'browser_click', + arguments: { + ref: 'e12', + element: 'Subscribe checkbox', + }, + })).toHaveTextResponse(`- [Page snapshot](output${path.sep}snapshot-6.yml)`); +}); + + +const formHtml = ` + + + +
+ + + + + +
+ + `; diff --git a/tests/mcp/test-debug.spec.ts b/tests/mcp/test-debug.spec.ts index fb43b8445da2f..fbda63b595b68 100644 --- a/tests/mcp/test-debug.spec.ts +++ b/tests/mcp/test-debug.spec.ts @@ -119,7 +119,7 @@ test('test_debug (browser_snapshot/network/console)', async ({ startClient, serv expect(await client.callTool({ name: 'browser_snapshot', })).toHaveResponse({ - pageState: expect.stringContaining(`generic [active] [ref=e1]: Hello, world!`), + snapshot: expect.stringContaining(`generic [active] [ref=e1]: Hello, world!`), }); }); @@ -201,7 +201,7 @@ Try recovering from the error prior to continuing`); expect(await client.callTool({ name: 'browser_snapshot', })).toHaveResponse({ - pageState: expect.stringContaining(`- button \"Submit\" [ref=e2]`), + snapshot: expect.stringContaining(`- button \"Submit\" [ref=e2]`), }); expect(await client.callTool({ @@ -435,6 +435,6 @@ Error: expect(locator).toBeVisible() failed`)); name: 'browser_navigate', arguments: { url: server.HELLO_WORLD, intent: 'Go to hello world' }, })).toHaveResponse({ - pageState: expect.stringContaining(`- Page URL: ${server.HELLO_WORLD}\n- Page Title: Title2`), + page: expect.stringContaining(`- Page URL: ${server.HELLO_WORLD}\n- Page Title: Title2`), }); }); diff --git a/tests/mcp/timeouts.spec.ts b/tests/mcp/timeouts.spec.ts index 467f9fb611144..4f99cb5ffef28 100644 --- a/tests/mcp/timeouts.spec.ts +++ b/tests/mcp/timeouts.spec.ts @@ -41,7 +41,8 @@ test('action timeout (default)', async ({ server, startClient }) => { submit: true, }, })).toHaveResponse({ - result: expect.stringContaining(`Timeout 5000ms exceeded.`), + error: expect.stringContaining(`Timeout 5000ms exceeded.`), + isError: true, }); }); @@ -70,7 +71,8 @@ test('action timeout (custom)', async ({ startClient, server }) => { submit: true, }, })).toHaveResponse({ - result: expect.stringContaining(`Timeout 1234ms exceeded.`), + error: expect.stringContaining(`Timeout 1234ms exceeded.`), + isError: true, }); }); @@ -93,6 +95,7 @@ test('navigation timeout', async ({ startClient, server }) => { url: server.PREFIX + '/slow', }, })).toHaveResponse({ - result: expect.stringContaining(`Timeout 1234ms exceeded.`), + error: expect.stringContaining(`Timeout 1234ms exceeded.`), + isError: true, }); }); diff --git a/tests/mcp/type.spec.ts b/tests/mcp/type.spec.ts index 70ddef9d31c40..1932e8c4f72d1 100644 --- a/tests/mcp/type.spec.ts +++ b/tests/mcp/type.spec.ts @@ -44,7 +44,7 @@ test('browser_type', async ({ client, server }) => { expect(response).toHaveResponse({ code: `await page.getByRole('textbox').fill('Hi!'); await page.getByRole('textbox').press('Enter');`, - pageState: expect.stringMatching(/textbox (\[active\] )?\[ref=e2\]: Hi!/), + snapshot: expect.stringMatching(/textbox (\[active\] )?\[ref=e2\]: Hi!/), }); } @@ -79,7 +79,7 @@ test('browser_type (slowly)', async ({ client, server }) => { expect(response).toHaveResponse({ code: `await page.getByRole('textbox').pressSequentially('Hi!');`, - pageState: expect.stringMatching(/textbox (\[active\] )?\[ref=e2\]: Hi!/), + snapshot: expect.stringMatching(/textbox (\[active\] )?\[ref=e2\]: Hi!/), }); } const response = await client.callTool({ @@ -109,7 +109,7 @@ test('browser_type (no submit)', async ({ client, server }) => { }, }); expect(response).toHaveResponse({ - pageState: expect.stringContaining(`- textbox`), + snapshot: expect.stringContaining(`- textbox`), }); } { @@ -124,7 +124,7 @@ test('browser_type (no submit)', async ({ client, server }) => { expect(response).toHaveResponse({ code: expect.stringContaining(`fill('Hi!')`), // Should yield no snapshot. - pageState: expect.not.stringContaining(`- textbox`), + snapshot: expect.not.stringContaining(`- textbox`), }); } { diff --git a/tests/mcp/verify.spec.ts b/tests/mcp/verify.spec.ts index 392b8b5b819ea..e4b8da64bdde0 100644 --- a/tests/mcp/verify.spec.ts +++ b/tests/mcp/verify.spec.ts @@ -84,7 +84,7 @@ test('browser_verify_element_visible (not found)', async ({ client, server }) => }, })).toHaveResponse({ isError: true, - result: 'Element with role "button" and accessible name "Cancel" not found', + error: 'Element with role "button" and accessible name "Cancel" not found', }); }); @@ -150,7 +150,7 @@ test('browser_verify_text_visible (not found)', async ({ client, server }) => { }, })).toHaveResponse({ isError: true, - result: 'Text not found', + error: 'Text not found', }); }); @@ -276,7 +276,7 @@ test('browser_verify_list_visible (item not found)', async ({ client, server }) }, })).toHaveResponse({ isError: true, - result: 'Item "Cherry" not found', + error: 'Item "Cherry" not found', }); }); @@ -344,7 +344,7 @@ test('browser_verify_value (textbox wrong value)', async ({ client, server }) => }, })).toHaveResponse({ isError: true, - result: 'Expected value "Jane Smith", but got "John Doe"', + error: 'Expected value "Jane Smith", but got "John Doe"', }); }); @@ -428,7 +428,7 @@ test('browser_verify_value (checkbox wrong value)', async ({ client, server }) = }, })).toHaveResponse({ isError: true, - result: 'Expected value "false", but got "true"', + error: 'Expected value "false", but got "true"', }); }); diff --git a/tests/mcp/wait.spec.ts b/tests/mcp/wait.spec.ts index 02f631020c539..a0c9d2ea6629e 100644 --- a/tests/mcp/wait.spec.ts +++ b/tests/mcp/wait.spec.ts @@ -35,7 +35,7 @@ test('browser_wait_for(text)', async ({ client, server }) => { name: 'browser_navigate', arguments: { url: server.PREFIX }, })).toHaveResponse({ - pageState: expect.stringContaining(`- generic [ref=e3]: Text to disappear`), + snapshot: expect.stringContaining(`- generic [ref=e3]: Text to disappear`), }); await client.callTool({ @@ -54,7 +54,7 @@ test('browser_wait_for(text)', async ({ client, server }) => { expect(await client.callTool({ name: 'browser_snapshot', })).toHaveResponse({ - pageState: expect.stringContaining(`- generic [ref=e3]: Text to appear`), + snapshot: expect.stringContaining(`- generic [ref=e3]: Text to appear`), }); }); @@ -77,7 +77,7 @@ test('browser_wait_for(textGone)', async ({ client, server }) => { name: 'browser_navigate', arguments: { url: server.PREFIX }, })).toHaveResponse({ - pageState: expect.stringContaining(`- generic [ref=e3]: Text to disappear`), + snapshot: expect.stringContaining(`- generic [ref=e3]: Text to disappear`), }); await client.callTool({ @@ -96,7 +96,7 @@ test('browser_wait_for(textGone)', async ({ client, server }) => { expect(await client.callTool({ name: 'browser_snapshot', })).toHaveResponse({ - pageState: expect.stringContaining(`- generic [ref=e3]: Text to appear`), + snapshot: expect.stringContaining(`- generic [ref=e3]: Text to appear`), }); }); diff --git a/tests/mcp/webdriver.spec.ts b/tests/mcp/webdriver.spec.ts index 9467056f9efff..99496cb187864 100644 --- a/tests/mcp/webdriver.spec.ts +++ b/tests/mcp/webdriver.spec.ts @@ -35,6 +35,6 @@ test('do not falsely advertise user agent as a test driver', async ({ client, se url: server.PREFIX, }, })).toHaveResponse({ - pageState: expect.stringContaining(`webdriver: false`), + snapshot: expect.stringContaining(`webdriver: false`), }); });