[pull] main from microsoft:main

docs/src/api/class-page.md

@@ -722,6 +722,11 @@ Initialize page agent with the llm provider and cache.
   - `cacheFile` ?<[string]> Cache file to use/generate code for performed actions into. Cache is not used if not specified (default).
   - `cacheOutFile` ?<[string]> When specified, generated entries are written into the `cacheOutFile` instead of updating the `cacheFile`.
 
+### option: Page.agent.expect
+* since: v1.58
+- `expect` <[Object]>
+  - `timeout` ?<[int]> Default timeout for expect calls in milliseconds, defaults to 5000ms.
+
 ### option: Page.agent.limits
 * since: v1.58
 - `limits` <[Object]>

docs/src/api/class-pageagent.md

@@ -35,6 +35,12 @@ await agent.expect('"0 items" to be reported');
 
 Expectation to assert.
 
+### option: PageAgent.expect.timeout
+* since: v1.58
+- `timeout` <[float]>
+
+Expect timeout in milliseconds. Defaults to `5000`. The default value can be changed via `expect.timeout` option in the config, or by specifying the `expect` property of the [`option: Page.agent.expect`] option. Pass `0` to disable timeout.
+
 ### option: PageAgent.expect.-inline- = %%-page-agent-call-options-v1.58-%%
 * since: v1.58
 
@@ -68,6 +74,13 @@ Task to perform using agentic loop.
 * since: v1.58
 - `schema` <[z.ZodSchema]>
 
+### option: PageAgent.extract.timeout
+* since: v1.58
+- `timeout` <[float]>
+
+Extract timeout in milliseconds. Defaults to `5000`. The default value can be changed via `actionTimeout` option in the config, or by using the [`method: BrowserContext.setDefaultTimeout`] or
+[`method: Page.setDefaultTimeout`] methods. Pass `0` to disable timeout.
+
 ### option: PageAgent.extract.-inline- = %%-page-agent-call-options-v1.58-%%
 * since: v1.58
 
@@ -94,6 +107,13 @@ await agent.perform('Click submit button');
 
 Task to perform using agentic loop.
 
+### option: PageAgent.perform.timeout
+* since: v1.58
+- `timeout` <[float]>
+
+Perform timeout in milliseconds. Defaults to `5000`. The default value can be changed via `actionTimeout` option in the config, or by using the [`method: BrowserContext.setDefaultTimeout`] or
+[`method: Page.setDefaultTimeout`] methods. Pass `0` to disable timeout.
+
 ### option: PageAgent.perform.-inline- = %%-page-agent-call-options-v1.58-%%
 * since: v1.58
 

docs/src/api/params.md

@@ -396,18 +396,11 @@ Maximum number of agentic actions to generate, defaults to context-wide value sp
 
 Maximum number of retries when generating each action, defaults to context-wide value specified in `agent` property.
 
-## page-agent-timeout
-* since: v1.58
-- `timeout` <[int]>
-
-Request timeout in milliseconds. Defaults to action timeout. Pass `0` to disable timeout.
-
 ## page-agent-call-options-v1.58
 - %%-page-agent-cache-key-%%
 - %%-page-agent-max-tokens-%%
 - %%-page-agent-max-actions-%%
 - %%-page-agent-max-action-retries-%%
-- %%-page-agent-timeout-%%
 
 ## fetch-param-url
 - `url` <[string]>

packages/injected/src/ariaSnapshot.ts

@@ -39,6 +39,7 @@ let lastRef = 0;
 export type AriaTreeOptions = {
   mode: 'ai' | 'expect' | 'codegen' | 'autoexpect';
   refPrefix?: string;
+  doNotRenderActive?: boolean;
 };
 
 type InternalOptions = {
@@ -59,7 +60,7 @@ function toInternalOptions(options: AriaTreeOptions): InternalOptions {
       refs: 'interactable',
       refPrefix: options.refPrefix,
       includeGenericRole: true,
-      renderActive: true,
+      renderActive: !options.doNotRenderActive,
       renderCursorPointer: true,
     };
   }

packages/injected/src/injectedScript.ts

@@ -50,8 +50,7 @@ import type { Builtins } from './utilityScript';
 export type FrameExpectParams = Omit<channels.FrameExpectParams, 'expectedValue' | 'timeout'> & {
   expectedValue?: any;
   timeoutForLogs?: number;
-  explicitTimeout?: number;
-  noPreChecks?: boolean;
+  noAutoWaiting?: boolean;
 };
 
 export type ElementState = 'visible' | 'hidden' | 'enabled' | 'disabled' | 'editable' | 'checked' | 'unchecked' | 'indeterminate' | 'stable';

packages/playwright-client/types/types.d.ts

@@ -2112,6 +2112,13 @@ export interface Page {
       cacheOutFile?: string;
     };
 
+    expect?: {
+      /**
+       * Default timeout for expect calls in milliseconds, defaults to 5000ms.
+       */
+      timeout?: number;
+    };
+
     /**
      * Limits to use for the agentic loop.
      */
@@ -5440,7 +5447,10 @@ export interface PageAgent {
     maxTokens?: number;
 
     /**
-     * Request timeout in milliseconds. Defaults to action timeout. Pass `0` to disable timeout.
+     * Expect timeout in milliseconds. Defaults to `5000`. The default value can be changed via `expect.timeout` option in
+     * the config, or by specifying the `expect` property of the
+     * [`expect`](https://playwright.dev/docs/api/class-page#page-agent-option-expect) option. Pass `0` to disable
+     * timeout.
      */
     timeout?: number;
   }): Promise<void>;
@@ -5482,7 +5492,11 @@ export interface PageAgent {
     maxTokens?: number;
 
     /**
-     * Request timeout in milliseconds. Defaults to action timeout. Pass `0` to disable timeout.
+     * Perform timeout in milliseconds. Defaults to `5000`. The default value can be changed via `actionTimeout` option in
+     * the config, or by using the
+     * [browserContext.setDefaultTimeout(timeout)](https://playwright.dev/docs/api/class-browsercontext#browser-context-set-default-timeout)
+     * or [page.setDefaultTimeout(timeout)](https://playwright.dev/docs/api/class-page#page-set-default-timeout) methods.
+     * Pass `0` to disable timeout.
      */
     timeout?: number;
   }): Promise<{

packages/playwright-core/src/client/page.ts

@@ -854,6 +854,7 @@ export class Page extends ChannelOwner<channels.PageChannel> implements api.Page
       apiKey: options.provider?.apiKey,
       apiTimeout: options.provider?.apiTimeout,
       apiCacheFile: (options.provider as any)?._apiCacheFile,
+      doNotRenderActive: (options as any)._doNotRenderActive,
       model: options.provider?.model,
       cacheFile: options.cache?.cacheFile,
       cacheOutFile: options.cache?.cacheOutFile,
@@ -864,7 +865,9 @@ export class Page extends ChannelOwner<channels.PageChannel> implements api.Page
       systemPrompt: options.systemPrompt,
     };
     const { agent } = await this._channel.agent(params);
-    return PageAgent.from(agent);
+    const pageAgent = PageAgent.from(agent);
+    pageAgent._expectTimeout = options?.expect?.timeout;
+    return pageAgent;
   }
 
   async _snapshotForAI(options: TimeoutOptions & { track?: string } = {}): Promise<{ full: string, incremental?: string }> {

packages/playwright-core/src/client/pageAgent.ts

@@ -22,14 +22,9 @@ import { Page } from './page';
 import type * as api from '../../types/types';
 import type * as channels from '@protocol/channels';
 
-type PageAgentOptions = {
-  maxTokens?: number;
-  maxTurns?: number;
-  cacheKey?: string;
-};
-
 export class PageAgent extends ChannelOwner<channels.PageAgentChannel> implements api.PageAgent {
   private _page: Page;
+  _expectTimeout?: number;
 
   static from(channel: channels.PageAgentChannel): PageAgent {
     return (channel as any)._object;
@@ -41,17 +36,20 @@ export class PageAgent extends ChannelOwner<channels.PageAgentChannel> implement
     this._channel.on('turn', params => this.emit(Events.Page.AgentTurn, params));
   }
 
-  async expect(expectation: string, options: PageAgentOptions = {}) {
-    await this._channel.expect({ expectation, ...options });
+  async expect(expectation: string, options: channels.PageAgentExpectOptions = {}) {
+    const timeout = options.timeout ?? this._expectTimeout ?? 5000;
+    await this._channel.expect({ expectation, ...options, timeout });
   }
 
-  async perform(task: string, options: PageAgentOptions = {}) {
-    const { usage } = await this._channel.perform({ task, ...options });
+  async perform(task: string, options: channels.PageAgentPerformOptions = {}) {
+    const timeout = this._page._timeoutSettings.timeout(options);
+    const { usage } = await this._channel.perform({ task, ...options, timeout });
     return { usage };
   }
 
-  async extract<Schema extends any>(query: string, schema: Schema, options: PageAgentOptions = {}): Promise<{ result: any, usage: channels.AgentUsage }> {
-    const { result, usage } = await this._channel.extract({ query, schema: this._page._platform.zodToJsonSchema(schema), ...options });
+  async extract<Schema extends any>(query: string, schema: Schema, options: channels.PageAgentExtractOptions = {}): Promise<{ result: any, usage: channels.AgentUsage }> {
+    const timeout = this._page._timeoutSettings.timeout(options);
+    const { result, usage } = await this._channel.extract({ query, schema: this._page._platform.zodToJsonSchema(schema), ...options, timeout });
     return { result, usage };
   }
 

packages/playwright-core/src/protocol/validator.ts

@@ -1495,6 +1495,7 @@ scheme.PageAgentParams = tObject({
   apiCacheFile: tOptional(tString),
   cacheFile: tOptional(tString),
   cacheOutFile: tOptional(tString),
+  doNotRenderActive: tOptional(tBoolean),
   maxActions: tOptional(tInt),
   maxActionRetries: tOptional(tInt),
   maxTokens: tOptional(tInt),

packages/playwright-core/src/server/agent/actionRunner.ts

@@ -143,14 +143,11 @@ async function innerRunAction(progress: Progress, mode: 'generate' | 'run', page
 }
 
 async function runExpect(frame: Frame, progress: Progress, mode: 'generate' | 'run', selector: string | undefined, options: FrameExpectParams, expected: string | RegExp, matcherName: string, expectation: string) {
-  // Pass explicit timeout to limit the single expect action inside the overall "agentic expect" multi-step progress.
-  const timeout = expectTimeout(mode);
   const result = await frame.expect(progress, selector, {
     ...options,
-    timeoutForLogs: timeout,
-    explicitTimeout: timeout,
-    // Disable pre-checks to avoid them timing out, model has seen the snapshot anyway.
-    noPreChecks: mode === 'generate',
+    // When generating, we want the expect to pass or fail immediately and give feedback to the model.
+    noAutoWaiting: mode === 'generate',
+    timeoutForLogs: mode === 'generate' ? undefined : progress.timeout,
   });
   if (!result.matches === !options.isNot) {
     const received = matcherName === 'toMatchAriaSnapshot' ? '\n' + result.received.raw : result.received;
@@ -162,7 +159,7 @@ async function runExpect(frame: Frame, progress: Progress, mode: 'generate' | 'r
       expectation,
       locator: selector ? asLocatorDescription('javascript', selector) : undefined,
       timedOut: result.timedOut,
-      timeout,
+      timeout: mode === 'generate' ? undefined : progress.timeout,
       printedExpected: options.isNot ? `Expected${expectedSuffix}: not ${expectedDisplay}` : `Expected${expectedSuffix}: ${expectedDisplay}`,
       printedReceived: result.errorMessage ? '' : `Received: ${received}`,
       errorMessage: result.errorMessage,
@@ -266,7 +263,7 @@ export function traceParamsForAction(progress: Progress, action: actions.Action,
           expression: 'to.have.value',
           expectedText,
           isNot: !!action.isNot,
-          timeout: expectTimeout(mode),
+          timeout,
         };
         return { type: 'Frame', method: 'expect', title: 'Expect Value', params };
       } else if (action.type === 'checkbox' || action.type === 'radio') {
@@ -275,7 +272,7 @@ export function traceParamsForAction(progress: Progress, action: actions.Action,
           selector: action.selector,
           expression: 'to.be.checked',
           isNot: !!action.isNot,
-          timeout: expectTimeout(mode),
+          timeout,
         };
         return { type: 'Frame', method: 'expect', title: 'Expect Checked', params };
       } else {
@@ -287,7 +284,7 @@ export function traceParamsForAction(progress: Progress, action: actions.Action,
         selector: action.selector,
         expression: 'to.be.visible',
         isNot: !!action.isNot,
-        timeout: expectTimeout(mode),
+        timeout,
       };
       return { type: 'Frame', method: 'expect', title: 'Expect Visible', params };
     }
@@ -298,7 +295,7 @@ export function traceParamsForAction(progress: Progress, action: actions.Action,
         expression: 'to.match.snapshot',
         expectedText: [],
         isNot: !!action.isNot,
-        timeout: expectTimeout(mode),
+        timeout,
       };
       return { type: 'Frame', method: 'expect', title: 'Expect Aria Snapshot', params };
     }
@@ -310,7 +307,7 @@ export function traceParamsForAction(progress: Progress, action: actions.Action,
         expression: 'to.have.url',
         expectedText,
         isNot: !!action.isNot,
-        timeout: expectTimeout(mode),
+        timeout,
       };
       return { type: 'Frame', method: 'expect', title: 'Expect URL', params };
     }
@@ -321,7 +318,7 @@ export function traceParamsForAction(progress: Progress, action: actions.Action,
         expression: 'to.have.title',
         expectedText,
         isNot: !!action.isNot,
-        timeout: expectTimeout(mode),
+        timeout,
       };
       return { type: 'Frame', method: 'expect', title: 'Expect Title', params };
     }
@@ -341,7 +338,3 @@ function callMetadataForAction(progress: Progress, frame: Frame, action: actions
   };
   return callMetadata;
 }
-
-function expectTimeout(mode: 'generate' | 'run') {
-  return mode === 'generate' ? 0 : 5000;
-}

packages/playwright-core/src/server/agent/context.ts

@@ -132,22 +132,27 @@ export class Context {
     return result;
   }
 
+  async takeSnapshot(progress: Progress) {
+    const { full } = await this.page.snapshotForAI(progress, { doNotRenderActive: this.agentParams.doNotRenderActive });
+    // TODO: it seems like redactText should be here.
+    return full;
+  }
+
   async snapshotResult(progress: Progress, error?: Error): Promise<loopTypes.ToolResult> {
-    let { full } = await this.page.snapshotForAI(progress);
-    full = this._redactText(full);
+    const snapshot = this._redactText(await this.takeSnapshot(progress));
 
     const text: string[] = [];
     if (error)
       text.push(`# Error\n${stripAnsiEscapes(error.message)}`);
     else
       text.push(`# Success`);
 
-    text.push(`# Page snapshot\n${full}`);
+    text.push(`# Page snapshot\n${snapshot}`);
 
     return {
       _meta: {
         'dev.lowire/state': {
-          'Page snapshot': full
+          'Page snapshot': snapshot
         },
         'dev.lowire/history': error ? [{
           category: 'error',

packages/playwright-core/src/server/agent/pageAgent.ts

@@ -50,7 +50,7 @@ export async function pageAgentPerform(progress: Progress, context: Context, use
 ### Task
 ${userTask}
 `;
-
+  progress.disableTimeout();
   await runLoop(progress, context, performTools, task, undefined, callParams);
   await updateCache(context, cacheKey);
 }
@@ -68,7 +68,7 @@ export async function pageAgentExpect(progress: Progress, context: Context, expe
 ### Expectation
 ${expectation}
 `;
-
+  progress.disableTimeout();
   await runLoop(progress, context, expectTools, task, undefined, callParams);
   await updateCache(context, cacheKey);
 }
@@ -88,19 +88,20 @@ ${query}`;
 async function runLoop(progress: Progress, context: Context, toolDefinitions: ToolDefinition[], userTask: string, resultSchema: loopTypes.Schema | undefined, params: CallParams): Promise<{
   result: any
 }> {
-  const { page } = context;
   if (!context.agentParams.api || !context.agentParams.model)
     throw new Error(`This action requires the API and API key to be set on the page agent. Did you mean to --run-agents=missing?`);
   if (!context.agentParams.apiKey)
     throw new Error(`This action requires API key to be set on the page agent.`);
+  if (context.agentParams.apiEndpoint && !URL.canParse(context.agentParams.apiEndpoint))
+    throw new Error(`Agent API endpoint "${context.agentParams.apiEndpoint}" is not a valid URL.`);
 
-  const { full } = await page.snapshotForAI(progress);
+  const snapshot = await context.takeSnapshot(progress);
   const { tools, callTool, reportedResult, refusedToPerformReason } = toolsForLoop(progress, context, toolDefinitions, { resultSchema, refuseToPerform: 'allow' });
   const secrets = Object.fromEntries((context.agentParams.secrets || [])?.map(s => ([s.name, s.value])));
 
   const apiCacheTextBefore = context.agentParams.apiCacheFile ?
     await fs.promises.readFile(context.agentParams.apiCacheFile, 'utf-8').catch(() => '{}') : '{}';
-  const apiCacheBefore = JSON.parse(apiCacheTextBefore);
+  const apiCacheBefore = JSON.parse(apiCacheTextBefore || '{}');
 
   const loop = new Loop({
     api: context.agentParams.api as any,
@@ -136,7 +137,7 @@ async function runLoop(progress: Progress, context: Context, toolDefinitions: To
     task.push('');
   }
   task.push('### Page snapshot');
-  task.push(full);
+  task.push(snapshot);
   task.push('');
 
   const { error, usage } = await loop.run(task.join('\n'), { signal: progress.signal });
@@ -205,7 +206,13 @@ const allCaches = new Map<string, Cache>();
 async function cachedActions(cacheFile: string): Promise<Cache> {
   let cache = allCaches.get(cacheFile);
   if (!cache) {
-    const json = await fs.promises.readFile(cacheFile, 'utf-8').then(text => JSON.parse(text)).catch(() => ({}));
+    const content = await fs.promises.readFile(cacheFile, 'utf-8').catch(() => '');
+    let json: any;
+    try {
+      json = JSON.parse(content.trim() || '{}');
+    } catch (error) {
+      throw new Error(`Failed to parse cache file ${cacheFile}:\n${error.message}`);
+    }
     const parsed = actions.cachedActionsSchema.safeParse(json);
     if (parsed.error)
       throw new Error(`Failed to parse cache file ${cacheFile}:\n${zod.prettifyError(parsed.error)}`);