diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js index 99b44f206..653b75b97 100644 --- a/src/chrome/src/agent/agent.js +++ b/src/chrome/src/agent/agent.js @@ -6627,7 +6627,8 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d } let coordCheck = { kind: 'none' }; if (fnName === 'click' && fnArgs?.x != null && fnArgs?.y != null) { - coordCheck = this._checkCoordClickLoop(tabId, fnArgs.x, fnArgs.y); + const point = this._resolveClickCoordsForLoop(tabId, fnArgs); + coordCheck = this._checkCoordClickLoop(tabId, point.x, point.y); } const axReadCheck = this._checkAccessibilityReadLoop(tabId, fnName, fnArgs, toolResult); const scrollCheck = this._checkNoProgressScroll(tabId, fnName, fnArgs, toolResult); @@ -8742,6 +8743,21 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d }; } + /** + * Resolve the point a click call will actually dispatch at, for loop + * detection. Loop detection must measure the same CSS-pixel point the + * click dispatches: under downscaled screenshots the model's image-pixel + * nudges collapse to the same CSS pixel and the detector must see that, + * otherwise "slightly different spot" clicks never register as the loop + * they are. Non-numeric args fall back to Number() so the batch guard + * (x != null) and the detector keep their existing shape. + */ + _resolveClickCoordsForLoop(tabId, args) { + const mapped = this._screenshotClickCoords(tabId, args); + if (!mapped) return { x: Number(args.x), y: Number(args.y) }; + return { x: mapped.x, y: mapped.y }; + } + _coordinateReconciliationDiagnostic(point, resolution, clickPath, fallbackReason) { const target = resolution?.success === true ? resolution.semanticTarget : null; const resolved = clickPath === 'semantic'; diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js index 89c239aa5..e01294308 100644 --- a/src/firefox/src/agent/agent.js +++ b/src/firefox/src/agent/agent.js @@ -5568,7 +5568,8 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d } let coordCheck = { kind: 'none' }; if (fnName === 'click' && fnArgs?.x != null && fnArgs?.y != null) { - coordCheck = this._checkCoordClickLoop(tabId, fnArgs.x, fnArgs.y); + const point = this._resolveClickCoordsForLoop(tabId, fnArgs); + coordCheck = this._checkCoordClickLoop(tabId, point.x, point.y); } const axReadCheck = this._checkAccessibilityReadLoop(tabId, fnName, fnArgs, toolResult); const scrollCheck = this._checkNoProgressScroll(tabId, fnName, fnArgs, toolResult); @@ -6159,6 +6160,21 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d }; } + /** + * Resolve the point a click call will actually dispatch at, for loop + * detection. Loop detection must measure the same CSS-pixel point the + * click dispatches: under downscaled screenshots the model's image-pixel + * nudges collapse to the same CSS pixel and the detector must see that, + * otherwise "slightly different spot" clicks never register as the loop + * they are. Non-numeric args fall back to Number() so the batch guard + * (x != null) and the detector keep their existing shape. + */ + _resolveClickCoordsForLoop(tabId, args) { + const mapped = this._screenshotClickCoords(tabId, args); + if (!mapped) return { x: Number(args.x), y: Number(args.y) }; + return { x: mapped.x, y: mapped.y }; + } + _coordinateReconciliationDiagnostic(point, resolution, clickPath, fallbackReason) { const target = resolution?.success === true ? resolution.semanticTarget : null; const resolved = clickPath === 'semantic'; diff --git a/test/run.js b/test/run.js index cbab07c8a..339b899c6 100644 --- a/test/run.js +++ b/test/run.js @@ -11733,6 +11733,87 @@ test('screenshot click scale: from_screenshot converts image px to CSS px', () = } }); +test('coord click loop: helper resolves the CSS-pixel point the click dispatches', () => { + for (const AgentClass of [AgentCh, AgentFx]) { + const agent = new AgentClass({}); + const tabId = 9; + agent.screenshotClickScale.set(tabId, { scaleX: 2, scaleY: 2 }); + + // from_screenshot:true converts image px to the CSS px the click fires at. + assert.deepEqual( + agent._resolveClickCoordsForLoop(tabId, { x: 100, y: 200, from_screenshot: true }), + { x: 200, y: 400 }, + `${AgentClass.name}: from_screenshot coords not converted`, + ); + + // Without the flag, coords pass through unchanged. + assert.deepEqual( + agent._resolveClickCoordsForLoop(tabId, { x: 100, y: 200 }), + { x: 100, y: 200 }, + `${AgentClass.name}: non-screenshot coords not passed through`, + ); + + // Flag set but no stored scale: no conversion. + agent.screenshotClickScale.delete(tabId); + assert.deepEqual( + agent._resolveClickCoordsForLoop(tabId, { x: 100, y: 200, from_screenshot: true }), + { x: 100, y: 200 }, + `${AgentClass.name}: no stored scale should pass through`, + ); + } +}); + +test('coord click loop: batch flow feeds converted CSS px into the loop detector', async () => { + for (const [label, AgentClass] of [['chrome', AgentCh], ['firefox', AgentFx]]) { + const agent = new AgentClass({ getVisionProvider: async () => null }); + const tabId = label === 'chrome' ? 52705 : 52706; + agent._persist = () => {}; + agent._skipPermissionGate = true; + agent.screenshotClickScale.set(tabId, { scaleX: 2, scaleY: 2 }); + const seen = []; + agent._checkCoordClickLoop = (tid, x, y) => { + seen.push([x, y]); + return { kind: 'none' }; + }; + agent.executeTool = async () => ({ success: true }); + agent._preflightRichTextToolbarTarget = async () => ({ block: null }); + + const messages = []; + await agent._executeToolBatch( + tabId, + [ + { + id: 'loop_click_1', + function: { name: 'click', arguments: JSON.stringify({ x: 100, y: 200, from_screenshot: true }) }, + }, + { + id: 'loop_click_2', + function: { name: 'click', arguments: JSON.stringify({ x: 101, y: 201, from_screenshot: true }) }, + }, + { + id: 'loop_click_3', + function: { name: 'click', arguments: JSON.stringify({ x: 50, y: 60 }) }, + }, + ], + messages, + () => {}, + { supportsVision: false }, + null, + new Set(['click']), + 1, + ); + + // The downscaled-screenshot clicks land on CSS pixels, not image pixels: + // (100,200) and (101,201) are distinct image pixels but collapse to the + // same CSS bucket, so the detector must be fed the converted values. + assert.deepEqual( + seen, + [[200, 400], [202, 402], [50, 60]], + `${label}: detector did not see the converted click points`, + ); + } +}); + test('visual target resolution stays private across Chrome and Firefox tools and prompts', () => { for (const [label, getTools, prompts] of [ ['chrome', getToolsForModeCh, [