Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 17 additions & 1 deletion src/chrome/src/agent/agent.js
Original file line number Diff line number Diff line change
Expand Up @@ -6627,7 +6627,8 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
}
let coordCheck = { kind: 'none' };
if (fnName === 'click' && fnArgs?.x != null && fnArgs?.y != null) {
coordCheck = this._checkCoordClickLoop(tabId, fnArgs.x, fnArgs.y);
const point = this._resolveClickCoordsForLoop(tabId, fnArgs);
coordCheck = this._checkCoordClickLoop(tabId, point.x, point.y);
}
const axReadCheck = this._checkAccessibilityReadLoop(tabId, fnName, fnArgs, toolResult);
const scrollCheck = this._checkNoProgressScroll(tabId, fnName, fnArgs, toolResult);
Expand Down Expand Up @@ -8742,6 +8743,21 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
};
}

/**
* Resolve the point a click call will actually dispatch at, for loop
* detection. Loop detection must measure the same CSS-pixel point the
* click dispatches: under downscaled screenshots the model's image-pixel
* nudges collapse to the same CSS pixel and the detector must see that,
* otherwise "slightly different spot" clicks never register as the loop
* they are. Non-numeric args fall back to Number() so the batch guard
* (x != null) and the detector keep their existing shape.
*/
_resolveClickCoordsForLoop(tabId, args) {
const mapped = this._screenshotClickCoords(tabId, args);
if (!mapped) return { x: Number(args.x), y: Number(args.y) };
return { x: mapped.x, y: mapped.y };
}

_coordinateReconciliationDiagnostic(point, resolution, clickPath, fallbackReason) {
const target = resolution?.success === true ? resolution.semanticTarget : null;
const resolved = clickPath === 'semantic';
Expand Down
18 changes: 17 additions & 1 deletion src/firefox/src/agent/agent.js
Original file line number Diff line number Diff line change
Expand Up @@ -5568,7 +5568,8 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
}
let coordCheck = { kind: 'none' };
if (fnName === 'click' && fnArgs?.x != null && fnArgs?.y != null) {
coordCheck = this._checkCoordClickLoop(tabId, fnArgs.x, fnArgs.y);
const point = this._resolveClickCoordsForLoop(tabId, fnArgs);
coordCheck = this._checkCoordClickLoop(tabId, point.x, point.y);
}
const axReadCheck = this._checkAccessibilityReadLoop(tabId, fnName, fnArgs, toolResult);
const scrollCheck = this._checkNoProgressScroll(tabId, fnName, fnArgs, toolResult);
Expand Down Expand Up @@ -6159,6 +6160,21 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
};
}

/**
* Resolve the point a click call will actually dispatch at, for loop
* detection. Loop detection must measure the same CSS-pixel point the
* click dispatches: under downscaled screenshots the model's image-pixel
* nudges collapse to the same CSS pixel and the detector must see that,
* otherwise "slightly different spot" clicks never register as the loop
* they are. Non-numeric args fall back to Number() so the batch guard
* (x != null) and the detector keep their existing shape.
*/
_resolveClickCoordsForLoop(tabId, args) {
const mapped = this._screenshotClickCoords(tabId, args);
if (!mapped) return { x: Number(args.x), y: Number(args.y) };
return { x: mapped.x, y: mapped.y };
}

_coordinateReconciliationDiagnostic(point, resolution, clickPath, fallbackReason) {
const target = resolution?.success === true ? resolution.semanticTarget : null;
const resolved = clickPath === 'semantic';
Expand Down
81 changes: 81 additions & 0 deletions test/run.js
Original file line number Diff line number Diff line change
Expand Up @@ -11733,6 +11733,87 @@ test('screenshot click scale: from_screenshot converts image px to CSS px', () =
}
});

test('coord click loop: helper resolves the CSS-pixel point the click dispatches', () => {
for (const AgentClass of [AgentCh, AgentFx]) {
const agent = new AgentClass({});
const tabId = 9;
agent.screenshotClickScale.set(tabId, { scaleX: 2, scaleY: 2 });

// from_screenshot:true converts image px to the CSS px the click fires at.
assert.deepEqual(
agent._resolveClickCoordsForLoop(tabId, { x: 100, y: 200, from_screenshot: true }),
{ x: 200, y: 400 },
`${AgentClass.name}: from_screenshot coords not converted`,
);

// Without the flag, coords pass through unchanged.
assert.deepEqual(
agent._resolveClickCoordsForLoop(tabId, { x: 100, y: 200 }),
{ x: 100, y: 200 },
`${AgentClass.name}: non-screenshot coords not passed through`,
);

// Flag set but no stored scale: no conversion.
agent.screenshotClickScale.delete(tabId);
assert.deepEqual(
agent._resolveClickCoordsForLoop(tabId, { x: 100, y: 200, from_screenshot: true }),
{ x: 100, y: 200 },
`${AgentClass.name}: no stored scale should pass through`,
);
}
});

test('coord click loop: batch flow feeds converted CSS px into the loop detector', async () => {
for (const [label, AgentClass] of [['chrome', AgentCh], ['firefox', AgentFx]]) {
const agent = new AgentClass({ getVisionProvider: async () => null });
const tabId = label === 'chrome' ? 52705 : 52706;
agent._persist = () => {};
agent._skipPermissionGate = true;
agent.screenshotClickScale.set(tabId, { scaleX: 2, scaleY: 2 });
const seen = [];
agent._checkCoordClickLoop = (tid, x, y) => {
seen.push([x, y]);
return { kind: 'none' };
};
agent.executeTool = async () => ({ success: true });
agent._preflightRichTextToolbarTarget = async () => ({ block: null });

const messages = [];
await agent._executeToolBatch(
tabId,
[
{
id: 'loop_click_1',
function: { name: 'click', arguments: JSON.stringify({ x: 100, y: 200, from_screenshot: true }) },
},
{
id: 'loop_click_2',
function: { name: 'click', arguments: JSON.stringify({ x: 101, y: 201, from_screenshot: true }) },
},
{
id: 'loop_click_3',
function: { name: 'click', arguments: JSON.stringify({ x: 50, y: 60 }) },
},
],
messages,
() => {},
{ supportsVision: false },
null,
new Set(['click']),
1,
);

// The downscaled-screenshot clicks land on CSS pixels, not image pixels:
// (100,200) and (101,201) are distinct image pixels but collapse to the
// same CSS bucket, so the detector must be fed the converted values.
assert.deepEqual(
seen,
[[200, 400], [202, 402], [50, 60]],
`${label}: detector did not see the converted click points`,
);
}
});

test('visual target resolution stays private across Chrome and Firefox tools and prompts', () => {
for (const [label, getTools, prompts] of [
['chrome', getToolsForModeCh, [
Expand Down
Loading