Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 74 additions & 0 deletions src/chrome/src/agent/adapters.js
Original file line number Diff line number Diff line change
Expand Up @@ -17032,12 +17032,24 @@ const ADAPTERS = [
category: 'general',
matches: (url) => /^https?:\/\/(www\.)?instagram\.com\//.test(url),
fullPageCapture: { infiniteScroll: isInstagramInfiniteScrollUrl },
carousel: {
kind: 'indexed-query',
indexParam: 'img_index',
matches: (url) => {
try {
const parsed = new URL(url);
return /^(?:www\.)?instagram\.com$/i.test(parsed.hostname)
&& /^\/p\/[^/]+\/?$/.test(parsed.pathname);
} catch { return false; }
},
},
notes: `
- Login wall pops mid-scroll on the home feed (/), Explore (/explore), and Reels (/reels). Without sign-in, beyond a handful of posts the user can't view anything — surface that, don't loop trying to scroll past.
- Story bar at top of profile / feed is keyboard-driven: left/right arrows advance, Esc closes. Clicking is unreliable.
- Profile grid (/<user>) lazy-loads via IntersectionObserver — scroll the page (not a sub-container) to load more posts.
- DMs at /direct/inbox — sign-in required.
- Hashtag pages: /explore/tags/<tag>. Location pages: /explore/locations/<id>.
- Post carousels at /p/<id>/ expose deterministic ?img_index=N routes. Use carousel_navigate({index:N}) to visit slides directly and monotonically; decrease only for a fresh user-requested reverse scan. Never use ArrowLeft/ArrowRight, coordinate clicks, or alternate Next/Go back while enumerating a carousel.
- "Add to story / Add to post" actions require the mobile app for most content types — surface the limitation.
- Saving images / videos directly is blocked by the UI. If the user asks to download, use an enabled media download skill tool such as \`download_public_media\` first; otherwise use \`download_social_media\`.`,
},
Expand Down Expand Up @@ -17221,6 +17233,68 @@ export function getActiveAdapter(url) {
return null;
}

/** Return deterministic indexed-carousel metadata for the active URL. */
export function getCarouselNavigationPolicy(url) {
const adapter = getActiveAdapter(url);
const carousel = adapter?.carousel;
if (!carousel || carousel.kind !== 'indexed-query') return null;
try {
if (typeof carousel.matches === 'function' && !carousel.matches(url)) return null;
const parsed = new URL(url);
const rawIndex = Number(parsed.searchParams.get(carousel.indexParam));
const currentIndex = Number.isInteger(rawIndex) && rawIndex >= 1 ? rawIndex : 1;
parsed.search = '';
parsed.hash = '';
return {
adapterName: adapter.name,
kind: carousel.kind,
indexParam: carousel.indexParam,
currentIndex,
canonicalPostUrl: parsed.href,
};
} catch {
return null;
}
}

export function getCarouselNavigationTarget(url, index) {
const policy = getCarouselNavigationPolicy(url);
const targetIndex = Number(index);
if (!policy || !Number.isInteger(targetIndex) || targetIndex < 1) return null;
const target = new URL(policy.canonicalPostUrl);
target.searchParams.set(policy.indexParam, String(targetIndex));
return { ...policy, requestedIndex: targetIndex, targetUrl: target.href };
}

/**
* Infer a carousel total from aria-labels. Prefer an explicit "N of M" / "N/M"
* total; never treat a lone current-position label such as "Slide 3" as the
* last slide, which would abort a forward scan.
*/
export function parseCarouselSlideCount(labels) {
if (!Array.isArray(labels) || !labels.length) return null;
let total = null;
const indexes = [];
for (const raw of labels) {
const label = String(raw || '');
const ofMatch = /(?:slide|image)\s+(\d+)\s*(?:of|\/|de|von|sur)\s+(\d+)/i.exec(label);
if (ofMatch) {
const count = Number(ofMatch[2]);
if (Number.isInteger(count) && count >= 1) total = Math.max(total || 0, count);
continue;
}
const slideMatch = /(?:slide|image)\s+(\d+)/i.exec(label);
if (slideMatch) {
const n = Number(slideMatch[1]);
if (Number.isInteger(n) && n >= 1) indexes.push(n);
}
}
if (Number.isInteger(total) && total >= 1) return total;
const unique = [...new Set(indexes)];
if (unique.length < 2) return null;
return Math.max(...unique);
}

/**
* Return machine-readable full-page capture behavior for the active URL.
* This is runtime policy, not prompt guidance, so callers do not need an LLM
Expand Down
814 changes: 742 additions & 72 deletions src/chrome/src/agent/agent.js

Large diffs are not rendered by default.

83 changes: 82 additions & 1 deletion src/chrome/src/agent/loop-detector.js
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,10 @@ export class LoopDetector {
// separately so ref churn and interleaved close/Continue calls cannot
// disguise the same challenge loop.
this.verificationChallengeStates = new Map(); // tabId -> { key, active, reopenCount }
// Semantic carousel intent survives tool/selector/ref changes so a model
// cannot hide Next/Previous ping-pong by alternating arrows, AX clicks,
// screenshot points, and adapter navigation.
this.carouselIntentStates = new Map(); // tabId -> { history, cycleCount }
}

/**
Expand Down Expand Up @@ -161,6 +165,77 @@ export class LoopDetector {
return { buf, key };
}

_semanticCarouselIntent(name, args = {}, result = {}) {
let direction = '';
const key = String(args?.key || '');
if (name === 'carousel_navigate') direction = result?.traversalDirection === 'reverse' ? 'backward' : 'forward';
else if (name === 'press_keys' && key === 'ArrowRight') direction = 'forward';
else if (name === 'press_keys' && key === 'ArrowLeft') direction = 'backward';
else if (name === 'go_back') direction = 'backward';
const label = String(
args?.expected_name || args?.text || result?.name || result?.text
|| result?.coordinateReconciliation?.target?.name || '',
).replace(/\s+/g, ' ').trim().toLowerCase();
if (/^(next|sonraki)$/.test(label)) direction = 'forward';
if (/^(previous|prev|go back|önceki|geri)$/.test(label)) direction = 'backward';
if (!direction) return null;

const url = String(result?.resolvedUrl || result?.currentUrl || result?.pageUrl || '');
let canonical = String(result?.canonicalPostUrl || '');
let index = Number(result?.resolvedIndex ?? result?.requestedIndex);
try {
const parsed = new URL(url);
const parsedIndex = Number(parsed.searchParams.get('img_index'));
if (!Number.isInteger(index) && Number.isInteger(parsedIndex)) index = parsedIndex;
parsed.searchParams.delete('img_index');
if (!canonical) canonical = parsed.href;
} catch {}
return {
direction,
canonical: canonical || 'carousel',
index: Number.isInteger(index) ? index : null,
fingerprint: String(result?.visibleMediaFingerprint || '').slice(0, 320),
};
}

_checkSemanticCarouselLoop(tabId, name, args, result) {
const intent = this._semanticCarouselIntent(name, args, result);
if (!intent) return { kind: 'none', intent: null };
const previous = this.carouselIntentStates.get(tabId) || { history: [], cycleCount: 0 };
const history = previous.history.filter(entry => entry.canonical === intent.canonical);
history.push(intent);
if (history.length > 8) history.shift();
const last4 = history.slice(-4);
const directionCycle = last4.length === 4
&& last4[0].direction === last4[2].direction
&& last4[1].direction === last4[3].direction
&& last4[0].direction !== last4[1].direction;
const indexCycle = last4.length === 4
&& last4.every(entry => Number.isInteger(entry.index))
&& last4[0].index === last4[2].index
&& last4[1].index === last4[3].index
&& last4[0].index !== last4[1].index;
if (!directionCycle && !indexCycle) {
this.carouselIntentStates.set(tabId, { history, cycleCount: previous.cycleCount });
return { kind: 'none', intent };
}
const cycleCount = previous.cycleCount + 1;
this.carouselIntentStates.set(tabId, { history, cycleCount });
if (cycleCount >= 2) {
this.carouselIntentStates.delete(tabId);
return {
kind: 'stop',
intent,
message: 'Stopped: carousel state oscillated between the same two slides twice across mixed Next/Previous tools. Re-read the current slide; do not press arrows, click Previous/Next, or reuse screenshot coordinates.',
};
}
return {
kind: 'nudge',
intent,
warning: '[CAROUSEL OSCILLATION: The page returned to a recently visited slide after alternating forward/back actions. Stop using arrows, coordinate clicks, and Previous/Next. On a supported Instagram permalink use carousel_navigate with a strictly increasing index; otherwise re-read the current page once.]',
};
}

_detectLoop(buf, activeKey = null) {
if (!buf || buf.length < 3) return null;
// 1. Same key 3+ times in the window.
Expand Down Expand Up @@ -255,6 +330,7 @@ export class LoopDetector {
this.recentNavUrls.delete(tabId);
this._clearLoopState(tabId);
this.verificationChallengeStates.delete(tabId);
this.carouselIntentStates.delete(tabId);
}

/**
Expand Down Expand Up @@ -527,9 +603,14 @@ export class LoopDetector {
return this._noteHealthyLoopCall(tabId);
}
const { buf, key } = this._recordCall(tabId, toolName, toolArgs, toolResult);
const carouselLoop = this._checkSemanticCarouselLoop(tabId, toolName, toolArgs, toolResult);
if (carouselLoop.kind !== 'none') return carouselLoop;
if (this._isBrowserMutationTool(toolName)) {
const normalizeFailureScope = value => String(value).slice(0, 320);
const defaultFailureScope = normalizeFailureScope(`${toolName}|${bucketArgsKey(toolName, toolArgs)}`);
const carouselIntent = carouselLoop.intent;
const defaultFailureScope = normalizeFailureScope(carouselIntent
? `carousel-${carouselIntent.direction}|${carouselIntent.canonical}`
: `${toolName}|${bucketArgsKey(toolName, toolArgs)}`);
const failureScope = normalizeFailureScope(toolResult?.failureScope || defaultFailureScope);
const equivalentFailureScopes = new Set([failureScope, defaultFailureScope]);
if ((toolName === 'set_field' || toolName === 'type_ax') && typeof toolArgs?.ref_id === 'string') {
Expand Down
2 changes: 1 addition & 1 deletion src/chrome/src/agent/mutation-tools.js
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@

/** Tools that change page or browser state, gating auto-screenshots and
* unknown-outcome normalization as well as loop detection. */
export const STATE_CHANGE_TOOLS = new Set(['navigate', 'promote_iframe', 'new_tab', 'go_back', 'go_forward', 'click', 'click_ax', 'set_checked', 'iframe_click', 'type_text', 'type_ax', 'set_field', 'iframe_type', 'press_keys', 'scroll', 'hover', 'drag_drop', 'inject_css', 'remove_injected_css', 'patch_element', 'revert_patch', 'execute_js', 'inspect_event_listeners', 'highlight_element', 'execute_webmcp_tool']);
export const STATE_CHANGE_TOOLS = new Set(['navigate', 'carousel_navigate', 'promote_iframe', 'new_tab', 'go_back', 'go_forward', 'click', 'click_ax', 'set_checked', 'iframe_click', 'type_text', 'type_ax', 'set_field', 'iframe_type', 'press_keys', 'scroll', 'hover', 'drag_drop', 'inject_css', 'remove_injected_css', 'patch_element', 'revert_patch', 'execute_js', 'inspect_event_listeners', 'highlight_element', 'execute_webmcp_tool']);

/**
* Everything the failed-action loop counters treat as a browser mutation.
Expand Down
Loading
Loading