diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js index a7060c8cb..853ee5ee4 100644 --- a/src/chrome/src/agent/agent.js +++ b/src/chrome/src/agent/agent.js @@ -746,6 +746,7 @@ export class Agent extends LoopDetector { // answer. Track long observation-only streaks and remind it to deliver a // useful result before exhausting the run. this.deliveryObservationStreaks = new Map(); // tabId -> count + this.deliveryActionableDiscoveryResets = new Set(); // tabIds that used their one discovery reset since meaningful progress this.lastAutoScreenshotTs = new Map(); // tabId -> ms — defensive debounce this.lastSeenAdapter = new Map(); // tabId -> adapter name from last enrichment // Per-tab opt-in: when true, the agent is allowed to use API mutations @@ -2555,6 +2556,7 @@ export class Agent extends LoopDetector { this._uploadSelectorRecoveryRequired.delete(tabId); this._compactUploadTargets.delete(tabId); this.deliveryObservationStreaks.delete(tabId); + this.deliveryActionableDiscoveryResets.delete(tabId); this.bulkApiMutationClicks.delete(tabId); this.bulkApiMutationHints.delete(tabId); const replayFailurePrefix = `${tabId}|`; @@ -3247,6 +3249,17 @@ export class Agent extends LoopDetector { _checkDeliveryObservationStreak(tabId, name, args = {}, result = null, options = {}) { const observation = this.constructor.DELIVERY_OBSERVATION_TOOLS.has(name) && !isNetworkMutation(name, args); + if (observation + && options.discoveredActionableTargets === true + && !this.deliveryActionableDiscoveryResets.has(tabId)) { + // Give structured target discovery one free observation per verified + // progress interval. Paginating through newly discovered controls cannot + // repeatedly erase the delivery guard; meaningful consequential progress + // below rearms the one-shot reset. + this.deliveryActionableDiscoveryResets.add(tabId); + this.deliveryObservationStreaks.delete(tabId); + return { kind: 'none' }; + } if (observation && options.requiredReadProgress === true) { // A new page in the runtime-required complete-thread scope is bounded, // deterministic progress, not aimless research drift. Let exact trusted @@ -3267,6 +3280,7 @@ export class Agent extends LoopDetector { // verified consequential progress or a real progress-ledger mutation. if (this._deliveryCheckpointMadeMeaningfulProgress(name, result, options)) { this.deliveryObservationStreaks.delete(tabId); + this.deliveryActionableDiscoveryResets.delete(tabId); } return { kind: 'none' }; } @@ -7329,6 +7343,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d completionStateBeforeTool, completionStateAfterTool, ), + discoveredActionableTargets: Number(progressObserved?.addedPending || 0) > 0, requiredReadProgress, // Ask research can lose a useful deliverable to the same observation // drift as Act/Dev. Any interactive mode that advertises `done` @@ -7435,7 +7450,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d }); } if (progressObserved) { - resultContent += `\n[PROGRESS LEDGER OBSERVED: GitHub stargazers buttons observed=${progressObserved.observedButtons}; added ${progressObserved.addedPending} pending Follow row(s); skipped ${progressObserved.alreadyFollowedSkipped} already-followed row(s) and ${progressObserved.excludedSkipped} excluded row(s). Only rows created from visible Follow buttons need follow action.]`; + resultContent += `\n[PROGRESS LEDGER OBSERVED: GitHub follow buttons observed=${progressObserved.observedButtons}; added ${progressObserved.addedPending} pending Follow row(s); skipped ${progressObserved.alreadyFollowedSkipped} already-followed row(s) and ${progressObserved.excludedSkipped} excluded row(s). Only rows created from visible Follow buttons need follow action.]`; } if (progressAuto) { resultContent += '\n' + this._progressAutoRecordedNote(progressAuto.item); @@ -12426,6 +12441,18 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d }; } + _deterministicDeliveryProgressPartial(tabId) { + const rows = this._currentTaskLedgerRows(tabId); + if (!rows.length) return ''; + const counts = progressCounts(rows); + const summary = [ + 'Browser observation limit reached before the full task scope could be verified.', + `Partial progress was preserved from the app-owned ledger: ${counts.total} recorded item(s) — ${counts.processed} processed, ${counts.skipped} skipped, ${counts.failed} failed, ${counts.pending} pending, and ${counts.acted} acted but not fully resolved.`, + 'No further browser observations or actions were performed after the cutoff.', + ].join(' '); + return this._appendProgressLedgerToFinal(tabId, summary); + } + async _recoverDeliveryCheckpointTurn( tabId, messages, @@ -12469,13 +12496,16 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d const stopped = this._consumeContextOnlyAbort(tabId, messages, onUpdate); if (stopped) return stopped; if (!recovered) { - const content = fallbackMessage || (protectedPageRecovery + const deterministicPartial = protectedPageRecovery + ? '' + : this._deterministicDeliveryProgressPartial(tabId); + const content = deterministicPartial || fallbackMessage || (protectedPageRecovery ? 'Chrome protected this Chrome Web Store page, and WebBrain could not produce a useful answer from the one visual fallback. Leave the page open and continue manually.' : 'I gathered information but could not produce a valid partial result after reaching the browser observation limit.'); - const status = preservedStatus || 'delivery_recovery_failed'; + const status = preservedStatus || (deterministicPartial ? 'partial' : 'delivery_recovery_failed'); messages.push({ role: 'assistant', content }); onUpdate('text', { content, replace: true }); - onUpdate('error', { message: content }); + onUpdate(deterministicPartial ? 'warning' : 'error', { message: content }); onUpdate('run_status', { status, message: content }); this._persist(tabId); return { content, status }; @@ -16262,6 +16292,19 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d } } + _isGithubFollowListUrl(url) { + if (this._isGithubStargazersUrl(url)) return true; + try { + const parsed = new URL(url); + if (parsed.hostname !== 'github.com') return false; + const parts = parsed.pathname.split('/').filter(Boolean); + if (parts.length >= 3 && parts[0] === 'orgs' && parts[2] === 'followers') return true; + return parts.length === 1 && ['followers', 'following'].includes(parsed.searchParams.get('tab') || ''); + } catch { + return false; + } + } + _mastodonPageContentFromResult(result = {}) { if (!result || typeof result !== 'object') return ''; const candidates = [ @@ -16311,7 +16354,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d const pageContent = result.pageContent || result.text || ''; if (!pageContent || (!pageContent.includes('button "Follow ') && !pageContent.includes('button "Unfollow '))) return null; const url = result.url || result.pageUrl || await this._currentUrl(tabId); - if (!this._isGithubStargazersUrl(url)) return null; + if (!this._isGithubFollowListUrl(url)) return null; const pageScope = this._rememberProgressPageScope(tabId, url); const session = this._progressSessionForObservation(tabId, { pageScope }); if (!isProgressActionAllowed(session, 'follow')) return null; diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js index 0f735856b..5704ccc8d 100644 --- a/src/firefox/src/agent/agent.js +++ b/src/firefox/src/agent/agent.js @@ -563,6 +563,7 @@ export class Agent extends LoopDetector { // answer. Track long observation-only streaks and remind it to deliver a // useful result before exhausting the run. this.deliveryObservationStreaks = new Map(); + this.deliveryActionableDiscoveryResets = new Set(); // Local screenshot redaction (issue #312). When true, screenshots sent // to a Vision endpoint are pixelated over DOM-detected PII regions // (form fields + email/phone text) BEFORE leaving the extension. Off by @@ -2340,6 +2341,7 @@ export class Agent extends LoopDetector { this._uploadSelectorRecoveryRequired.delete(tabId); this._compactUploadTargets.delete(tabId); this.deliveryObservationStreaks.delete(tabId); + this.deliveryActionableDiscoveryResets.delete(tabId); this.bulkApiMutationClicks.delete(tabId); this.bulkApiMutationHints.delete(tabId); const replayFailurePrefix = `${tabId}|`; @@ -3019,6 +3021,17 @@ export class Agent extends LoopDetector { _checkDeliveryObservationStreak(tabId, name, args = {}, result = null, options = {}) { const observation = this.constructor.DELIVERY_OBSERVATION_TOOLS.has(name) && !isNetworkMutation(name, args); + if (observation + && options.discoveredActionableTargets === true + && !this.deliveryActionableDiscoveryResets.has(tabId)) { + // Give structured target discovery one free observation per verified + // progress interval. Paginating through newly discovered controls cannot + // repeatedly erase the delivery guard; meaningful consequential progress + // below rearms the one-shot reset. + this.deliveryActionableDiscoveryResets.add(tabId); + this.deliveryObservationStreaks.delete(tabId); + return { kind: 'none' }; + } if (observation && options.requiredReadProgress === true) { // A new page in the runtime-required complete-thread scope is bounded, // deterministic progress, not aimless research drift. Let exact trusted @@ -3039,6 +3052,7 @@ export class Agent extends LoopDetector { // verified consequential progress or a real progress-ledger mutation. if (this._deliveryCheckpointMadeMeaningfulProgress(name, result, options)) { this.deliveryObservationStreaks.delete(tabId); + this.deliveryActionableDiscoveryResets.delete(tabId); } return { kind: 'none' }; } @@ -5897,6 +5911,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d completionStateBeforeTool, completionStateAfterTool, ), + discoveredActionableTargets: Number(progressObserved?.addedPending || 0) > 0, requiredReadProgress, // Ask research can lose a useful deliverable to the same observation // drift as Act/Dev. Any interactive mode that advertises `done` @@ -5987,7 +6002,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d }); } if (progressObserved) { - resultContent += `\n[PROGRESS LEDGER OBSERVED: GitHub stargazers buttons observed=${progressObserved.observedButtons}; added ${progressObserved.addedPending} pending Follow row(s); skipped ${progressObserved.alreadyFollowedSkipped} already-followed row(s) and ${progressObserved.excludedSkipped} excluded row(s). Only rows created from visible Follow buttons need follow action.]`; + resultContent += `\n[PROGRESS LEDGER OBSERVED: GitHub follow buttons observed=${progressObserved.observedButtons}; added ${progressObserved.addedPending} pending Follow row(s); skipped ${progressObserved.alreadyFollowedSkipped} already-followed row(s) and ${progressObserved.excludedSkipped} excluded row(s). Only rows created from visible Follow buttons need follow action.]`; } if (progressAuto) { resultContent += '\n' + this._progressAutoRecordedNote(progressAuto.item); @@ -10336,6 +10351,18 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d }; } + _deterministicDeliveryProgressPartial(tabId) { + const rows = this._currentTaskLedgerRows(tabId); + if (!rows.length) return ''; + const counts = progressCounts(rows); + const summary = [ + 'Browser observation limit reached before the full task scope could be verified.', + `Partial progress was preserved from the app-owned ledger: ${counts.total} recorded item(s) — ${counts.processed} processed, ${counts.skipped} skipped, ${counts.failed} failed, ${counts.pending} pending, and ${counts.acted} acted but not fully resolved.`, + 'No further browser observations or actions were performed after the cutoff.', + ].join(' '); + return this._appendProgressLedgerToFinal(tabId, summary); + } + async _recoverDeliveryCheckpointTurn( tabId, messages, @@ -10368,13 +10395,15 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d const stopped = this._consumeContextOnlyAbort(tabId, messages, onUpdate); if (stopped) return stopped; if (!recovered) { - const content = fallbackMessage || 'I gathered information but could not produce a valid partial result after reaching the browser observation limit.'; + const deterministicPartial = this._deterministicDeliveryProgressPartial(tabId); + const content = deterministicPartial || fallbackMessage || 'I gathered information but could not produce a valid partial result after reaching the browser observation limit.'; + const status = deterministicPartial ? 'partial' : 'delivery_recovery_failed'; messages.push({ role: 'assistant', content }); onUpdate('text', { content, replace: true }); - onUpdate('error', { message: content }); - onUpdate('run_status', { status: 'delivery_recovery_failed', message: content }); + onUpdate(deterministicPartial ? 'warning' : 'error', { message: content }); + onUpdate('run_status', { status, message: content }); this._persist(tabId); - return { content, status: 'delivery_recovery_failed' }; + return { content, status }; } const toolResult = { done: true, @@ -14010,6 +14039,19 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d } } + _isGithubFollowListUrl(url) { + if (this._isGithubStargazersUrl(url)) return true; + try { + const parsed = new URL(url); + if (parsed.hostname !== 'github.com') return false; + const parts = parsed.pathname.split('/').filter(Boolean); + if (parts.length >= 3 && parts[0] === 'orgs' && parts[2] === 'followers') return true; + return parts.length === 1 && ['followers', 'following'].includes(parsed.searchParams.get('tab') || ''); + } catch { + return false; + } + } + _mastodonPageContentFromResult(result = {}) { if (!result || typeof result !== 'object') return ''; const candidates = [ @@ -14059,7 +14101,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d const pageContent = result.pageContent || result.text || ''; if (!pageContent || (!pageContent.includes('button "Follow ') && !pageContent.includes('button "Unfollow '))) return null; const url = result.url || result.pageUrl || await this._currentUrl(tabId); - if (!this._isGithubStargazersUrl(url)) return null; + if (!this._isGithubFollowListUrl(url)) return null; const pageScope = this._rememberProgressPageScope(tabId, url); const session = this._progressSessionForObservation(tabId, { pageScope }); if (!isProgressActionAllowed(session, 'follow')) return null; diff --git a/test/run.js b/test/run.js index 12708ead2..b5fe4a9b2 100644 --- a/test/run.js +++ b/test/run.js @@ -9869,6 +9869,50 @@ test('delivery checkpoints escalate at eight and reset only after meaningful pro assert.match(forcedDelivery.warning, /call done exactly once/i, `${label}: terminal instruction missing`); assert.match(forcedDelivery.warning, /partial or failed/i, `${label}: recovery outcomes must exclude success`); + const discoveryTab = `${tab}-discovery`; + for (let i = 0; i < 4; i++) { + agent._checkDeliveryObservationStreak(discoveryTab, 'get_accessibility_tree', {}, { success: true }, enforced); + } + const discovery = agent._checkDeliveryObservationStreak( + discoveryTab, + 'get_accessibility_tree', + {}, + { success: true }, + { ...enforced, discoveredActionableTargets: true }, + ); + assert.equal(discovery.kind, 'none', `${label}: newly discovered action targets should count as progress`); + assert.equal(agent.deliveryObservationStreaks.has(discoveryTab), false, `${label}: first target discovery should reset the observation streak`); + let repeatedDiscovery = null; + for (let page = 2; page <= 9; page++) { + repeatedDiscovery = agent._checkDeliveryObservationStreak( + discoveryTab, + 'get_accessibility_tree', + { page }, + { success: true }, + { ...enforced, discoveredActionableTargets: true }, + ); + } + assert.equal(repeatedDiscovery.kind, 'deliver', `${label}: repeated target discovery should remain bounded`); + assert.equal(repeatedDiscovery.count, 8, `${label}: repeated target discovery should reach forced delivery`); + agent._checkDeliveryObservationStreak( + discoveryTab, + 'click_ax', + { ref: 'ref_follow' }, + { success: true }, + { ...enforced, consequential: true }, + ); + assert.equal(agent.deliveryObservationStreaks.has(discoveryTab), false, `${label}: consequential action should rearm the discovery reset`); + assert.equal(agent.deliveryActionableDiscoveryResets.has(discoveryTab), false, `${label}: consequential action should restore the one-shot discovery reset`); + const discoveryAfterAction = agent._checkDeliveryObservationStreak( + discoveryTab, + 'get_accessibility_tree', + { page: 10 }, + { success: true }, + { ...enforced, discoveredActionableTargets: true }, + ); + assert.equal(discoveryAfterAction.kind, 'none', `${label}: discovery after consequential progress should reset again`); + assert.equal(agent.deliveryObservationStreaks.has(discoveryTab), false, `${label}: rearmed discovery should restart from zero`); + const requiredReadTab = `${tab}-required-read`; for (let page = 1; page <= 12; page++) { const requiredRead = agent._checkDeliveryObservationStreak( @@ -9964,6 +10008,7 @@ test('delivery checkpoint enforcement is wired into both agent loops', () => { assert.match(source, /deliveryCheck\.kind === 'nudge'/, `${browserName}: warning must reach the model`); assert.match(source, /deliveryCheck\.kind === 'deliver'[\s\S]{0,900}?action: 'deliver'/, `${browserName}: second checkpoint must leave the browser loop`); assert.match(source, /batchResult\.action === 'deliver'[\s\S]{0,300}?_recoverDeliveryCheckpointTurn/, `${browserName}: caller must enter done-only recovery`); + assert.match(source, /discoveredActionableTargets:\s*Number\(progressObserved\?\.addedPending \|\| 0\) > 0/, `${browserName}: newly observed progress rows must reset delivery drift`); assert.match(source, /this\.deliveryObservationStreaks\.delete\(tabId\)/, `${browserName}: run cleanup must clear state`); } }); @@ -10278,6 +10323,52 @@ test('delivery recovery rejects plain text or success and shows a runtime blocke } }); +test('delivery recovery preserves a deterministic ledger partial when the model output is invalid', async () => { + for (const [label, AgentClass] of [['chrome', AgentCh], ['firefox', AgentFx]]) { + const tabId = label === 'chrome' ? 918 : 919; + const messages = [ + { role: 'system', content: 'ordinary agent prompt' }, + { role: 'user', content: 'Follow every user on the current page.' }, + ]; + const agent = new AgentClass({}); + const updates = []; + agent._persist = () => {}; + agent.conversations.set(tabId, messages); + const seeded = agent._progressUpdate(tabId, { + items: [ + { id: 'alice', label: 'alice', action: 'follow', status: 'processed' }, + { id: 'bob', label: 'bob', action: 'follow', status: 'pending' }, + { id: 'carol', label: 'carol', action: 'follow', status: 'acted' }, + ], + }); + assert.equal(seeded.success, true, `${label}: test ledger did not seed`); + agent._chatWithCostAllowance = async () => ({ content: 'I will continue.', toolCalls: [] }); + + const recovery = await agent._recoverDeliveryCheckpointTurn( + tabId, + messages, + (type, data) => updates.push({ type, data }), + { model: 'test-model' }, + {}, + null, + 8, + 'generic fallback should not be used when ledger progress exists', + ); + + assert.equal(recovery.status, 'partial', `${label}: deterministic ledger recovery should be partial`); + assert.match(recovery.content, /Partial progress was preserved from the app-owned ledger/i); + assert.match(recovery.content, /3 recorded item\(s\).*1 processed.*1 pending.*1 acted/i); + assert.match(recovery.content, /- processed: alice/); + assert.match(recovery.content, /- pending: bob/); + assert.match(recovery.content, /- acted: carol/); + assert.doesNotMatch(recovery.content, /generic fallback should not be used|I will continue/); + assert.equal(updates.some(update => update.type === 'warning'), true, `${label}: deterministic partial should be surfaced as a warning`); + assert.equal(updates.some(update => update.type === 'error'), false, `${label}: useful deterministic partial should not be reported as an error`); + assert.equal(updates.some(update => update.type === 'run_status' && update.data?.status === 'partial'), true, `${label}: partial run status missing`); + assert.equal(messages.at(-1)?.content, recovery.content, `${label}: deterministic partial was not persisted`); + } +}); + test('tool-free response and recovery calls honor Stop before rendering model output', async () => { for (const [label, AgentClass] of [['chrome', AgentCh], ['firefox', AgentFx]]) { for (const phase of ['response_only', 'terminal_recovery']) { @@ -66520,6 +66611,50 @@ test('agent records GitHub stargazer observations into the progress ledger', asy } }); +test('agent records GitHub organization follower observations as actionable progress', async () => { + const page = ` + button "Follow alice" [ref_51] + button "Unfollow bob" [ref_52] + button "Follow carol" [ref_53] + `; + for (const [label, AgentClass] of [['chrome', AgentCh], ['firefox', AgentFx]]) { + const agent = new AgentClass({ getActive: () => ({ contextWindow: 128000, supportsVision: false }) }); + const tabId = label === 'chrome' ? 920 : 921; + agent.conversations.set(tabId, [ + { role: 'system', content: 'sys' }, + { role: 'user', content: 'Follow every user on this organization followers page.' }, + ]); + agent._currentUrl = async () => 'https://github.com/orgs/thebrowsercompany/followers?page=2'; + agent._progressUpdate(tabId, { + items: [{ id: 'seed-user', label: 'seed-user', action: 'follow', status: 'processed' }], + }, { sessionId: `followers-${label}` }); + + const result = { success: true, pageContent: page }; + const note = await agent._recordProgressObservation(tabId, 'get_accessibility_tree', result); + assert.equal(note.observedButtons, 3, `${label}: follower buttons were not observed`); + assert.equal(note.addedPending, 2, `${label}: new follower targets were not recorded`); + assert.deepEqual( + agent._currentTaskLedgerRows(tabId).map(row => [row.id, row.status]), + [['seed-user', 'processed'], ['alice', 'pending'], ['carol', 'pending']], + `${label}: organization follower ledger mismatch`, + ); + + const enforced = { enforceTerminal: true }; + for (let i = 0; i < 4; i++) { + agent._checkDeliveryObservationStreak(tabId, 'get_accessibility_tree', {}, { success: true }, enforced); + } + const checkpoint = agent._checkDeliveryObservationStreak( + tabId, + 'get_accessibility_tree', + {}, + result, + { ...enforced, discoveredActionableTargets: note.addedPending > 0 }, + ); + assert.equal(checkpoint.kind, 'none', `${label}: actionable follower discovery triggered delivery drift`); + assert.equal(agent.deliveryObservationStreaks.has(tabId), false, `${label}: actionable follower discovery did not reset drift`); + } +}); + test('agent ignores stale terminal follow rows when observing a new stargazer task', async () => { const page = 'button "Follow alice" [ref_41]'; for (const AgentClass of [AgentCh, AgentFx]) {