Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 46 additions & 7 deletions services/ensemble-service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,10 @@ const COMPLETION_SIGNAL_WINDOW_MS = 180_000
// unaffected: a team that is really done still ends within seconds.
const SINGLE_SIGNAL_IDLE_THRESHOLD_MS = 480_000
const TWO_SIGNAL_IDLE_THRESHOLD_MS = 300_000
/** A sign-off is short; long analysis prose that mentions "done" is not one. */
const MAX_COMPLETION_SIGNAL_LENGTH = 400
/** Machinery an agent can discuss without meaning "I am finished". */
const ORCHESTRATION_VOCABULARY = /---STATUS|COLLAB_DONE|team-say|team-read|DONE protocol|\bsentinel\b/i
const MIN_MESSAGES_BEFORE_AUTO_DISBAND = 10
// Explicit sentinel: when every active agent sends this exact marker as a full
// message, the team auto-disbands immediately — no idle wait, no minimum
Expand Down Expand Up @@ -127,6 +131,14 @@ interface CompletionSignal {
*/
function isCompletionStatement(content: string): boolean {
if (CLOSING_PATTERNS.some(pattern => pattern.test(content))) return true
// Below this line only the heuristic path remains, so the shape of the message
// matters. A sign-off is short: a long analysis that happens to contain "done"
// is an agent working, not an agent finishing.
if (content.length > MAX_COMPLETION_SIGNAL_LENGTH) return false
// And an agent describing the orchestration is not reporting on itself. Seen on
// a review task pointed at collab-poll.sh: both agents quoted the script's own
// ---STATUS:{ACTIVE,QUIET,DONE,WAITING} sentinel, which ended the run.
if (ORCHESTRATION_VOCABULARY.test(content)) return false
if (!COMPLETION_PATTERNS.some(pattern => pattern.test(content))) return false
return !CONTINUATION_PATTERNS.some(pattern => pattern.test(content))
}
Expand Down Expand Up @@ -270,7 +282,21 @@ class EnsembleService {
}

private shouldAutoDisband(team: EnsembleTeam): boolean {
const messages = getMessages(team.id)
const allMessages = getMessages(team.id)
// Completion counts only from the last message the user sent. `ensemble steer`
// exists so a user can redirect a running team, so a sentinel or sign-off from
// before that redirect no longer describes the work: an agent that had already
// finished is not finished with the new instruction. Without this the team can
// disband while an agent is mid-answer to the user.
const lastUserMessageMs = allMessages
.filter(message => message.from === 'user' && message.timestamp)
.reduce((latest, message) => Math.max(latest, new Date(message.timestamp).getTime() || 0), 0)
const messages = lastUserMessageMs
? allMessages.filter(message => {
const ts = message.timestamp ? new Date(message.timestamp).getTime() : NaN
return Number.isNaN(ts) || ts >= lastUserMessageMs
})
: allMessages
const nonEnsembleMessages = messages.filter(message => message.from !== 'ensemble')
const lastMessage = nonEnsembleMessages[nonEnsembleMessages.length - 1]
if (!lastMessage) return false
Expand Down Expand Up @@ -317,7 +343,7 @@ class EnsembleService {
// third agent mid-task. The exact sentinel above is the fast path; these
// patterns are only a safety net for teams that go quiet without sending it.
if (idleForMs <= TWO_SIGNAL_IDLE_THRESHOLD_MS) return false
if (this.hasTwoRecentCompletionSignals(completionSignals)) return true
if (this.hasRecentCompletionSignalsFromAll(completionSignals, activeAgentNames.size)) return true
if (idleForMs <= SINGLE_SIGNAL_IDLE_THRESHOLD_MS) return false
return completionSignals.length >= 1
}
Expand All @@ -326,11 +352,22 @@ class EnsembleService {
return isCompletionStatement(content)
}

private hasTwoRecentCompletionSignals(signals: CompletionSignal[]): boolean {
for (let i = 0; i < signals.length; i++) {
for (let j = i + 1; j < signals.length; j++) {
if (signals[j].timestamp - signals[i].timestamp > COMPLETION_SIGNAL_WINDOW_MS) break
if (signals[i].agentName !== signals[j].agentName) return true
/**
* True when EVERY active agent produced a completion signal inside one
* COMPLETION_SIGNAL_WINDOW_MS window — the same bar the sentinel path uses.
* Two agents agreeing is not the team agreeing: in a trio that ends the run
* for the third, which is what the comment above this call warns about.
*/
private hasRecentCompletionSignalsFromAll(
signals: CompletionSignal[], activeAgentCount: number,
): boolean {
if (activeAgentCount < 2) return false
for (let start = 0; start < signals.length; start++) {
const seen = new Set<string>([signals[start].agentName])
for (let end = start + 1; end < signals.length; end++) {
if (signals[end].timestamp - signals[start].timestamp > COMPLETION_SIGNAL_WINDOW_MS) break
seen.add(signals[end].agentName)
if (seen.size >= activeAgentCount) return true
}
}
return false
Expand Down Expand Up @@ -1225,4 +1262,6 @@ export const __testing = {
SINGLE_SIGNAL_IDLE_THRESHOLD_MS,
COMPLETION_PATTERNS,
CONTINUATION_PATTERNS,
MAX_COMPLETION_SIGNAL_LENGTH,
ORCHESTRATION_VOCABULARY,
}
81 changes: 81 additions & 0 deletions tests/ensemble.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,23 @@ function fillerMessages(count: number, teamId = 'team-1'): EnsembleMessage[] {
)
}

/**
* Filler older than the messages a wording-path test cares about. The wording path
* only fires while idle sits between TWO_SIGNAL_IDLE_THRESHOLD_MS and
* SINGLE_SIGNAL_IDLE_THRESHOLD_MS, so the signals under test must be the newest
* messages in the feed.
*/
function olderFiller(count: number, teamId = 'team-1'): EnsembleMessage[] {
return Array.from({ length: count }, (_, i) =>
makeMessage({
teamId,
from: i % 2 === 0 ? 'codex-1' : 'claude-2',
content: `analysis step ${i}`,
timestamp: `2026-03-18T11:50:${String(i).padStart(2, '0')}.000Z`,
}),
)
}

/** A three-agent team, for checks that must not assume a pair. */
function makeTrioTeam(): EnsembleTeam {
return makeTeam({
Expand Down Expand Up @@ -318,6 +335,70 @@ describe('shouldAutoDisband() — tested via checkIdleTeams()', () => {
expect(appendedMessages.some(m => m.content.includes('Auto-disband'))).toBe(false)
})

it('does NOT auto-disband a trio when only two of three used completion wording', async () => {
// The sentinel path already requires every agent. The wording path did not, so a
// quiet trio could still lose its third agent to the other two agreeing.
const team = makeTrioTeam()
const messages: EnsembleMessage[] = [
...olderFiller(8),
makeMessage({ from: 'codex-1', teamId: 'team-1', content: 'Ik ben klaar', timestamp: '2026-03-18T11:58:00.000Z' }),
makeMessage({ from: 'claude-2', teamId: 'team-1', content: 'Ik ben ook klaar', timestamp: '2026-03-18T11:58:30.000Z' }),
]

const { mod, appendedMessages } = await setupServiceWithMocks(team, messages)
await mod.checkIdleTeams()

expect(appendedMessages.some(m => m.content.includes('Auto-disband'))).toBe(false)
})

it('auto-disbands a trio once all three used completion wording', async () => {
const team = makeTrioTeam()
const messages: EnsembleMessage[] = [
...olderFiller(8),
makeMessage({ from: 'codex-1', teamId: 'team-1', content: 'Ik ben klaar', timestamp: '2026-03-18T11:58:00.000Z' }),
makeMessage({ from: 'claude-2', teamId: 'team-1', content: 'Ik ben ook klaar', timestamp: '2026-03-18T11:58:20.000Z' }),
makeMessage({ from: 'grok-3', teamId: 'team-1', content: 'Ik ben klaar', timestamp: '2026-03-18T11:58:40.000Z' }),
]

const { mod, appendedMessages } = await setupServiceWithMocks(team, messages)
await mod.checkIdleTeams()

expect(appendedMessages.some(m => m.content.includes('Auto-disband'))).toBe(true)
})

it('voids done sentinels sent before the user steered the team', async () => {
// An agent that had already signalled is not finished with the new instruction:
// the team must not disband while it answers.
const team = makeTeam()
const messages: EnsembleMessage[] = [
...fillerMessages(8),
makeMessage({ from: 'codex-1', teamId: 'team-1', content: '<<COLLAB_DONE>>', timestamp: '2026-03-18T12:04:20.000Z' }),
makeMessage({ from: 'claude-2', teamId: 'team-1', content: '<<COLLAB_DONE>>', timestamp: '2026-03-18T12:04:30.000Z' }),
makeMessage({ from: 'user', teamId: 'team-1', content: 'One more thing: check the retry path', timestamp: '2026-03-18T12:04:45.000Z' }),
makeMessage({ from: 'claude-2', teamId: 'team-1', content: 'ack: checking the retry path now', timestamp: '2026-03-18T12:04:50.000Z' }),
]

const { mod, appendedMessages } = await setupServiceWithMocks(team, messages)
await mod.checkIdleTeams()

expect(appendedMessages.some(m => m.content.includes('Auto-disband'))).toBe(false)
})

it('still disbands on sentinels sent after the user steered the team', async () => {
const team = makeTeam()
const messages: EnsembleMessage[] = [
...fillerMessages(8),
makeMessage({ from: 'user', teamId: 'team-1', content: 'wrap it up please', timestamp: '2026-03-18T12:04:20.000Z' }),
makeMessage({ from: 'codex-1', teamId: 'team-1', content: '<<COLLAB_DONE>>', timestamp: '2026-03-18T12:04:40.000Z' }),
makeMessage({ from: 'claude-2', teamId: 'team-1', content: '<<COLLAB_DONE>>', timestamp: '2026-03-18T12:04:50.000Z' }),
]

const { mod, appendedMessages } = await setupServiceWithMocks(team, messages)
await mod.checkIdleTeams()

expect(appendedMessages.some(m => m.content.includes('Auto-disband'))).toBe(true)
})

it('auto-disbands when two different agents send completion signals and the team went quiet', async () => {
const team = makeTeam()
const messages: EnsembleMessage[] = [
Expand Down
29 changes: 29 additions & 0 deletions tests/premature-disband.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -60,3 +60,32 @@ it('idle thresholds leave room for an agent that is reading', () => {
)
})
})

// A message about the orchestration is not a message about the agent. Seen on a
// review task pointed at scripts/collab-poll.sh: both agents quoted its own
// ---STATUS:{ACTIVE,QUIET,DONE,WAITING} sentinel, and the run ended mid-analysis.
describe('describing the machinery is not finishing', () => {
it('quoting the DONE sentinel of a script under review is not an ending', () => {
assert.strictEqual(
hasCompletionSignal('collab-poll.sh emits TSV lines terminated by a ---STATUS:{ACTIVE,QUIET,DONE,WAITING} sentinel'),
false,
)
})

it('talking about the DONE protocol is not an ending', () => {
assert.strictEqual(hasCompletionSignal('I will follow the DONE protocol once we agree'), false)
})

it('announcing a team-say is not an ending', () => {
assert.strictEqual(hasCompletionSignal('I will send the done sentinel via team-say when you agree'), false)
})

it('a long analysis that merely contains finishing words is not an ending', () => {
const analysis = 'Exchange 2: the two scripts disagree about what finished means, and the cursor is done differently in each. '.padEnd(600, 'x')
assert.strictEqual(hasCompletionSignal(analysis), false)
})

it('but a short genuine sign-off still counts', () => {
assert.strictEqual(hasCompletionSignal('Ik ben klaar'), true)
})
})