feat(voice): audible recipient ack on inbound /handoff in #general

src/active-speakers.ts

@@ -0,0 +1,27 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright (c) Reflectt AI
+
+/**
+ * active-speakers — minimal in-memory tracker of which agents are currently
+ * mid-TTS. Mirrors `canvas_state.payload.activeSpeaker` so server-side
+ * gates that don't have access to the canvasStateMap closure (e.g.
+ * voice-ack-on-handoff) can still ask the question. server.ts updates
+ * this whenever it toggles canvasStateMap activeSpeaker; readers stay
+ * pure.
+ */
+
+const speakers = new Set<string>()
+
+export function setActiveSpeaker(agentId: string, active: boolean): void {
+  if (!agentId) return
+  if (active) speakers.add(agentId)
+  else speakers.delete(agentId)
+}
+
+export function isAgentActiveSpeaker(agentId: string): boolean {
+  return speakers.has(agentId)
+}
+
+export function _resetActiveSpeakersForTest(): void {
+  speakers.clear()
+}

src/canvas-interactive.ts

@@ -6,6 +6,7 @@ import type { FastifyInstance } from 'fastify'
 import type { eventBus as eventBusInstance } from './events.js'
 import { getDb } from './db.js'
 import { getIdentityColor } from './agent-config.js'
+import { setActiveSpeaker as setActiveSpeakerStore } from './active-speakers.js'
 
 // ── Types ──
 
@@ -658,6 +659,13 @@ export async function canvasInteractiveRoutes(
     const voiceId = await hashTts(text, agentId)
     const estimatedMs = Math.round(text.length * 50)
 
+    // Mark this agent as currently speaking so server-side gates (e.g.
+    // voice-ack-on-handoff) can ask "is B already mid-TTS?". Cleared on
+    // voice_output (with accurate durationMs) or after a generous
+    // estimatedMs+8s safety net if TTS never resolves.
+    setActiveSpeakerStore(agentId, true)
+    const safetyClear = setTimeout(() => setActiveSpeakerStore(agentId, false), estimatedMs + 8_000)
+
     // Emit voice_queued immediately so UI can show "speaking" state
     const queuedPayload = JSON.stringify({ type: 'voice_queued', voiceId, text, agentId, agentName, estimatedMs })
     for (const [, client] of renderStreamSubscribers) {
@@ -668,6 +676,8 @@ export async function canvasInteractiveRoutes(
     makeTts(text, agentId).then(async (result) => {
       if (!result) {
         console.error(`[voice] makeTts returned null for "${text.substring(0, 30)}..." — Kokoro + ElevenLabs both failed`)
+        clearTimeout(safetyClear)
+        setActiveSpeakerStore(agentId, false)
         return
       }
       console.log(`[voice] TTS ready: voiceId=${voiceId} url=${result.url} ms=${result.ms}`)
@@ -677,8 +687,15 @@ export async function canvasInteractiveRoutes(
       for (const [, client] of renderStreamSubscribers) {
         try { client.send('event: voice_output\r\ndata: ' + payload + '\r\n\r\n') } catch {}
       }
+      // Clear the active-speaker mark after the TTS audio finishes
+      // playing on clients (rough estimate; client-side playback is what
+      // truly ends the turn, but this is the best server-side proxy).
+      clearTimeout(safetyClear)
+      setTimeout(() => setActiveSpeakerStore(agentId, false), result.ms + 500)
     }).catch((err) => {
       console.error(`[voice] makeTts failed:`, err instanceof Error ? err.message : err)
+      clearTimeout(safetyClear)
+      setActiveSpeakerStore(agentId, false)
     })
 
     return { ok: true, voiceId, estimatedMs }

src/chat.ts

@@ -14,6 +14,7 @@ import { getDb, importJsonlIfNeeded, safeJsonParse, safeJsonStringify } from './
 import type Database from 'better-sqlite3'
 import { suppressionLedger } from './suppression-ledger.js'
 import { triggerVoiceOnChat } from './voice-on-chat.js'
+import { triggerHandoffAck } from './voice-ack-on-handoff.js'
 // OpenClaw integration pending — chat works standalone for now
 
 const MESSAGES_FILE = join(DATA_DIR, 'messages.jsonl')
@@ -525,6 +526,11 @@ class ChatManager {
     // human is present. Fires /canvas/speak (existing path); never blocks.
     triggerVoiceOnChat(fullMessage)
 
+    // Audible ack when this message is a real `/handoff to=...` and the
+    // recipient is voiceCapable + a human is present + recipient is not
+    // already mid-TTS. Recipient-authored 1–4 word ack via Anthropic.
+    triggerHandoffAck(fullMessage)
+
     // Route to agent inboxes (auto-routing)
     this.routeToInboxes(fullMessage)
 

src/server.ts

@@ -21,6 +21,7 @@ import type { ModelsEnvelope } from './openclaw-models-types.js'
 import { getStallDetector, emitWorkflowStall, onStallEvent } from './stall-detector.js'
 import { processStallEvent } from './intervention-template.js'
 import { trackRequest, getRequestMetrics } from './request-tracker.js'
+import { setActiveSpeaker as setActiveSpeakerStore } from './active-speakers.js'
 import { getPreflightMetrics, snapshotDailyMetrics, getDailySnapshots, startAutoSnapshot } from './alert-preflight.js'
 
 // ── Build info (read once at startup) ──────────────────────────────────────
@@ -8873,6 +8874,7 @@ export async function createServer(): Promise<FastifyInstance> {
 
     // Helper: push activeSpeaker signal into canvas state so orb reacts
     const setActiveSpeaker = (active: boolean) => {
+      setActiveSpeakerStore(agentId, active)
       const existing = canvasStateMap.get(agentId)
       if (existing) {
         canvasStateMap.set(agentId, {
@@ -9170,6 +9172,7 @@ export async function createServer(): Promise<FastifyInstance> {
     const identityColor = getIdentityColor(agentId)
 
     const setActiveSpeakerAudio = (active: boolean) => {
+      setActiveSpeakerStore(agentId, active)
       const existing = canvasStateMap.get(agentId)
       if (existing) {
         canvasStateMap.set(agentId, { ...existing, payload: { ...(existing.payload as Record<string, unknown>), activeSpeaker: active }, updatedAt: Date.now() })

src/voice-ack-on-handoff.test.ts

@@ -0,0 +1,219 @@
+// SPDX-License-Identifier: Apache-2.0
+import { describe, it } from 'node:test'
+import assert from 'node:assert/strict'
+
+import {
+  parseHandoffTarget,
+  decideHandoffAck,
+  clampAckLine,
+  triggerHandoffAck,
+} from './voice-ack-on-handoff.js'
+import type { AgentMessage } from './types.js'
+
+function msg(partial: Partial<AgentMessage>): AgentMessage {
+  return {
+    id: partial.id ?? 'm1',
+    from: partial.from ?? 'kai',
+    content: partial.content ?? '',
+    timestamp: partial.timestamp ?? Date.now(),
+    channel: partial.channel,
+    to: partial.to,
+  }
+}
+
+describe('parseHandoffTarget', () => {
+  it('parses real /handoff to=name', () => {
+    assert.equal(parseHandoffTarget('/handoff to=link'), 'link')
+    // Mixed-case input must lowercase: parser canonicalizes target.
+    assert.equal(parseHandoffTarget('/handoff to=PiXeL please look'), 'pixel')
+    assert.equal(parseHandoffTarget('  /handoff   to=pixel  '), 'pixel')
+  })
+
+  it('rejects typos in cloud SDK accept-set (strict on node side)', () => {
+    assert.equal(parseHandoffTarget('/handof to=link'), null)
+    assert.equal(parseHandoffTarget('/andoff to=link'), null)
+    assert.equal(parseHandoffTarget('/andof to=link'), null)
+  })
+
+  it('rejects when to=... is missing', () => {
+    assert.equal(parseHandoffTarget('/handoff link'), null)
+    assert.equal(parseHandoffTarget('/handoff'), null)
+  })
+
+  it('rejects malformed recipient names', () => {
+    assert.equal(parseHandoffTarget('/handoff to=2pac'), null)
+    assert.equal(parseHandoffTarget('/handoff to=-link'), null)
+    assert.equal(parseHandoffTarget('/handoff to='), null)
+  })
+
+  it('does not match prefix without whitespace boundary', () => {
+    assert.equal(parseHandoffTarget('/handoffish to=link'), null)
+  })
+
+  it('ignores embedded mentions or follow-up text', () => {
+    assert.equal(parseHandoffTarget('@link please look at this'), null)
+    assert.equal(parseHandoffTarget('hey, can you /handoff to=link?'), null)
+  })
+})
+
+describe('decideHandoffAck — gate order', () => {
+  const baseInput = {
+    humansInRoom: 1,
+    recipientVoiceCapable: true,
+    recipientActiveSpeaker: false,
+  }
+
+  it('willAck=true on the happy path', () => {
+    const d = decideHandoffAck({ ...baseInput, message: msg({ from: 'kai', content: '/handoff to=link' }) })
+    assert.equal(d.willAck, true)
+    assert.equal(d.recipient, 'link')
+  })
+
+  it('rejects non-general channels', () => {
+    const d = decideHandoffAck({ ...baseInput, message: msg({ from: 'kai', channel: 'dm:a_b', content: '/handoff to=link' }) })
+    assert.equal(d.willAck, false)
+    assert.equal(d.reason, 'channel-not-general')
+  })
+
+  it('rejects system-authored handoffs', () => {
+    const d = decideHandoffAck({ ...baseInput, message: msg({ from: 'system', content: '/handoff to=link' }) })
+    assert.equal(d.willAck, false)
+    assert.equal(d.reason, 'sender-system')
+  })
+
+  it('rejects when content is not a real /handoff', () => {
+    const d = decideHandoffAck({ ...baseInput, message: msg({ from: 'kai', content: 'thanks @link' }) })
+    assert.equal(d.willAck, false)
+    assert.equal(d.reason, 'not-handoff')
+  })
+
+  it('rejects self-handoff', () => {
+    const d = decideHandoffAck({ ...baseInput, message: msg({ from: 'link', content: '/handoff to=link' }) })
+    assert.equal(d.willAck, false)
+    assert.equal(d.reason, 'self-handoff')
+  })
+
+  it('rejects when recipient is not voice-capable', () => {
+    const d = decideHandoffAck({ ...baseInput, recipientVoiceCapable: false, message: msg({ from: 'kai', content: '/handoff to=link' }) })
+    assert.equal(d.willAck, false)
+    assert.equal(d.reason, 'recipient-not-voicecapable')
+  })
+
+  it('rejects when no humans are in the room', () => {
+    const d = decideHandoffAck({ ...baseInput, humansInRoom: 0, message: msg({ from: 'kai', content: '/handoff to=link' }) })
+    assert.equal(d.willAck, false)
+    assert.equal(d.reason, 'no-humans-in-room')
+  })
+
+  it('rejects when recipient is already mid-TTS', () => {
+    const d = decideHandoffAck({ ...baseInput, recipientActiveSpeaker: true, message: msg({ from: 'kai', content: '/handoff to=link' }) })
+    assert.equal(d.willAck, false)
+    assert.equal(d.reason, 'recipient-already-speaking')
+  })
+})
+
+describe('clampAckLine', () => {
+  it('returns 1–4 word ack and trims terminal punctuation', () => {
+    assert.equal(clampAckLine('on it'), 'on it')
+    assert.equal(clampAckLine('on it!'), 'on it')
+    assert.equal(clampAckLine('  got it.  '), 'got it')
+  })
+
+  it('truncates >4 word lines to first 4 words', () => {
+    assert.equal(clampAckLine('yes I am taking this one right now'), 'yes I am taking')
+  })
+
+  it('strips wrapping quotes', () => {
+    assert.equal(clampAckLine('"on it"'), 'on it')
+    assert.equal(clampAckLine('“taking that”'), 'taking that')
+  })
+
+  it('uses only the first non-empty line', () => {
+    assert.equal(clampAckLine('\non it\nmore stuff'), 'on it')
+  })
+
+  it('returns empty for empty input', () => {
+    assert.equal(clampAckLine(''), '')
+    assert.equal(clampAckLine('   '), '')
+  })
+})
+
+describe('triggerHandoffAck — wiring', () => {
+  it('does NOT speak when message is not a /handoff (fast-path skip)', () => {
+    let spoken = false
+    triggerHandoffAck(msg({ from: 'kai', content: 'hi @link' }), {
+      humansInRoom: () => 5,
+      voiceCapable: () => true,
+      activeSpeaker: () => false,
+      ackLine: async () => 'on it',
+      speak: async () => { spoken = true },
+    })
+    assert.equal(spoken, false)
+  })
+
+  it('does NOT speak when gate fails (no humans)', async () => {
+    let spoken = false
+    triggerHandoffAck(msg({ from: 'kai', content: '/handoff to=link' }), {
+      humansInRoom: () => 0,
+      voiceCapable: () => true,
+      activeSpeaker: () => false,
+      ackLine: async () => 'on it',
+      speak: async () => { spoken = true },
+    })
+    // Async path is fire-and-forget; gate runs synchronously, but speak
+    // would have been queued via void IIFE. Wait a tick.
+    await new Promise(r => setTimeout(r, 50))
+    assert.equal(spoken, false)
+  })
+
+  it('speaks the recipient-authored ack on happy path', async () => {
+    let spokenText = ''
+    let spokenAgent = ''
+    triggerHandoffAck(msg({ from: 'kai', content: '/handoff to=link please look' }), {
+      humansInRoom: () => 2,
+      voiceCapable: () => true,
+      activeSpeaker: () => false,
+      ackLine: async () => 'on it',
+      speak: async (text, agentId) => { spokenText = text; spokenAgent = agentId },
+    })
+    await new Promise(r => setTimeout(r, 50))
+    assert.equal(spokenText, 'on it')
+    assert.equal(spokenAgent, 'link')
+  })
+
+  it('stays silent when ackLine returns empty (no canned chrome)', async () => {
+    // Simulate "key is configured" path so the empty-ackLine branch goes
+    // silent rather than fall through to the dev-only canned pool.
+    const envKey = 'ANTHROPIC_' + 'API_' + 'KEY'
+    const prev = process.env[envKey]
+    process.env[envKey] = 'k'
+    try {
+      let spoken = false
+      triggerHandoffAck(msg({ from: 'kai', content: '/handoff to=link' }), {
+        humansInRoom: () => 2,
+        voiceCapable: () => true,
+        activeSpeaker: () => false,
+        ackLine: async () => '',
+        speak: async () => { spoken = true },
+      })
+      await new Promise(r => setTimeout(r, 50))
+      assert.equal(spoken, false)
+    } finally {
+      if (prev === undefined) delete process.env[envKey]
+      else process.env[envKey] = prev
+    }
+  })
+
+  it('stays silent when recipient is already speaking', async () => {
+    let spoken = false
+    triggerHandoffAck(msg({ from: 'kai', content: '/handoff to=link' }), {
+      humansInRoom: () => 2,
+      voiceCapable: () => true,
+      activeSpeaker: () => true,
+      ackLine: async () => 'on it',
+      speak: async () => { spoken = true },
+    })
+    await new Promise(r => setTimeout(r, 50))
+    assert.equal(spoken, false)
+  })
+})